Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(graph): stop re-reading the repo during graph build (#1648)

Raghav Chamadiyacef5f83df655
diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.py
index 4ddfeb2ea..33ee37d89 100644
--- a/packages/core/src/repowise/core/fs_walk.py
+++ b/packages/core/src/repowise/core/fs_walk.py
@@ -377,6 +377,25 @@ class WalkSnapshot:
yield dirpath / name
+def glob_via(
+ snapshot: WalkSnapshot | None,
+ root: Path | str,
+ patterns: str | Iterable[str],
+ *,
+ prune_nested_git: bool = True,
+) -> Iterator[Path]:
+ """:func:`iter_glob`, answered from *snapshot* when the caller has one.
+
+ The seam for scans that run several queries against one tree and are
+ reached from a context that can hold the walk. Yield order and matching
+ are identical either way, so a caller cannot behave differently for
+ having been given a snapshot.
+ """
+ if snapshot is not None:
+ return snapshot.iter_glob(root, patterns)
+ return iter_glob(root, patterns, prune_nested_git=prune_nested_git)
+
+
def iter_glob(
root: Path | str,
patterns: str | Iterable[str],
diff --git a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
index adf28e62e..7c6ca9b05 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
@@ -102,7 +102,7 @@ class ResolveMixin:
if progress:
progress.on_phase_start(phase, None)
try:
- cs_texts = collect_csharp_source_texts(self._parsed_files)
+ cs_texts = collect_csharp_source_texts(self._parsed_files, self._source_map)
type_to_file = build_csharp_type_to_file(self._parsed_files)
added = resolve_csharp_member_reads(self._graph, cs_texts, type_to_file)
log.info("member_read_edges", language="csharp", added=added)
@@ -141,7 +141,7 @@ class ResolveMixin:
progress.on_phase_start(phase, None)
try:
jvm_index = get_or_build_jvm_index(ctx)
- texts = collect_jvm_source_texts(self._parsed_files)
+ texts = collect_jvm_source_texts(self._parsed_files, self._source_map)
added = resolve_jvm_same_package_refs(self._graph, jvm_index, texts)
log.info("same_package_edges", added=added)
except Exception as exc:
@@ -178,7 +178,7 @@ class ResolveMixin:
progress.on_phase_start(phase, None)
try:
index = get_or_build_index(ctx)
- cs_texts = collect_csharp_source_texts(self._parsed_files)
+ cs_texts = collect_csharp_source_texts(self._parsed_files, self._source_map)
repo = getattr(index, "repo_path", None) if index is not None else None
added = resolve_csharp_same_namespace_refs(
self._graph, index, cs_texts, repo
@@ -410,7 +410,7 @@ class ResolveMixin:
progress.on_phase_start(phase, None)
try:
swift_targets = get_or_build_swift_targets(ctx)
- texts = collect_swift_source_texts(self._parsed_files)
+ texts = collect_swift_source_texts(self._parsed_files, self._source_map)
added = resolve_swift_same_module_refs(self._graph, swift_targets, texts)
log.info("same_module_edges", language="swift", added=added)
except Exception as exc:
diff --git a/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py b/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py
index 3744e6bde..e6e46661b 100644
--- a/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py
+++ b/packages/core/src/repowise/core/ingestion/languages/csharp_member_reads.py
@@ -38,9 +38,9 @@ behind a feature flag if it ever proves noisy.
from __future__ import annotations
import re
-from pathlib import Path
from typing import TYPE_CHECKING, Any
+from ..source_text import source_text
from ..type_names import bare_type_name
if TYPE_CHECKING:
@@ -180,16 +180,22 @@ def build_csharp_type_to_file(parsed_files: dict[str, Any]) -> dict[str, str]:
return result
-def collect_csharp_source_texts(parsed_files: dict[str, Any]) -> dict[str, str]:
- """Read each parsed C# file's source from disk, keyed by repo path."""
+def collect_csharp_source_texts(
+ parsed_files: dict[str, Any], source_map: dict[str, bytes] | None = None
+) -> dict[str, str]:
+ """Text of each parsed C# file, keyed by repo path.
+
+ ``utf-8-sig``, unlike the JVM and Swift collectors: C# files are routinely
+ written with a BOM, and the scans below must not see it inside the first
+ identifier of the file.
+ """
out: dict[str, str] = {}
for path, parsed in parsed_files.items():
if parsed.file_info.language != "csharp":
continue
- try:
- out[path] = Path(parsed.file_info.abs_path).read_text(
- encoding="utf-8-sig", errors="ignore"
- )
- except OSError:
- continue
+ text = source_text(
+ path, parsed.file_info.abs_path, source_map, encoding="utf-8-sig"
+ )
+ if text is not None:
+ out[path] = text
return out
diff --git a/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py b/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py
index f75ac1078..5e420e9f4 100644
--- a/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py
+++ b/packages/core/src/repowise/core/ingestion/languages/jvm_same_package.py
@@ -157,6 +157,8 @@ def resolve_jvm_same_package_refs(
)
-def collect_jvm_source_texts(parsed_files: dict[str, Any]) -> dict[str, str]:
- """Read each parsed JVM file's source from disk, keyed by repo path."""
- return collect_source_texts(parsed_files, _JVM_LANGUAGES)
+def collect_jvm_source_texts(
+ parsed_files: dict[str, Any], source_map: dict[str, bytes] | None = None
+) -> dict[str, str]:
+ """Text of each parsed JVM file, keyed by repo path."""
+ return collect_source_texts(parsed_files, _JVM_LANGUAGES, source_map)
diff --git a/packages/core/src/repowise/core/ingestion/languages/scope_scan.py b/packages/core/src/repowise/core/ingestion/languages/scope_scan.py
index ff87f4dcf..085081e1a 100644
--- a/packages/core/src/repowise/core/ingestion/languages/scope_scan.py
+++ b/packages/core/src/repowise/core/ingestion/languages/scope_scan.py
@@ -12,9 +12,10 @@ from __future__ import annotations
import re
from collections.abc import Callable, Collection, Iterable
from dataclasses import dataclass, field
-from pathlib import Path
from typing import TYPE_CHECKING, Any
+from repowise.core.ingestion.source_text import source_text
+
if TYPE_CHECKING:
import networkx as nx
@@ -112,17 +113,16 @@ def emit_scope_edges(
def collect_source_texts(
- parsed_files: dict[str, Any], languages: Collection[str]
+ parsed_files: dict[str, Any],
+ languages: Collection[str],
+ source_map: dict[str, bytes] | None = None,
) -> dict[str, str]:
- """Read each parsed file of *languages* from disk, keyed by repo path."""
+ """Text of each parsed file of *languages*, keyed by repo path."""
out: dict[str, str] = {}
for path, parsed in parsed_files.items():
if parsed.file_info.language not in languages:
continue
- try:
- out[path] = Path(parsed.file_info.abs_path).read_text(
- encoding="utf-8", errors="ignore"
- )
- except OSError:
- continue
+ text = source_text(path, parsed.file_info.abs_path, source_map)
+ if text is not None:
+ out[path] = text
return out
diff --git a/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py b/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py
index 79398bea8..c8599f6bc 100644
--- a/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py
+++ b/packages/core/src/repowise/core/ingestion/languages/swift_same_module.py
@@ -144,6 +144,8 @@ def resolve_swift_same_module_refs(
)
-def collect_swift_source_texts(parsed_files: dict[str, Any]) -> dict[str, str]:
- """Read each parsed Swift file's source from disk, keyed by repo path."""
- return collect_source_texts(parsed_files, ("swift",))
+def collect_swift_source_texts(
+ parsed_files: dict[str, Any], source_map: dict[str, bytes] | None = None
+) -> dict[str, str]:
+ """Text of each parsed Swift file, keyed by repo path."""
+ return collect_source_texts(parsed_files, ("swift",), source_map)
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/context.py b/packages/core/src/repowise/core/ingestion/resolvers/context.py
index b7c0c4d14..a9fa9761b 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/context.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/context.py
@@ -60,6 +60,38 @@ class ResolverContext:
default=None, init=False, repr=False, compare=False
)
+ # One pruned walk of the repo, replayed for every resolver glob that asks
+ # for it. Built on first use and never from ``path_set``: these scans look
+ # for files the traverser does not index — ``META-INF/services`` entries,
+ # ``spring.factories``, ``*.sln`` — so a snapshot of the indexed set would
+ # answer "absent" for exactly the files they exist to find.
+ _walk_snapshot_cache: Any | None = field(
+ default=None, init=False, repr=False, compare=False
+ )
+
+ @property
+ def walk_snapshot(self) -> Any | None:
+ """Shared :class:`~repowise.core.fs_walk.WalkSnapshot`, or None.
+
+ Rooted at the *resolved* repo path, because every scan that asks for
+ this resolves its own root first; rooting it anywhere else would leave
+ the snapshot correct but unused — a query it cannot place falls back to
+ a live walk, which is exactly the cost this removes.
+ """
+ if self._walk_snapshot_cache is None:
+ if self.repo_path is None:
+ return None
+ from repowise.core.fs_walk import WalkSnapshot
+
+ try:
+ root = self.repo_path.resolve()
+ except OSError:
+ root = self.repo_path
+ self._walk_snapshot_cache = WalkSnapshot(
+ root, prune_nested_git=self.prune_nested_git
+ )
+ return self._walk_snapshot_cache
+
@property
def sorted_paths(self) -> tuple[str, ...]:
"""Deterministically ordered view of ``path_set`` (cached)."""
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py
index 127a7f544..14e297705 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/cpp_workspace.py
@@ -49,6 +49,7 @@ from ..external_systems.cmake import (
discover_cmake_reactor,
parse_cmake_file_api_reply,
)
+from ..source_text import decode_source
if TYPE_CHECKING:
from .context import ResolverContext
@@ -238,15 +239,19 @@ _PROJECT_EXPORT_MACRO_RE = re.compile(
_EXPORT_LIKE_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]{2,}_(?:EXPORT|API|PUBLIC|DLL|VISIBLE)$")
+def _export_macros_in(text: str) -> tuple[str, ...]:
+ # Limit to the first ~8KB — export macros are nearly always near the top.
+ return tuple({m.group(1) for m in _PROJECT_EXPORT_MACRO_RE.finditer(text[:8192])})
+
+
@lru_cache(maxsize=8192)
def _scan_header_for_export_macros(abs_path: str) -> tuple[str, ...]:
+ """:func:`_export_macros_in` for a header the source map did not have."""
try:
text = Path(abs_path).read_text(encoding="utf-8", errors="ignore")
except OSError:
return ()
- # Limit to the first ~8KB — export macros are nearly always near the top.
- head = text[:8192]
- return tuple({m.group(1) for m in _PROJECT_EXPORT_MACRO_RE.finditer(head)})
+ return _export_macros_in(text)
def _classify_dir(root_dir: str) -> dict[str, bool]:
@@ -362,6 +367,11 @@ def build_cpp_workspace_index(ctx: ResolverContext) -> CppWorkspaceIndex:
repo_path = ctx.repo_path.resolve()
path_set = set(ctx.path_set)
+ # ``getattr``, not attribute access: the call resolver builds these
+ # indexes from a minimal stand-in context that carries neither field
+ # (``call_resolver._Ctx``). A miss means disk and a live walk, which
+ # is what that path did before.
+ source_map = getattr(ctx, "source_map", None)
cmake_files = discover_cmake_reactor(repo_path)
file_api_targets = parse_cmake_file_api_reply(repo_path)
@@ -409,8 +419,11 @@ def build_cpp_workspace_index(ctx: ResolverContext) -> CppWorkspaceIndex:
if len(header_candidates) > 200:
break
for rel in header_candidates:
- abs_p = str((repo_path / rel).resolve())
- macros.update(_scan_header_for_export_macros(abs_p))
+ data = source_map.get(rel) if source_map is not None else None
+ if data is not None:
+ macros.update(_export_macros_in(decode_source(data)))
+ else:
+ macros.update(_scan_header_for_export_macros(str((repo_path / rel).resolve())))
_scan_header_for_export_macros.cache_clear()
# Filter out anything that doesn't look like an export marker name.
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dart.py b/packages/core/src/repowise/core/ingestion/resolvers/dart.py
index c91694a33..0a92bdec8 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dart.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dart.py
@@ -21,6 +21,7 @@ import posixpath
import re
from typing import TYPE_CHECKING
+from ..source_text import source_text
from .module_name_index import get_or_build_module_index, lookup_module
if TYPE_CHECKING:
@@ -42,9 +43,10 @@ def _pubspec_map(ctx: ResolverContext) -> dict[str, str]:
continue
if repo is None:
continue
- try:
- text = (repo / path).read_text(encoding="utf-8", errors="replace")
- except OSError:
+ text = source_text(
+ path, repo / path, getattr(ctx, "source_map", None), errors="replace"
+ )
+ if text is None:
continue
m = _PUBSPEC_NAME_RE.search(text)
if m:
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py
index c401009d6..2bb5566d9 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py
@@ -4,11 +4,12 @@ from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
import structlog
-from repowise.core.fs_walk import iter_glob
+from repowise.core.fs_walk import glob_via
+from repowise.core.ingestion.source_text import source_text
from .global_usings import collect_project_global_usings
from .msbuild import (
@@ -156,7 +157,9 @@ class DotNetProjectIndex:
return package_id in self.package_refs.get(csproj, set())
-def _walk_repo_cs_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:
+def _walk_repo_cs_files(
+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None
+) -> list[Path]:
"""Single repo-wide rglob for ``*.cs`` files, dedup by resolved path.
Lives at module scope (not nested inside ``build_index``) so it's
@@ -165,7 +168,7 @@ def _walk_repo_cs_files(repo_path: Path, *, prune_nested_git: bool = True) -> li
"""
seen: set[Path] = set()
out: list[Path] = []
- for cs in iter_glob(repo_path, "*.cs", prune_nested_git=prune_nested_git):
+ for cs in glob_via(snapshot, repo_path, "*.cs", prune_nested_git=prune_nested_git):
if path_has_dotnet_scan_skip_dir(cs, repo_path):
continue
try:
@@ -214,7 +217,13 @@ def _bucket_files_by_project(
return out
-def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjectIndex:
+def build_index(
+ repo_path: Path,
+ *,
+ prune_nested_git: bool = True,
+ snapshot: Any | None = None,
+ source_map: dict[str, bytes] | None = None,
+) -> DotNetProjectIndex:
"""Walk *repo_path* and construct a fully-populated DotNetProjectIndex.
Performance note: a previous version of this function walked the
@@ -227,12 +236,20 @@ def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProj
ONE master walk, reads each file ONCE, and dispatches the cached
texts to both the namespace-map pass and per-project global-usings
collection. No data-quality loss — same regexes, same outputs.
+
+ That master read is now itself avoidable: ingestion already read every
+ indexed ``.cs`` file, so *source_map* answers most of it and only the
+ files the traverser skipped reach the disk. *snapshot* does the same for
+ the three walks — .csproj, .sln, .cs — which become dict replays of the
+ context's single walk.
"""
repo_path = repo_path.resolve()
index = DotNetProjectIndex(repo_path=repo_path)
# ---- 1. Parse every .csproj ----
- for csproj_path in find_csproj_files(repo_path, prune_nested_git=prune_nested_git):
+ for csproj_path in find_csproj_files(
+ repo_path, prune_nested_git=prune_nested_git, snapshot=snapshot
+ ):
proj = parse_csproj(csproj_path)
if proj is None:
continue
@@ -241,7 +258,9 @@ def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProj
index.package_refs[proj.path] = set(proj.package_references)
# ---- 2. Walk .sln files (informational; surfaces orphaned .csprojs) ----
- index.sln_paths = find_sln_files(repo_path, prune_nested_git=prune_nested_git)
+ index.sln_paths = find_sln_files(
+ repo_path, prune_nested_git=prune_nested_git, snapshot=snapshot
+ )
for sln in index.sln_paths:
for entry in parse_sln(sln):
if entry.csproj not in index.projects:
@@ -255,13 +274,21 @@ def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProj
index.package_refs.setdefault(proj.path, set()).update(proj.package_references)
# ---- 3. Single master walk: enumerate .cs files & read each once ----
- all_cs_files = _walk_repo_cs_files(repo_path, prune_nested_git=prune_nested_git)
+ all_cs_files = _walk_repo_cs_files(
+ repo_path, prune_nested_git=prune_nested_git, snapshot=snapshot
+ )
cs_texts: dict[Path, str] = {}
for f in all_cs_files:
+ # ``_walk_repo_cs_files`` resolves every path and *repo_path* is
+ # resolved too, so a file the traverser indexed keys the map exactly.
+ # Files it skipped — and anything outside the root — miss and read.
try:
- cs_texts[f] = f.read_text(encoding="utf-8-sig", errors="replace")
- except OSError:
- continue
+ rel: str | None = f.relative_to(repo_path).as_posix()
+ except ValueError:
+ rel = None
+ text = source_text(rel, f, source_map, encoding="utf-8-sig", errors="replace")
+ if text is not None:
+ cs_texts[f] = text
# Build the file → project map by longest-prefix match. The bucketer
# walks each file's parent chain (deepest-first) against a dict of
@@ -326,6 +353,11 @@ def get_or_build_index(ctx: ResolverContext) -> DotNetProjectIndex | None:
cached = getattr(ctx, _INDEX_KEY, None)
if cached is not None:
return cached
- index = build_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
+ index = build_index(
+ ctx.repo_path,
+ prune_nested_git=ctx.prune_nested_git,
+ snapshot=getattr(ctx, "walk_snapshot", None),
+ source_map=getattr(ctx, "source_map", None),
+ )
setattr(ctx, _INDEX_KEY, index)
return index
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py
index 58f91bfc2..30ebf7aa6 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py
@@ -10,11 +10,12 @@ from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
+from typing import Any
from xml.etree import ElementTree as ET
import structlog
-from repowise.core.fs_walk import iter_glob
+from repowise.core.fs_walk import glob_via
log = structlog.get_logger(__name__)
@@ -113,10 +114,14 @@ def parse_csproj(csproj_path: Path) -> MSBuildProject | None:
return project
-def find_csproj_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:
+def find_csproj_files(
+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None
+) -> list[Path]:
"""Return all .csproj files under *repo_path*, skipping bin/obj output."""
out: list[Path] = []
- for csproj in iter_glob(repo_path, "*.csproj", prune_nested_git=prune_nested_git):
+ for csproj in glob_via(
+ snapshot, repo_path, "*.csproj", prune_nested_git=prune_nested_git
+ ):
if path_has_dotnet_scan_skip_dir(csproj, repo_path):
continue
out.append(csproj)
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py
index 1821f062a..eb7f73ffc 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py
@@ -13,10 +13,11 @@ from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
+from typing import Any
import structlog
-from repowise.core.fs_walk import iter_glob
+from repowise.core.fs_walk import glob_via
from .msbuild import path_has_dotnet_scan_skip_dir
@@ -60,9 +61,11 @@ def parse_sln(sln_path: Path) -> list[SolutionEntry]:
return out
-def find_sln_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:
+def find_sln_files(
+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None
+) -> list[Path]:
out: list[Path] = []
- for sln in iter_glob(repo_path, "*.sln", prune_nested_git=prune_nested_git):
+ for sln in glob_via(snapshot, repo_path, "*.sln", prune_nested_git=prune_nested_git):
if path_has_dotnet_scan_skip_dir(sln, repo_path):
continue
out.append(sln)
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py
index 4260ba623..da79ecee7 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/go_workspace.py
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING
import structlog
+from ..source_text import source_text
from .go import read_go_modules
if TYPE_CHECKING:
@@ -133,10 +134,10 @@ def _scan_go_file(text: str) -> tuple[str, bool, bool, bool, bool]:
def _read_text(ctx: ResolverContext, rel_path: str) -> str:
if ctx.repo_path is None:
return ""
- try:
- return (ctx.repo_path / rel_path).read_text(encoding="utf-8", errors="ignore")
- except OSError:
- return ""
+ # ``getattr``: the call resolver builds this index from a stand-in
+ # context with no source map (``call_resolver._Ctx``).
+ source_map = getattr(ctx, "source_map", None)
+ return source_text(rel_path, ctx.repo_path / rel_path, source_map) or ""
def _import_path_for_dir(
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py
index d574983fd..4539c3043 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py
@@ -15,11 +15,12 @@ import re
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
import structlog
-from repowise.core.fs_walk import iter_glob
+from repowise.core.fs_walk import glob_via
+from repowise.core.ingestion.source_text import decode_source
if TYPE_CHECKING:
from .context import ResolverContext
@@ -251,20 +252,17 @@ def _detect_plugins(module_dir: Path) -> frozenset[str]:
# Package extraction (cached per file)
# ---------------------------------------------------------------------------
-@lru_cache(maxsize=8192)
-def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]:
+def _extract_text_info(
+ abs_path: str, text: str | None
+) -> tuple[str, tuple[str, ...], bool, bool]:
"""Return (package_decl, top_level_types, is_module_info, is_package_info).
- Reads the file once and caches the result. Cheap line scan, no AST.
+ Cheap line scan, no AST. *text* of None means the file was unreadable.
"""
- path = Path(abs_path)
- name = path.name
+ name = Path(abs_path).name
is_module_info = name == "module-info.java"
is_package_info = name == "package-info.java"
-
- try:
- text = path.read_text(encoding="utf-8", errors="ignore")
- except OSError:
+ if text is None:
return "", (), is_module_info, is_package_info
package = ""
@@ -284,12 +282,26 @@ def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]
return package, tuple(top_level), is_module_info, is_package_info
+@lru_cache(maxsize=8192)
+def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]:
+ """:func:`_extract_text_info` for a file the source map did not have."""
+ try:
+ text: str | None = Path(abs_path).read_text(encoding="utf-8", errors="ignore")
+ except OSError:
+ text = None
+ return _extract_text_info(abs_path, text)
+
+
# ---------------------------------------------------------------------------
# Index building
# ---------------------------------------------------------------------------
def build_jvm_gradle_index(
- repo_path: Path | None, *, prune_nested_git: bool = True
+ repo_path: Path | None,
+ *,
+ prune_nested_git: bool = True,
+ snapshot: Any | None = None,
+ source_map: dict[str, bytes] | None = None,
) -> JvmGradleIndex:
"""Walk Gradle config + JVM sources to build the package-resolution index."""
index = JvmGradleIndex()
@@ -351,14 +363,23 @@ def build_jvm_gradle_index(
continue
rel_roots.append(rel)
- for jvm_file in iter_glob(root_path, "*", prune_nested_git=prune_nested_git):
+ for jvm_file in glob_via(
+ snapshot, root_path, "*", prune_nested_git=prune_nested_git
+ ):
if jvm_file.suffix not in _JVM_EXTENSIONS or not jvm_file.is_file():
continue
try:
rel_file = jvm_file.relative_to(resolved_repo).as_posix()
except ValueError:
continue
- package, _types, _is_mod, _is_pkg = _extract_file_info(str(jvm_file.resolve()))
+ abs_file = str(jvm_file.resolve())
+ data = source_map.get(rel_file) if source_map is not None else None
+ if data is not None:
+ package, _types, _is_mod, _is_pkg = _extract_text_info(
+ abs_file, decode_source(data)
+ )
+ else:
+ package, _types, _is_mod, _is_pkg = _extract_file_info(abs_file)
if not package:
continue
index.package_to_files.setdefault(package, []).append(rel_file)
@@ -382,7 +403,12 @@ def get_or_build_jvm_gradle_index(ctx: ResolverContext) -> JvmGradleIndex:
cached = getattr(ctx, "_jvm_gradle_index", None)
if cached is not None:
return cached
- index = build_jvm_gradle_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
+ index = build_jvm_gradle_index(
+ ctx.repo_path,
+ prune_nested_git=ctx.prune_nested_git,
+ snapshot=getattr(ctx, "walk_snapshot", None),
+ source_map=getattr(ctx, "source_map", None),
+ )
ctx._jvm_gradle_index = index # type: ignore[attr-defined]
return index
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py
index 424162004..ae676f02d 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py
@@ -18,11 +18,12 @@ import re
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
import structlog
-from repowise.core.fs_walk import iter_glob
+from repowise.core.fs_walk import glob_via
+from repowise.core.ingestion.source_text import decode_source
if TYPE_CHECKING:
from .context import ResolverContext
@@ -206,30 +207,28 @@ class JvmWorkspaceIndex:
return bool(len(parts) == 1 and parts[0] in _JAVA_LANG_TYPES)
-@lru_cache(maxsize=16384)
-def _scan_jvm_file(abs_path: str) -> tuple[str, tuple[str, ...]]:
- """Return (package_fqn, top_level_type_names) for a JVM source file.
+def _scan_jvm_text(abs_path: str, text: str) -> tuple[str, tuple[str, ...]]:
+ """Return (package_fqn, top_level_type_names) from already-read text.
- Reads the file once and caches the result. Cheap line scan — no AST.
+ Cheap line scan — no AST.
"""
- try:
- text = Path(abs_path).read_text(encoding="utf-8", errors="ignore")
- except OSError:
- return "", ()
-
- package = ""
if abs_path.endswith(".scala"):
package = _scala_package(text)
else:
pkg_match = _PACKAGE_RE.search(text)
- if pkg_match:
- package = pkg_match.group(1)
+ package = pkg_match.group(1) if pkg_match else ""
- types: list[str] = []
- for m in _TOP_LEVEL_TYPE_RE.finditer(text):
- types.append(m.group(1))
+ return package, tuple(m.group(1) for m in _TOP_LEVEL_TYPE_RE.finditer(text))
- return package, tuple(types)
+
+@lru_cache(maxsize=16384)
+def _scan_jvm_file(abs_path: str) -> tuple[str, tuple[str, ...]]:
+ """:func:`_scan_jvm_text` for a file the source map did not have."""
+ try:
+ text = Path(abs_path).read_text(encoding="utf-8", errors="ignore")
+ except OSError:
+ return "", ()
+ return _scan_jvm_text(abs_path, text)
def _scala_package(text: str) -> str:
@@ -260,7 +259,7 @@ _JPMS_PROVIDES_RE = re.compile(
def _scan_jpms_provides(
- repo_path: Path, *, prune_nested_git: bool = True
+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None
) -> dict[str, tuple[str, ...]]:
"""Scan ``module-info.java`` files for ``provides X with Y, Z`` directives.
@@ -271,7 +270,9 @@ def _scan_jpms_provides(
the warmup fast.
"""
out: dict[str, list[str]] = {}
- for mi in iter_glob(repo_path, "module-info.java", prune_nested_git=prune_nested_git):
+ for mi in glob_via(
+ snapshot, repo_path, "module-info.java", prune_nested_git=prune_nested_git
+ ):
if not mi.is_file():
continue
try:
@@ -287,12 +288,12 @@ def _scan_jpms_provides(
def _scan_meta_inf_services(
- repo_path: Path, *, prune_nested_git: bool = True
+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None
) -> dict[str, tuple[str, ...]]:
"""Scan META-INF/services/ directories for SPI declarations."""
services: dict[str, list[str]] = {}
- for services_dir in iter_glob(
- repo_path, "META-INF/services", prune_nested_git=prune_nested_git
+ for services_dir in glob_via(
+ snapshot, repo_path, "META-INF/services", prune_nested_git=prune_nested_git
):
if not services_dir.is_dir():
continue
@@ -321,14 +322,14 @@ def _scan_meta_inf_services(
def _scan_spring_autoconfig(
- repo_path: Path, *, prune_nested_git: bool = True
+ repo_path: Path, *, prune_nested_git: bool = True, snapshot: Any | None = None
) -> dict[str, tuple[str, ...]]:
"""Scan spring.factories and Boot 3 AutoConfiguration.imports."""
result: dict[str, list[str]] = {}
# spring.factories (Boot 2 style)
- for factories in iter_glob(
- repo_path, "META-INF/spring.factories", prune_nested_git=prune_nested_git
+ for factories in glob_via(
+ snapshot, repo_path, "META-INF/spring.factories", prune_nested_git=prune_nested_git
):
if not factories.is_file():
continue
@@ -358,8 +359,8 @@ def _scan_spring_autoconfig(
result.setdefault(rel, []).extend(current_values)
# Boot 3 style: META-INF/spring/*.imports
- for imports_file in iter_glob(
- repo_path, "META-INF/spring/*.imports", prune_nested_git=prune_nested_git
+ for imports_file in glob_via(
+ snapshot, repo_path, "META-INF/spring/*.imports", prune_nested_git=prune_nested_git
):
if not imports_file.is_file():
continue
@@ -382,9 +383,11 @@ def _scan_spring_autoconfig(
def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:
"""Build the JVM workspace index from all ``.java`` and ``.kt`` files in the path set.
- One walk over the path set; each file read at most once (via
- ``_scan_jvm_file`` LRU cache). The index groups files by package
- and builds FQN → file mappings.
+ One pass over the path set, taking each file's text from the source map
+ the pipeline already filled and falling back to disk per file. The three
+ resource scans below share the context's single walk rather than each
+ walking the tree again — on a large JVM repo those walks, not the file
+ scan, were the bulk of this function.
"""
index = JvmWorkspaceIndex()
@@ -392,6 +395,12 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:
return index
repo_path = ctx.repo_path.resolve()
+ # ``getattr``, not attribute access: the call resolver builds these
+ # indexes from a minimal stand-in context that carries neither field
+ # (``call_resolver._Ctx``). A miss means disk and a live walk, which
+ # is what that path did before.
+ source_map = getattr(ctx, "source_map", None)
+ snapshot = getattr(ctx, "walk_snapshot", None)
# Group files by package
pkg_files: dict[str, list[str]] = {}
@@ -402,7 +411,13 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:
continue
abs_path = str((repo_path / path).resolve())
- package, top_types = _scan_jvm_file(abs_path)
+ # The map lookup is here, not inside the cached scan: an lru_cache
+ # cannot take a dict, and the fallback still wants its cache.
+ data = source_map.get(path) if source_map is not None else None
+ if data is not None:
+ package, top_types = _scan_jvm_text(abs_path, decode_source(data))
+ else:
+ package, top_types = _scan_jvm_file(abs_path)
if not package:
continue
@@ -432,8 +447,12 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:
# (cheap glob, O(matching files)). Both populate ``services``; later
# phases treat any FQN listed there as reachable.
try:
- services = _scan_meta_inf_services(repo_path, prune_nested_git=ctx.prune_nested_git)
- jpms = _scan_jpms_provides(repo_path, prune_nested_git=ctx.prune_nested_git)
+ services = _scan_meta_inf_services(
+ repo_path, prune_nested_git=ctx.prune_nested_git, snapshot=snapshot
+ )
+ jpms = _scan_jpms_provides(
+ repo_path, prune_nested_git=ctx.prune_nested_git, snapshot=snapshot
+ )
merged: dict[str, list[str]] = {k: list(v) for k, v in services.items()}
for iface, impls in jpms.items():
merged.setdefault(iface, []).extend(impls)
@@ -442,7 +461,7 @@ def build_jvm_workspace_index(ctx: ResolverContext) -> JvmWorkspaceIndex:
pass
with contextlib.suppress(Exception):
index.autoconfig_imports = _scan_spring_autoconfig(
- repo_path, prune_nested_git=ctx.prune_nested_git
+ repo_path, prune_nested_git=ctx.prune_nested_git, snapshot=snapshot
)
_scan_jvm_file.cache_clear()
diff --git a/packages/core/src/repowise/core/ingestion/source_text.py b/packages/core/src/repowise/core/ingestion/source_text.py
new file mode 100644
index 000000000..93abe1634
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/source_text.py
@@ -0,0 +1,59 @@
+"""One reader for file text the ingestion pipeline has already read.
+
+The pipeline reads every indexed file, hands the bytes to
+``GraphBuilder.set_source_map``, and they reach the resolvers as
+``ResolverContext.source_map``. The scans that run during ``build()`` used to
+re-open the repo anyway. This is the single place that prefers those bytes.
+
+Two things a caller must not get wrong, which is why they live here rather than
+at each site:
+
+* **The codec is the caller's, not a default.** JVM and Swift scans decode
+ utf-8, C# decodes utf-8-sig — only C# strips a BOM — and a couple of the
+ .NET readers pass ``errors="replace"`` where the rest pass ``"ignore"``.
+ Decoding a caller's bytes with another caller's codec silently changes what
+ its regexes match, and therefore which edges it emits.
+* **``Path.read_text`` translates newlines and ``bytes.decode`` does not.**
+ Text mode maps ``\\r\\n`` and lone ``\\r`` to ``\\n``; the raw bytes in the
+ source map still carry the ``\\r``. Every CRLF file — which on Windows is
+ most of a C# checkout — would decode to different text than the disk read it
+ replaces. :func:`decode_source` normalises, so a map hit and a disk fallback
+ produce the same string for the same file.
+
+The fallback is not optional. ``pipeline/incremental.py`` fills the map only
+when its caller asks for sources, so it can be partial or absent entirely.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def decode_source(data: bytes, *, encoding: str = "utf-8", errors: str = "ignore") -> str:
+ """Decode *data* the way ``Path.read_text(encoding, errors)`` would."""
+ text = data.decode(encoding, errors=errors)
+ if "\r" in text:
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
+ return text
+
+
+def source_text(
+ rel_path: str | None,
+ abs_path: str | Path,
+ source_map: dict[str, bytes] | None,
+ *,
+ encoding: str = "utf-8",
+ errors: str = "ignore",
+) -> str | None:
+ """Text of one file, from *source_map* if it holds it, else from disk.
+
+ Returns None when the file is neither in the map nor readable.
+ """
+ if source_map is not None and rel_path is not None:
+ data = source_map.get(rel_path)
+ if data is not None:
+ return decode_source(data, encoding=encoding, errors=errors)
+ try:
+ return Path(abs_path).read_text(encoding=encoding, errors=errors)
+ except OSError:
+ return None
diff --git a/tests/unit/ingestion/test_source_text.py b/tests/unit/ingestion/test_source_text.py
new file mode 100644
index 000000000..c422ebd48
--- /dev/null
+++ b/tests/unit/ingestion/test_source_text.py
@@ -0,0 +1,70 @@
+"""A source-map hit and a disk read must produce the same string.
+
+Every scan that moved onto the source map keeps a disk fallback, so the two
+paths run against the same repo in the same build. If they decode differently
+the scan sees different text for the same file depending on whether the
+pipeline happened to fill the map — which shows up as edges appearing and
+disappearing, not as an error.
+
+The failure this pins is not hypothetical: ``Path.read_text`` opens in text
+mode and translates newlines, ``bytes.decode`` does not, and CRLF is the
+majority line ending in a Windows-authored C# or Swift checkout.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.source_text import decode_source, source_text
+
+CASES = [
+ ("lf", b"class A {}\nclass B {}\n"),
+ ("crlf", b"class A {}\r\nclass B {}\r\n"),
+ ("cr", b"class A {}\rclass B {}\r"),
+ ("mixed", b"a\r\nb\rc\n"),
+ ("bom_lf", b"\xef\xbb\xbfclass A {}\n"),
+ ("bom_crlf", b"\xef\xbb\xbfclass A {}\r\n"),
+ ("non_ascii", "// café ☕\r\nclass A {}\r\n".encode()),
+ ("invalid_utf8", b"class A {}\r\n\xff\xfe\r\n"),
+ ("empty", b""),
+]
+
+
[email protected]("label,data", CASES, ids=[c[0] for c in CASES])
[email protected]("encoding", ["utf-8", "utf-8-sig"])
[email protected]("errors", ["ignore", "replace"])
+def test_map_hit_matches_disk_read(tmp_path, label, data, encoding, errors):
+ f = tmp_path / "Sample.cs"
+ f.write_bytes(data)
+
+ from_map = source_text(
+ "Sample.cs", f, {"Sample.cs": data}, encoding=encoding, errors=errors
+ )
+ from_disk = source_text("Sample.cs", f, None, encoding=encoding, errors=errors)
+
+ assert from_map == from_disk
+ assert from_map == f.read_text(encoding=encoding, errors=errors)
+
+
+def test_a_miss_falls_back_to_disk():
+ """An empty or partial map must not read as an empty file."""
+ import tempfile
+ from pathlib import Path
+
+ with tempfile.TemporaryDirectory() as d:
+ f = Path(d) / "Other.cs"
+ f.write_bytes(b"class Other {}\r\n")
+ assert source_text("Other.cs", f, {}) == "class Other {}\n"
+ assert source_text("Other.cs", f, {"Unrelated.cs": b"x"}) == "class Other {}\n"
+
+
+def test_unreadable_file_returns_none(tmp_path):
+ assert source_text("gone.cs", tmp_path / "gone.cs", None) is None
+ assert source_text("gone.cs", tmp_path / "gone.cs", {}) is None
+
+
+def test_only_utf_8_sig_strips_the_bom():
+ """The codecs differ by caller, and only C# asks for the BOM to go."""
+ data = b"\xef\xbb\xbfclass A {}\n"
+ assert decode_source(data, encoding="utf-8").startswith("")
+ assert decode_source(data, encoding="utf-8-sig").startswith("class")