Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(decisions): reuse ingestion's source_map for the inline-marker scan (#913)
Raghav Chamadiyab9555852a4f2
diff --git a/packages/core/src/repowise/core/analysis/decisions/extractor.py b/packages/core/src/repowise/core/analysis/decisions/extractor.pyindex 8e61218a6..dc566098d 100644--- a/packages/core/src/repowise/core/analysis/decisions/extractor.py+++ b/packages/core/src/repowise/core/analysis/decisions/extractor.py@@ -30,7 +30,7 @@ from __future__ import annotationsimport asyncioimport jsonimport re-from collections.abc import Collection+from collections.abc import Collection, Iteratorfrom dataclasses import dataclass, fieldfrom datetime import UTC, datetimefrom pathlib import Path@@ -343,12 +343,19 @@ class DecisionExtractor:graph: Any | None = None,git_meta_map: dict[str, dict] | None = None,parsed_files: list[Any] | None = None,+ source_map: dict[str, bytes] | None = None,) -> None:self._repo_path = Path(repo_path)self._provider = providerself._graph = graphself._git_meta_map = git_meta_map or {}self._parsed_files = parsed_files or []+ # ``source_map`` is ingestion's already-computed {rel_path: bytes} for+ # the indexed file set. When present, the inline-marker scan reuses it+ # for both discovery and reads instead of re-walking the tree and+ # re-reading every file from disk (redundant with ingestion). ``None``+ # keeps the legacy self-walk fallback for callers that don't thread it.+ self._source_map = source_map# ------------------------------------------------------------------# Source 1: Inline markers@@ -361,14 +368,10 @@ class DecisionExtractor:"""Scan source files for decision markers (WHY:, DECISION:, etc.)."""markers_by_file: dict[str, list[dict]] = {}- if restrict_to_files:- files_to_scan = [self._repo_path / fp for fp in restrict_to_files]- else:- files_to_scan = list(self._iter_source_files())-- total_files = len(files_to_scan)+ scan_targets = list(self._iter_scan_targets(restrict_to_files))+ total_files = len(scan_targets)logger.info("decision_extractor.scanning_inline_markers", total_files=total_files)- for idx, file_path in enumerate(files_to_scan):+ for idx, (rel_path, text) in enumerate(scan_targets):if idx > 0 and idx % 1000 == 0:logger.info("decision_extractor.scan_progress",@@ -376,17 +379,11 @@ class DecisionExtractor:total=total_files,markers_found=sum(len(v) for v in markers_by_file.values()),)- if not file_path.is_file():- continue- try:- text = file_path.read_text(encoding="utf-8", errors="replace")- except (OSError, UnicodeDecodeError):- continuelines = text.splitlines()# Track whether we're inside a fenced code block in markdown# files so we don't treat example markers as real decisions.- is_markdown = file_path.suffix.lower() in (".md", ".mdx", ".rst")+ is_markdown = Path(rel_path).suffix.lower() in (".md", ".mdx", ".rst")in_code_fence = Falsefor line_num, line in enumerate(lines, start=1):if is_markdown:@@ -415,11 +412,6 @@ class DecisionExtractor:ctx_end = min(len(lines), line_num + 20)context = "\n".join(lines[ctx_start:ctx_end])- try:- rel_path = str(file_path.relative_to(self._repo_path))- except ValueError:- rel_path = str(file_path)-markers_by_file.setdefault(rel_path, []).append({"keyword": m.group("keyword"),@@ -1500,6 +1492,67 @@ class DecisionExtractor:# Helpers# ------------------------------------------------------------------+ def _iter_scan_targets(+ self, restrict_to_files: list[str] | None+ ) -> Iterator[tuple[str, str]]:+ """Yield ``(rel_path, text)`` for every file the marker scan covers.++ Three sources, in priority order:++ * ``restrict_to_files`` (update path): the caller's explicit change set.+ Text comes from ``source_map`` when the file was just ingested, else a+ targeted disk read; deleted / unreadable paths are skipped.+ * ``source_map`` (init path): ingestion's already-decoded indexed set.+ Discovery AND reads are free — no tree walk, no per-file ``read_text``.+ Paths are POSIX (``FileInfo.path``), matching the graph node keys the+ neighbour lookup joins against.+ * legacy self-walk (``source_map is None``): the original ``os.walk`` ++ git-tracked filter, kept so callers that don't thread ``source_map``+ behave exactly as before.+ """+ if restrict_to_files:+ for rel_path in restrict_to_files:+ text = self._read_source_text(rel_path)+ if text is not None:+ yield rel_path, text+ return++ if self._source_map is not None:+ for rel_path, source in self._source_map.items():+ yield rel_path, source.decode("utf-8", errors="replace")+ return++ for file_path in self._iter_source_files():+ if not file_path.is_file():+ continue+ try:+ text = file_path.read_text(encoding="utf-8", errors="replace")+ except (OSError, UnicodeDecodeError):+ continue+ try:+ rel_path = str(file_path.relative_to(self._repo_path))+ except ValueError:+ rel_path = str(file_path)+ yield rel_path, text++ def _read_source_text(self, rel_path: str) -> str | None:+ """Decode one file's text, preferring ingestion's in-memory bytes.++ Falls back to a disk read (deleted / unreadable → ``None``) so the+ update path stays correct for files that aren't in ``source_map``.+ """+ if self._source_map is not None:+ source = self._source_map.get(rel_path)+ if source is not None:+ return source.decode("utf-8", errors="replace")+ abs_path = self._repo_path / rel_path+ if not abs_path.is_file():+ return None+ try:+ return abs_path.read_text(encoding="utf-8", errors="replace")+ except (OSError, UnicodeDecodeError):+ return None+def _tracked_files(self) -> set[Path] | None:"""Resolved paths git tracks under ``repo_path``, or ``None``.diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 33c3c956c..212cb9f14 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -482,6 +482,7 @@ async def run_pipeline(graph_builder=graph_builder,git_meta_map=git_meta_map,parsed_files=parsed_files,+ source_map=source_map,progress=progress,),)diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.pyindex 1f4e8265b..15297a623 100644--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py@@ -250,6 +250,7 @@ async def _run_decision_extraction(graph_builder: Any,git_meta_map: dict[str, dict],parsed_files: list[Any],+ source_map: dict[str, bytes] | None = None,progress: ProgressCallback | None,) -> Any | None:"""Extract architectural decisions from source and git history."""@@ -274,6 +275,7 @@ async def _run_decision_extraction(graph=graph_builder.graph(),git_meta_map=git_meta_map,parsed_files=parsed_files,+ source_map=source_map,)def _decision_step(_source: str) -> None:diff --git a/tests/unit/analysis/test_decision_source_map_reuse.py b/tests/unit/analysis/test_decision_source_map_reuse.pynew file mode 100644index 000000000..3af3a7332--- /dev/null+++ b/tests/unit/analysis/test_decision_source_map_reuse.py@@ -0,0 +1,117 @@+"""Inline-marker scan reuses ingestion's ``source_map`` for discovery + reads.++The init pipeline threads the already-computed ``{rel_path: bytes}`` map into+the extractor so it no longer re-walks the tree and re-reads every file. These+tests pin that the source_map path is behaviourally identical to the legacy+self-walk path, and that the update path (``restrict_to_files``) still reads+correctly, preferring in-memory bytes and falling back to disk.+"""++from __future__ import annotations++from pathlib import Path++from repowise.core.analysis.decision_extractor import DecisionExtractor++# Fixtures use ``\n``-escaped single-line string concatenation on purpose: no+# PHYSICAL line in this committed file begins with a comment marker, so a real+# repowise self-index does not harvest these examples as decisions. Same+# convention as test_decision_rationale_comments.py.+_PY = (+ "# WHY: cache the resolver because rebuilding it per call dominated latency\n"+ "def resolve():\n"+ " pass\n\n\n"+ "# DECISION: one engine per run keeps the counters honest\n"+ "class Engine:\n"+ " pass\n"+)++_GO = "// RATIONALE: sync.Pool avoids GC churn under sustained load\npackage util\n"++_MD = (+ "# ADR notes\n\n"+ "Some prose about the system.\n\n"+ "<!-- WHY: an HTML-comment line is not a recognised marker prefix -->\n\n"+ "```\n"+ "# DECISION: this one is inside a code fence and must be ignored\n"+ "```\n\n"+ "# TRADEOFF: accept eventual consistency for higher write throughput\n"+)+++def _write_fixture(root: Path) -> None:+ (root / "src").mkdir(parents=True)+ (root / "docs").mkdir(parents=True)+ (root / "src" / "app.py").write_text(_PY, encoding="utf-8")+ (root / "src" / "util.go").write_text(_GO, encoding="utf-8")+ (root / "docs" / "adr.md").write_text(_MD, encoding="utf-8")+++def _source_map(root: Path) -> dict[str, bytes]:+ """Mirror ingestion's map: POSIX rel path -> raw bytes."""+ out: dict[str, bytes] = {}+ for p in root.rglob("*"):+ if p.is_file():+ out[p.relative_to(root).as_posix()] = p.read_bytes()+ return out+++def _key(d) -> tuple:+ return (+ Path(d.evidence_file).as_posix(),+ d.evidence_line,+ (d.title or "")[:80],+ (d.decision or "")[:120],+ )+++async def test_source_map_path_matches_walk_path(tmp_path):+ _write_fixture(tmp_path)++ walk = await DecisionExtractor(repo_path=tmp_path, source_map=None).scan_inline_markers()+ reused = await DecisionExtractor(+ repo_path=tmp_path, source_map=_source_map(tmp_path)+ ).scan_inline_markers()++ assert {_key(d) for d in reused} == {_key(d) for d in walk}+ # Four real markers; the fenced DECISION in adr.md is excluded by both paths.+ assert len(reused) == 4+ assert not any("inside a fence" in (d.decision or "") for d in reused)+++async def test_source_map_evidence_paths_are_posix(tmp_path):+ _write_fixture(tmp_path)+ reused = await DecisionExtractor(+ repo_path=tmp_path, source_map=_source_map(tmp_path)+ ).scan_inline_markers()++ files = {d.evidence_file for d in reused}+ # Paths come straight from source_map keys (POSIX), matching graph node ids.+ assert "src/app.py" in files+ assert "docs/adr.md" in files+ assert all("\\" not in f for f in files)+++async def test_restrict_prefers_source_map_bytes_then_disk(tmp_path):+ _write_fixture(tmp_path)+ # source_map carries app.py; util.go is intentionally absent so the scan+ # must fall back to a disk read for it.+ sm = {"src/app.py": (tmp_path / "src" / "app.py").read_bytes()}+ ex = DecisionExtractor(repo_path=tmp_path, source_map=sm)++ decisions = await ex.scan_inline_markers(+ restrict_to_files=["src/app.py", "src/util.go"]+ )+ files = {d.evidence_file for d in decisions}+ assert "src/app.py" in files # from in-memory bytes+ assert "src/util.go" in files # from disk fallback+++async def test_restrict_skips_missing_files(tmp_path):+ _write_fixture(tmp_path)+ ex = DecisionExtractor(repo_path=tmp_path, source_map=None)+ # A deleted/renamed path in the change set must not raise or fabricate.+ decisions = await ex.scan_inline_markers(+ restrict_to_files=["src/app.py", "src/gone.py"]+ )+ assert {d.evidence_file for d in decisions} == {"src/app.py"}