Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(decisions): reuse ingestion's source_map for the inline-marker scan (#913)

Raghav Chamadiyab9555852a4f2
diff --git a/packages/core/src/repowise/core/analysis/decisions/extractor.py b/packages/core/src/repowise/core/analysis/decisions/extractor.py
index 8e61218a6..dc566098d 100644
--- a/packages/core/src/repowise/core/analysis/decisions/extractor.py
+++ b/packages/core/src/repowise/core/analysis/decisions/extractor.py
@@ -30,7 +30,7 @@ from __future__ import annotations
import asyncio
import json
import re
-from collections.abc import Collection
+from collections.abc import Collection, Iterator
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
@@ -343,12 +343,19 @@ class DecisionExtractor:
graph: Any | None = None,
git_meta_map: dict[str, dict] | None = None,
parsed_files: list[Any] | None = None,
+ source_map: dict[str, bytes] | None = None,
) -> None:
self._repo_path = Path(repo_path)
self._provider = provider
self._graph = graph
self._git_meta_map = git_meta_map or {}
self._parsed_files = parsed_files or []
+ # ``source_map`` is ingestion's already-computed {rel_path: bytes} for
+ # the indexed file set. When present, the inline-marker scan reuses it
+ # for both discovery and reads instead of re-walking the tree and
+ # re-reading every file from disk (redundant with ingestion). ``None``
+ # keeps the legacy self-walk fallback for callers that don't thread it.
+ self._source_map = source_map
# ------------------------------------------------------------------
# Source 1: Inline markers
@@ -361,14 +368,10 @@ class DecisionExtractor:
"""Scan source files for decision markers (WHY:, DECISION:, etc.)."""
markers_by_file: dict[str, list[dict]] = {}
- if restrict_to_files:
- files_to_scan = [self._repo_path / fp for fp in restrict_to_files]
- else:
- files_to_scan = list(self._iter_source_files())
-
- total_files = len(files_to_scan)
+ scan_targets = list(self._iter_scan_targets(restrict_to_files))
+ total_files = len(scan_targets)
logger.info("decision_extractor.scanning_inline_markers", total_files=total_files)
- for idx, file_path in enumerate(files_to_scan):
+ for idx, (rel_path, text) in enumerate(scan_targets):
if idx > 0 and idx % 1000 == 0:
logger.info(
"decision_extractor.scan_progress",
@@ -376,17 +379,11 @@ class DecisionExtractor:
total=total_files,
markers_found=sum(len(v) for v in markers_by_file.values()),
)
- if not file_path.is_file():
- continue
- try:
- text = file_path.read_text(encoding="utf-8", errors="replace")
- except (OSError, UnicodeDecodeError):
- continue
lines = text.splitlines()
# Track whether we're inside a fenced code block in markdown
# files so we don't treat example markers as real decisions.
- is_markdown = file_path.suffix.lower() in (".md", ".mdx", ".rst")
+ is_markdown = Path(rel_path).suffix.lower() in (".md", ".mdx", ".rst")
in_code_fence = False
for line_num, line in enumerate(lines, start=1):
if is_markdown:
@@ -415,11 +412,6 @@ class DecisionExtractor:
ctx_end = min(len(lines), line_num + 20)
context = "\n".join(lines[ctx_start:ctx_end])
- try:
- rel_path = str(file_path.relative_to(self._repo_path))
- except ValueError:
- rel_path = str(file_path)
-
markers_by_file.setdefault(rel_path, []).append(
{
"keyword": m.group("keyword"),
@@ -1500,6 +1492,67 @@ class DecisionExtractor:
# Helpers
# ------------------------------------------------------------------
+ def _iter_scan_targets(
+ self, restrict_to_files: list[str] | None
+ ) -> Iterator[tuple[str, str]]:
+ """Yield ``(rel_path, text)`` for every file the marker scan covers.
+
+ Three sources, in priority order:
+
+ * ``restrict_to_files`` (update path): the caller's explicit change set.
+ Text comes from ``source_map`` when the file was just ingested, else a
+ targeted disk read; deleted / unreadable paths are skipped.
+ * ``source_map`` (init path): ingestion's already-decoded indexed set.
+ Discovery AND reads are free — no tree walk, no per-file ``read_text``.
+ Paths are POSIX (``FileInfo.path``), matching the graph node keys the
+ neighbour lookup joins against.
+ * legacy self-walk (``source_map is None``): the original ``os.walk`` +
+ git-tracked filter, kept so callers that don't thread ``source_map``
+ behave exactly as before.
+ """
+ if restrict_to_files:
+ for rel_path in restrict_to_files:
+ text = self._read_source_text(rel_path)
+ if text is not None:
+ yield rel_path, text
+ return
+
+ if self._source_map is not None:
+ for rel_path, source in self._source_map.items():
+ yield rel_path, source.decode("utf-8", errors="replace")
+ return
+
+ for file_path in self._iter_source_files():
+ if not file_path.is_file():
+ continue
+ try:
+ text = file_path.read_text(encoding="utf-8", errors="replace")
+ except (OSError, UnicodeDecodeError):
+ continue
+ try:
+ rel_path = str(file_path.relative_to(self._repo_path))
+ except ValueError:
+ rel_path = str(file_path)
+ yield rel_path, text
+
+ def _read_source_text(self, rel_path: str) -> str | None:
+ """Decode one file's text, preferring ingestion's in-memory bytes.
+
+ Falls back to a disk read (deleted / unreadable → ``None``) so the
+ update path stays correct for files that aren't in ``source_map``.
+ """
+ if self._source_map is not None:
+ source = self._source_map.get(rel_path)
+ if source is not None:
+ return source.decode("utf-8", errors="replace")
+ abs_path = self._repo_path / rel_path
+ if not abs_path.is_file():
+ return None
+ try:
+ return abs_path.read_text(encoding="utf-8", errors="replace")
+ except (OSError, UnicodeDecodeError):
+ return None
+
def _tracked_files(self) -> set[Path] | None:
"""Resolved paths git tracks under ``repo_path``, or ``None``.
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 33c3c956c..212cb9f14 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -482,6 +482,7 @@ async def run_pipeline(
graph_builder=graph_builder,
git_meta_map=git_meta_map,
parsed_files=parsed_files,
+ source_map=source_map,
progress=progress,
),
)
diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.py
index 1f4e8265b..15297a623 100644
--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py
+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py
@@ -250,6 +250,7 @@ async def _run_decision_extraction(
graph_builder: Any,
git_meta_map: dict[str, dict],
parsed_files: list[Any],
+ source_map: dict[str, bytes] | None = None,
progress: ProgressCallback | None,
) -> Any | None:
"""Extract architectural decisions from source and git history."""
@@ -274,6 +275,7 @@ async def _run_decision_extraction(
graph=graph_builder.graph(),
git_meta_map=git_meta_map,
parsed_files=parsed_files,
+ source_map=source_map,
)
def _decision_step(_source: str) -> None:
diff --git a/tests/unit/analysis/test_decision_source_map_reuse.py b/tests/unit/analysis/test_decision_source_map_reuse.py
new file mode 100644
index 000000000..3af3a7332
--- /dev/null
+++ b/tests/unit/analysis/test_decision_source_map_reuse.py
@@ -0,0 +1,117 @@
+"""Inline-marker scan reuses ingestion's ``source_map`` for discovery + reads.
+
+The init pipeline threads the already-computed ``{rel_path: bytes}`` map into
+the extractor so it no longer re-walks the tree and re-reads every file. These
+tests pin that the source_map path is behaviourally identical to the legacy
+self-walk path, and that the update path (``restrict_to_files``) still reads
+correctly, preferring in-memory bytes and falling back to disk.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.analysis.decision_extractor import DecisionExtractor
+
+# Fixtures use ``\n``-escaped single-line string concatenation on purpose: no
+# PHYSICAL line in this committed file begins with a comment marker, so a real
+# repowise self-index does not harvest these examples as decisions. Same
+# convention as test_decision_rationale_comments.py.
+_PY = (
+ "# WHY: cache the resolver because rebuilding it per call dominated latency\n"
+ "def resolve():\n"
+ " pass\n\n\n"
+ "# DECISION: one engine per run keeps the counters honest\n"
+ "class Engine:\n"
+ " pass\n"
+)
+
+_GO = "// RATIONALE: sync.Pool avoids GC churn under sustained load\npackage util\n"
+
+_MD = (
+ "# ADR notes\n\n"
+ "Some prose about the system.\n\n"
+ "<!-- WHY: an HTML-comment line is not a recognised marker prefix -->\n\n"
+ "```\n"
+ "# DECISION: this one is inside a code fence and must be ignored\n"
+ "```\n\n"
+ "# TRADEOFF: accept eventual consistency for higher write throughput\n"
+)
+
+
+def _write_fixture(root: Path) -> None:
+ (root / "src").mkdir(parents=True)
+ (root / "docs").mkdir(parents=True)
+ (root / "src" / "app.py").write_text(_PY, encoding="utf-8")
+ (root / "src" / "util.go").write_text(_GO, encoding="utf-8")
+ (root / "docs" / "adr.md").write_text(_MD, encoding="utf-8")
+
+
+def _source_map(root: Path) -> dict[str, bytes]:
+ """Mirror ingestion's map: POSIX rel path -> raw bytes."""
+ out: dict[str, bytes] = {}
+ for p in root.rglob("*"):
+ if p.is_file():
+ out[p.relative_to(root).as_posix()] = p.read_bytes()
+ return out
+
+
+def _key(d) -> tuple:
+ return (
+ Path(d.evidence_file).as_posix(),
+ d.evidence_line,
+ (d.title or "")[:80],
+ (d.decision or "")[:120],
+ )
+
+
+async def test_source_map_path_matches_walk_path(tmp_path):
+ _write_fixture(tmp_path)
+
+ walk = await DecisionExtractor(repo_path=tmp_path, source_map=None).scan_inline_markers()
+ reused = await DecisionExtractor(
+ repo_path=tmp_path, source_map=_source_map(tmp_path)
+ ).scan_inline_markers()
+
+ assert {_key(d) for d in reused} == {_key(d) for d in walk}
+ # Four real markers; the fenced DECISION in adr.md is excluded by both paths.
+ assert len(reused) == 4
+ assert not any("inside a fence" in (d.decision or "") for d in reused)
+
+
+async def test_source_map_evidence_paths_are_posix(tmp_path):
+ _write_fixture(tmp_path)
+ reused = await DecisionExtractor(
+ repo_path=tmp_path, source_map=_source_map(tmp_path)
+ ).scan_inline_markers()
+
+ files = {d.evidence_file for d in reused}
+ # Paths come straight from source_map keys (POSIX), matching graph node ids.
+ assert "src/app.py" in files
+ assert "docs/adr.md" in files
+ assert all("\\" not in f for f in files)
+
+
+async def test_restrict_prefers_source_map_bytes_then_disk(tmp_path):
+ _write_fixture(tmp_path)
+ # source_map carries app.py; util.go is intentionally absent so the scan
+ # must fall back to a disk read for it.
+ sm = {"src/app.py": (tmp_path / "src" / "app.py").read_bytes()}
+ ex = DecisionExtractor(repo_path=tmp_path, source_map=sm)
+
+ decisions = await ex.scan_inline_markers(
+ restrict_to_files=["src/app.py", "src/util.go"]
+ )
+ files = {d.evidence_file for d in decisions}
+ assert "src/app.py" in files # from in-memory bytes
+ assert "src/util.go" in files # from disk fallback
+
+
+async def test_restrict_skips_missing_files(tmp_path):
+ _write_fixture(tmp_path)
+ ex = DecisionExtractor(repo_path=tmp_path, source_map=None)
+ # A deleted/renamed path in the change set must not raise or fabricate.
+ decisions = await ex.scan_inline_markers(
+ restrict_to_files=["src/app.py", "src/gone.py"]
+ )
+ assert {d.evidence_file for d in decisions} == {"src/app.py"}