Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
diff --git a/packages/cli/src/repowise/cli/commands/health_cmd.py b/packages/cli/src/repowise/cli/commands/health_cmd.py
index d0be19e4..7fb48981 100644
--- a/packages/cli/src/repowise/cli/commands/health_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/health_cmd.py
@@ -8,6 +8,7 @@ back to a live in-process analysis when run outside an indexed repo.
from __future__ import annotations
import json
+from pathlib import Path
import click
from rich.table import Table
@@ -215,6 +216,7 @@ def health_command(
git_meta_map=git_meta_map,
parsed_files=parsed_files,
coverage_map=coverage_map,
+ duplication_cache_dir=Path(repo_path) / ".repowise",
)
# Load any .repowise/health-rules.json the user keeps in the repo.
from repowise.core.analysis.health.config import HealthConfig
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
index d4c2098d..f33de34b 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
@@ -47,6 +47,8 @@ from repowise.cli.providers import resolve_embedder
from repowise.cli.state_persistence import build_kg_state, save_knowledge_graph_json
from repowise.cli.ui import (
BRAND,
+ BRAND_STYLE,
+ OWL_SPINNER,
MaybeCountColumn,
RichProgressCallback,
interactive_advanced_config,
@@ -455,7 +457,7 @@ def init_command(
scan_info = None
if is_interactive:
print_banner(console, repo_name=repo_path.name)
- with console.status(" Scanning repository…", spinner="dots"):
+ with console.status(" Scanning repository…", spinner=OWL_SPINNER):
scan_info = quick_repo_scan(repo_path)
print_scan_summary(console, scan_info)
mode = interactive_mode_select(console)
@@ -616,7 +618,7 @@ def init_command(
# Validate provider connection
from repowise.core.providers.llm.base import ProviderError
- with console.status(" Verifying provider connection…", spinner="dots"):
+ with console.status(" Verifying provider connection…", spinner=OWL_SPINNER):
try:
run_async(
provider.generate(
@@ -645,7 +647,7 @@ def init_command(
orchestrator_mode = OrchestratorMode.FAST if run_mode == "fast" else OrchestratorMode.STANDARD
with Progress(
- SpinnerColumn(),
+ SpinnerColumn(spinner_name=OWL_SPINNER, style=BRAND_STYLE),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
MaybeCountColumn(),
@@ -708,9 +710,11 @@ def init_command(
with cancellation_scope():
result = run_async(_index_with_resume())
except (PipelineCancelled, KeyboardInterrupt):
+ from repowise.cli.ui.mascot import EYES_SLEEPY, mini
+
console.print(
- "\n[yellow]Interrupted.[/] Indexed work so far has been saved — "
- "run [bold]repowise init --resume[/] to continue where it stopped."
+ f"\n{mini(EYES_SLEEPY)} [yellow]Interrupted.[/] Indexed work so far has been "
+ "saved — run [bold]repowise init --resume[/] to continue where it stopped."
)
return
@@ -757,7 +761,7 @@ def init_command(
console, 4, total_phases, "Persistence", "Saving to database and building search index"
)
- with console.status(" Persisting to database…", spinner="dots"):
+ with console.status(" Persisting to database…", spinner=OWL_SPINNER):
run_async(persist_result(result, repo_path))
console.print(" [green]✓[/green] Database updated")
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
index 29ecf4b8..ca41107e 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
@@ -27,7 +27,7 @@ from repowise.cli.providers import (
build_vector_store,
flush_cost_tracker,
)
-from repowise.cli.ui import MaybeCountColumn, RichProgressCallback
+from repowise.cli.ui import BRAND_STYLE, OWL_SPINNER, MaybeCountColumn, RichProgressCallback
# The LLM-cost confirmation threshold. A run whose estimate exceeds this asks
# for confirmation (unless ``--yes``); below it generation proceeds silently.
@@ -187,7 +187,7 @@ def _enrich_knowledge_graph(
result.knowledge_graph_result = _run()
return
- with console.status(" Enriching knowledge graph (layers + tour)…", spinner="dots"):
+ with console.status(" Enriching knowledge graph (layers + tour)…", spinner=OWL_SPINNER):
try:
result.knowledge_graph_result = _run()
enriched = result.knowledge_graph_result
@@ -237,7 +237,7 @@ def run_repo_generation(
provider._cost_tracker = cost_tracker
with Progress(
- SpinnerColumn(),
+ SpinnerColumn(spinner_name=OWL_SPINNER, style=BRAND_STYLE),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
MaybeCountColumn(),
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
index 84c9a05c..f0e15b7d 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
@@ -45,6 +45,8 @@ from repowise.cli.providers import resolve_embedder
from repowise.cli.state_persistence import build_kg_state, save_knowledge_graph_json
from repowise.cli.ui import (
BRAND,
+ BRAND_STYLE,
+ OWL_SPINNER,
MaybeCountColumn,
RichProgressCallback,
interactive_advanced_config,
@@ -211,7 +213,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
try:
with Progress(
- SpinnerColumn(),
+ SpinnerColumn(spinner_name=OWL_SPINNER, style=BRAND_STYLE),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
MaybeCountColumn(),
diff --git a/packages/cli/src/repowise/cli/commands/reindex_cmd.py b/packages/cli/src/repowise/cli/commands/reindex_cmd.py
index 17e9cee6..30c794bc 100644
--- a/packages/cli/src/repowise/cli/commands/reindex_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/reindex_cmd.py
@@ -12,6 +12,7 @@ from repowise.cli.helpers import (
resolve_repo_path,
run_async,
)
+from repowise.cli.ui import BRAND_STYLE, OWL_SPINNER
@click.command("reindex")
@@ -115,7 +116,7 @@ async def _reindex(repo_path, embedder_name: str, batch_size: int) -> None:
failed = 0
with Progress(
- SpinnerColumn(),
+ SpinnerColumn(spinner_name=OWL_SPINNER, style=BRAND_STYLE),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("{task.completed}/{task.total}"),
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd.py b/packages/cli/src/repowise/cli/commands/update_cmd.py
index ed1b58ca..1dd532e8 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd.py
@@ -346,22 +346,45 @@ def _build_repo_graph(
"""
from pathlib import Path as PathlibPath
- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
+ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder, compute_content_hash
traverser = FileTraverser(repo_path, extra_exclude_patterns=exclude_patterns or None)
file_infos = list(traverser.traverse())
repo_structure = traverser.get_repo_structure()
- parser = ASTParser()
+ # Content-hash parse cache: an incremental update re-ingests the whole
+ # repo, but only the changed files actually need a tree-sitter parse.
+ # Best-effort — any cache failure falls back to a full parse.
+ parse_cache = None
+ try:
+ from repowise.core.ingestion.parse_cache import ParseCache
+
+ parse_cache = ParseCache(PathlibPath(repo_path) / ".repowise")
+ parse_cache.load()
+ except Exception:
+ parse_cache = None
+
+ parser: Any = None # constructed lazily — every-file-cached updates skip query compilation
parsed_files: list = []
source_map: dict[str, bytes] = {}
- graph_builder = GraphBuilder(repo_path, exclude_patterns=exclude_patterns)
+ graph_builder = GraphBuilder(
+ repo_path,
+ exclude_patterns=exclude_patterns,
+ centrality_cache_dir=PathlibPath(repo_path) / ".repowise",
+ )
skipped = 0
for fi in file_infos:
try:
source = PathlibPath(fi.abs_path).read_bytes()
- parsed = parser.parse_file(fi, source)
+ content_hash = compute_content_hash(source)
+ parsed = parse_cache.get(fi, content_hash) if parse_cache is not None else None
+ if parsed is None:
+ if parser is None:
+ parser = ASTParser()
+ parsed = parser.parse_file(fi, source)
+ if parse_cache is not None:
+ parse_cache.put(parsed, content_hash)
except Exception:
skipped += 1
continue
@@ -370,6 +393,8 @@ def _build_repo_graph(
source_map[fi.path] = source
graph_builder.add_file(parsed)
graph_builder.build()
+ if parse_cache is not None:
+ parse_cache.save()
if skipped:
console.print(f"[yellow]Skipped {skipped} file(s) that failed to parse.[/yellow]")
@@ -393,10 +418,16 @@ def _rebuild_graph_and_git(
file_diffs: list,
cfg: dict,
exclude_patterns: list[str],
+ git_tier: str | None = None,
) -> tuple[list, dict[str, bytes], Any, Any, int, dict[str, dict]]:
"""Re-traverse + parse the repo, rebuild the graph (+ framework edges), and
re-index git metadata for the changed files.
+ ``git_tier`` is the persisted ``state.json:git_tier`` value: a fast-mode
+ (ESSENTIAL) repo must not pay per-file blame on every update for signals
+ its index never had. Unknown/missing values fall back to FULL, matching
+ the historical behavior for legacy state files.
+
Returns ``(parsed_files, source_map, graph_builder, repo_structure,
file_count, git_meta_map)``.
"""
@@ -409,7 +440,12 @@ def _rebuild_graph_and_git(
git_meta_map: dict[str, dict] = {}
try:
from repowise.core.ingestion.git_indexer import GitIndexer
+ from repowise.core.ingestion.git_indexer.tiers import GitIndexTier
+ try:
+ tier = GitIndexTier(git_tier) if git_tier else GitIndexTier.FULL
+ except ValueError:
+ tier = GitIndexTier.FULL
_commit_limit = cfg.get("commit_limit")
_follow_renames = cfg.get("follow_renames", False)
git_indexer = GitIndexer(
@@ -417,6 +453,7 @@ def _rebuild_graph_and_git(
commit_limit=_commit_limit,
follow_renames=_follow_renames,
exclude_patterns=exclude_patterns or None,
+ tier=tier,
)
changed_paths = _build_filtered_changed_paths(file_diffs, exclude_patterns)
updated_meta = run_async(git_indexer.index_changed_files(changed_paths))
@@ -425,6 +462,16 @@ def _rebuild_graph_and_git(
except Exception as exc:
console.print(f"[yellow]Git re-index skipped: {exc}[/yellow]")
+ # Pre-compute centrality/community metrics with the init path's fan-out
+ # parallelism. Without this, persist_graph_nodes computes the same
+ # metrics lazily one-by-one. Runs after the co-change edge refresh so
+ # the cached subgraphs reflect the final structure. Best-effort: every
+ # metric still falls back to lazy computation.
+ try:
+ run_async(graph_builder.compute_metrics_parallel())
+ except Exception as exc:
+ console.print(f"[yellow]Metric pre-computation skipped: {exc}[/yellow]")
+
return parsed_files, source_map, graph_builder, repo_structure, file_count, git_meta_map
@@ -453,6 +500,7 @@ def _run_partial_analysis(
graph_builder.graph(),
git_meta_map=git_meta_map,
parsed_files=parsed_files,
+ duplication_cache_dir=Path(repo_path) / ".repowise",
)
_health_changed = {fd.path for fd in file_diffs if fd.status in ("added", "modified")}
if _health_changed:
@@ -675,6 +723,13 @@ def _run_full_health_rescore(
repo_path, exclude_patterns
)
+ # Fan-out metric precompute (mirrors _rebuild_graph_and_git) — the
+ # rescore persists graph nodes too, which reads every metric.
+ try:
+ run_async(graph_builder.compute_metrics_parallel())
+ except Exception:
+ pass # metrics fall back to lazy computation
+
exclude_spec = (
pathspec.PathSpec.from_lines("gitwildmatch", exclude_patterns)
if exclude_patterns
@@ -737,6 +792,7 @@ def _run_full_health_rescore(
graph_builder.graph(),
git_meta_map=git_meta_map,
parsed_files=parsed_files,
+ duplication_cache_dir=Path(repo_path) / ".repowise",
)
hcfg = HealthConfig.load(repo_path)
analyzer_config = (
@@ -1082,26 +1138,15 @@ def update_command(
console.print(f" [{color}]{fd.status:>10}[/{color}] {fd.path}")
# Re-parse changed files and rebuild graph for affected pages
- from repowise.core.generation import ContextAssembler, GenerationConfig, PageGenerator
-
cfg = load_config(repo_path)
- language = cfg.get("language", "en")
- # Config-driven (saved by `repowise init`); CLI override not surfaced
- # on update yet — defaults to on to keep the onboarding collection
- # fresh as the codebase evolves.
- enable_onboarding_cfg = bool(cfg.get("enable_onboarding", True))
- config = GenerationConfig(
- max_concurrency=concurrency,
- language=language,
- reasoning=resolve_reasoning(reasoning, cfg),
- enable_onboarding=enable_onboarding_cfg,
- )
# Read exclude patterns from config (set during init or via web UI)
exclude_patterns: list[str] = list(cfg.get("exclude_patterns") or [])
parsed_files, source_map, graph_builder, repo_structure, file_count, git_meta_map = (
- _rebuild_graph_and_git(repo_path, file_diffs, cfg, exclude_patterns)
+ _rebuild_graph_and_git(
+ repo_path, file_diffs, cfg, exclude_patterns, git_tier=state.get("git_tier")
+ )
)
# Determine affected pages (auto-scale budget if not explicitly set)
@@ -1145,6 +1190,23 @@ def update_command(
)
return
+ # The generation/LLM layer is only needed past this point — importing it
+ # above the index-only branch would make every index-only update (the
+ # post-commit hook's hot path) pay the import for code it never runs.
+ from repowise.core.generation import ContextAssembler, GenerationConfig, PageGenerator
+
+ language = cfg.get("language", "en")
+ # Config-driven (saved by `repowise init`); CLI override not surfaced
+ # on update yet — defaults to on to keep the onboarding collection
+ # fresh as the codebase evolves.
+ enable_onboarding_cfg = bool(cfg.get("enable_onboarding", True))
+ config = GenerationConfig(
+ max_concurrency=concurrency,
+ language=language,
+ reasoning=resolve_reasoning(reasoning, cfg),
+ enable_onboarding=enable_onboarding_cfg,
+ )
+
provider = resolve_provider(provider_name, model, repo_path=repo_path)
# Attach a DB-backed CostTracker so every LLM call made during this update
diff --git a/packages/cli/src/repowise/cli/editor_integrations/claude.py b/packages/cli/src/repowise/cli/editor_integrations/claude.py
index 42c6cda6..ba721988 100644
--- a/packages/cli/src/repowise/cli/editor_integrations/claude.py
+++ b/packages/cli/src/repowise/cli/editor_integrations/claude.py
@@ -123,8 +123,11 @@ def maybe_generate_claude_md(
return
if not _claude_md_enabled(repo_path):
return
+
+ from repowise.cli.ui import OWL_SPINNER
+
try:
- with console_obj.status(" Generating .claude/CLAUDE.md…", spinner="dots"):
+ with console_obj.status(" Generating .claude/CLAUDE.md…", spinner=OWL_SPINNER):
run_async(_write_claude_md_async(repo_path))
console_obj.print(" [green]✓[/green] .claude/CLAUDE.md updated")
except Exception as exc:
diff --git a/packages/cli/src/repowise/cli/editor_integrations/codex.py b/packages/cli/src/repowise/cli/editor_integrations/codex.py
index a6310d50..e8cf602c 100644
--- a/packages/cli/src/repowise/cli/editor_integrations/codex.py
+++ b/packages/cli/src/repowise/cli/editor_integrations/codex.py
@@ -123,6 +123,7 @@ def maybe_generate_agents_md(
from repowise.cli.editor_files import should_generate_editor_file
from repowise.cli.helpers import run_async
+ from repowise.cli.ui import OWL_SPINNER
if not should_generate_editor_file(
repo_path,
@@ -132,7 +133,7 @@ def maybe_generate_agents_md(
):
return
try:
- with console_obj.status(" Generating AGENTS.md...", spinner="dots"):
+ with console_obj.status(" Generating AGENTS.md...", spinner=OWL_SPINNER):
run_async(_write_agents_md_async(repo_path))
console_obj.print(" [green]✓[/green] AGENTS.md updated")
except Exception as exc:
diff --git a/packages/cli/src/repowise/cli/ui/__init__.py b/packages/cli/src/repowise/cli/ui/__init__.py
index 7a37e3fa..b3a28a40 100644
--- a/packages/cli/src/repowise/cli/ui/__init__.py
+++ b/packages/cli/src/repowise/cli/ui/__init__.py
@@ -19,6 +19,12 @@ from repowise.cli.ui.brand import (
print_phase_header,
)
from repowise.cli.ui.env_persistence import load_dotenv
+from repowise.cli.ui.mascot import (
+ OWL_SPINNER,
+ THINKING_FRAMES,
+ banner_text,
+ mini,
+)
from repowise.cli.ui.mode_selection import (
LARGE_REPO_FILE_THRESHOLD,
interactive_advanced_config,
@@ -55,11 +61,14 @@ __all__ = [
"ERR",
"LARGE_REPO_FILE_THRESHOLD",
"OK",
+ "OWL_SPINNER",
+ "THINKING_FRAMES",
"WARN",
"MaybeCountColumn",
"ProviderSelection",
"RepoScanInfo",
"RichProgressCallback",
+ "banner_text",
"build_analysis_summary_panel",
"build_completion_panel",
"build_contextual_next_steps",
@@ -72,6 +81,7 @@ __all__ = [
"interactive_provider_select",
"interactive_repo_select",
"load_dotenv",
+ "mini",
"print_banner",
"print_index_only_intro",
"print_phase_header",
diff --git a/packages/cli/src/repowise/cli/ui/brand.py b/packages/cli/src/repowise/cli/ui/brand.py
index 5814f1c1..9e2a57fa 100644
--- a/packages/cli/src/repowise/cli/ui/brand.py
+++ b/packages/cli/src/repowise/cli/ui/brand.py
@@ -3,8 +3,8 @@
from __future__ import annotations
from rich.console import Console
+from rich.markup import escape
from rich.rule import Rule
-from rich.text import Text
# ---------------------------------------------------------------------------
# Brand / theme
@@ -18,24 +18,33 @@ WARN = "yellow"
ERR = "bold red"
# ---------------------------------------------------------------------------
-# ASCII art — bold half-block, compact lowercase, 2 lines
+# Banner — owl mascot + heatmap wordmark (art lives in ui/mascot.py)
# ---------------------------------------------------------------------------
-_LOGO = " █▀█ █▀▀ █▀█ █▀█ █ █ █ ▀ █▀▀ █▀▀\n █▀▄ ██▄ █▀▀ █▄█ ▀▄▀▄▀ █ ▄▄█ ██▄"
+# Breathing room required beyond the rendered banner width before we pick the
+# full variant; below that we fall back to compact (same design, 1-char strokes).
+_BANNER_WIDTH_MARGIN = 4
def print_banner(console: Console, repo_name: str | None = None) -> None:
- """Print the repowise logo, tagline, and optional repo name."""
+ """Print the repowise owl banner, tagline, and optional repo name."""
from repowise.cli import __version__
+ from repowise.cli.ui import mascot
+ compact = console.width < mascot.banner_width() + _BANNER_WIDTH_MARGIN
console.print()
- console.print(Text(_LOGO, style=BRAND_STYLE))
- console.print(
- f" [dim]codebase intelligence for developers and AI[/dim] [dim]v{__version__}[/dim]"
- )
+ console.print(mascot.banner_text(repo_name, compact=compact))
+ console.print()
+ if compact:
+ console.print(f" [dim]codebase intelligence · v{__version__}[/dim]", highlight=False)
+ else:
+ console.print(
+ f" [dim]codebase intelligence for developers and AI · v{__version__}[/dim]",
+ highlight=False,
+ )
if repo_name:
console.print()
- console.print(f" Repository: [bold]{repo_name}[/bold]")
+ console.print(f" Repository: [bold]{escape(repo_name)}[/bold]", highlight=False)
console.print()
diff --git a/packages/cli/src/repowise/cli/ui/mascot.py b/packages/cli/src/repowise/cli/ui/mascot.py
new file mode 100644
index 00000000..ca2e7c9e
--- /dev/null
+++ b/packages/cli/src/repowise/cli/ui/mascot.py
@@ -0,0 +1,212 @@
+"""Owl mascot: banner art, state frames, and the heatmap wordmark.
+
+Single home for every piece of the init-banner identity: the owl, the
+REPOWISE block wordmark (full + compact variants), the GitHub-heatmap
+colouring, and the spinner frames used while repowise thinks.
+
+All render functions are pure and deterministic — the heatmap pattern is
+seeded from the repo name, so the same repo always gets the same banner.
+"""
+
+from __future__ import annotations
+
+import zlib
+from functools import cache
+from random import Random
+
+from rich.text import Text
+
+from repowise.cli.ui.brand import BRAND
+
+# ---------------------------------------------------------------------------
+# Palette
+# ---------------------------------------------------------------------------
+
+WHITE = "#E8E8E8" # owl body strokes (logo white)
+
+# Floored 5-shade heatmap ramp: deep ember → blazing. No near-black shade so
+# the wordmark always reads, even at the cold end of the gradient.
+HEAT = ["#5C3208", "#9C5710", "#D97A16", "#F59520", "#FFC06A"]
+
+# ---------------------------------------------------------------------------
+# Owl
+# ---------------------------------------------------------------------------
+
+OWL_FULL = [
+ " ,_____,",
+ " (◉ , ◉)",
+ " ( ▼ )",
+ " /)___(\\",
+ ' " " ',
+]
+
+OWL_COMPACT = [
+ " ,___,",
+ " (◉,◉)",
+ " ( ▼ )",
+ " /)_(\\",
+ ' " " ',
+]
+
+EYES_THINKING = ["◐", "◓", "◑", "◒"] # eye-roll cycle (spinner frames)
+EYES_IDLE = "◉"
+EYES_SLEEPY = "─" # interrupt message
+EYES_HAPPY = "^" # completion panels
+EYES_ERROR = "x" # reserved: error output deliberately stays sober (no owl) for now
+
+
+def mini(eyes: str = EYES_IDLE) -> str:
+ """Single-line mini owl face, e.g. ``{◉ ◉}`` — always 5 chars wide."""
+ return "{" + eyes + " " + eyes + "}"
+
+
+THINKING_FRAMES = [mini(e) for e in EYES_THINKING]
+
+# ---------------------------------------------------------------------------
+# Spinner registration — ``rich._spinners.SPINNERS`` is private API (a plain
+# dict consumed by ``rich.spinner.Spinner``), so registration is wrapped: if a
+# future rich version moves it, everything silently stays on "dots". Always
+# reference ``OWL_SPINNER`` at call sites, never the literal ``"owl"``.
+# ---------------------------------------------------------------------------
+
+try:
+ from rich._spinners import SPINNERS
+
+ SPINNERS["owl"] = {"interval": 200, "frames": THINKING_FRAMES}
+ OWL_SPINNER = "owl"
+except Exception: # pragma: no cover — depends on rich internals
+ OWL_SPINNER = "dots"
+
+# ---------------------------------------------------------------------------
+# Wordmark fonts — half-block finished letterforms (R tail = stepped taper).
+# FONT_FULL uses 2-char stroke cells; FONT_COMPACT is the same design at
+# 1-char strokes for narrow terminals.
+# ---------------------------------------------------------------------------
+
+FONT_FULL = {
+ "R": ["██████▄", "██ ██", "██████▀", "██ ▀█▄ ", "██ ▀█▄"],
+ "E": ["██████", "██ ", "█████ ", "██ ", "██████"],
+ "P": ["██████▄", "██ ██", "██████▀", "██ ", "██ "],
+ "O": ["▄█████▄", "██ ██", "██ ██", "██ ██", "▀█████▀"],
+ "W": ["██ ██", "██ ██", "██ █ ██", "██ ███ ██", "▀██▀ ▀██▀"],
+ "I": ["████", " ██ ", " ██ ", " ██ ", "████"],
+ "S": ["▄█████▄", "██ ", "▀█████▄", " ██", "▀█████▀"],
+}
+
+FONT_COMPACT = {
+ "R": ["███▄ ", "█ █ ", "███▀ ", "█ ▀▄ ", "█ ▀▄"],
+ "E": ["███", "█ ", "██ ", "█ ", "███"],
+ "P": ["███▄", "█ █", "███▀", "█ ", "█ "],
+ "O": ["▄██▄", "█ █", "█ █", "█ █", "▀██▀"],
+ "W": ["█ █", "█ █", "█ █ █", "█ █ █", "▀█▀█▀"],
+ "I": ["███", " █ ", " █ ", " █ ", "███"],
+ "S": ["▄██▄", "█ ", "▀██▄", " █", "▀██▀"],
+}
+
+_WORD = "REPOWISE"
+
+
+@cache
+def render_wordmark(compact: bool = False) -> tuple[tuple[str, ...], tuple[int, ...]]:
+ """Wordmark rows + per-column heat-cell ids.
+
+ Cell ids are aligned to each letter's own grid (one cell = ``cell_w``
+ columns starting at the letter's left edge) so every coloured unit is a
+ true square — a cell never slices a stroke or bleeds across the
+ inter-letter gap. Gap columns map to ``-1`` (unpainted).
+ """
+ font = FONT_COMPACT if compact else FONT_FULL
+ cell_w = 1 if compact else 2
+ gap = 1 if compact else 2
+
+ rows = ["", "", "", "", ""]
+ cellmap: list[int] = []
+ cell_base = 0
+ for li, ch in enumerate(_WORD):
+ glyph = font[ch]
+ width = len(glyph[0])
+ n_cells = (width + cell_w - 1) // cell_w
+ for lx in range(width):
+ cellmap.append(cell_base + min(lx // cell_w, n_cells - 1))
+ cell_base += n_cells
+ for r in range(5):
+ rows[r] += glyph[r]
+ if li < len(_WORD) - 1:
+ for r in range(5):
+ rows[r] += " " * gap
+ cellmap.extend([-1] * gap)
+ return tuple(rows), tuple(cellmap)
+
+
+def banner_width(compact: bool = False) -> int:
+ """Total rendered banner width in columns (indent + owl + gap + wordmark).
+
+ Single source of truth for layout maths — ``print_banner`` derives its
+ full/compact switch threshold from this, so font or owl changes can't
+ silently desync the two.
+ """
+ owl = OWL_COMPACT if compact else OWL_FULL
+ rows, _ = render_wordmark(compact)
+ return 1 + max(len(line) for line in owl) + 2 + len(rows[0])
+
+
+def seed_for(repo_name: str | None) -> int:
+ """Stable cross-process seed for a repo's heatmap pattern.
+
+ Uses ``zlib.crc32`` deliberately — the built-in ``hash()`` is salted per
+ process, which would give every run a different banner.
+ """
+ return zlib.crc32((repo_name or "repowise").encode())
+
+
+def heat_grid(seed: int, n_cells: int) -> list[list[int]]:
+ """Gradient + jitter shade grid (5 rows by *n_cells*), deterministic.
+
+ Base shade sweeps linearly left→right across cells; each cell jitters
+ ±1 shade around the sweep, clamped to the palette bounds.
+ """
+ rng = Random(seed)
+ span = max(n_cells - 1, 1)
+ grid = [[0] * n_cells for _ in range(5)]
+ for r in range(5):
+ for c in range(n_cells):
+ base = (c / span) * (len(HEAT) - 1)
+ idx = round(base + rng.uniform(-1.1, 1.1))
+ grid[r][c] = min(max(idx, 0), len(HEAT) - 1)
+ return grid
+
+
+def _paint_owl(line: str) -> Text:
+ """Owl strokes in logo white, eyes in brand orange."""
+ out = Text()
+ for ch in line:
+ if ch == EYES_IDLE:
+ out.append(ch, style=f"bold {BRAND}")
+ elif ch == " ":
+ out.append(" ")
+ else:
+ out.append(ch, style=f"bold {WHITE}")
+ return out
+
+
+def banner_text(repo_name: str | None, compact: bool = False) -> Text:
+ """Compose owl + heatmap wordmark into a single renderable ``Text``."""
+ owl = OWL_COMPACT if compact else OWL_FULL
+ rows, cellmap = render_wordmark(compact)
+ n_cells = max(c for c in cellmap if c >= 0) + 1
+ grid = heat_grid(seed_for(repo_name), n_cells)
+
+ owl_w = max(len(line) for line in owl)
+ out = Text()
+ for r in range(5):
+ if r:
+ out.append("\n")
+ out.append(" ")
+ out.append_text(_paint_owl(owl[r].ljust(owl_w)))
+ out.append(" ")
+ for x, ch in enumerate(rows[r]):
+ if ch == " " or cellmap[x] < 0:
+ out.append(" ")
+ else:
+ out.append(ch, style=f"bold {HEAT[grid[r][cellmap[x]]]}")
+ return out
diff --git a/packages/cli/src/repowise/cli/ui/result_panels.py b/packages/cli/src/repowise/cli/ui/result_panels.py
index 1d09e8f4..31ddff66 100644
--- a/packages/cli/src/repowise/cli/ui/result_panels.py
+++ b/packages/cli/src/repowise/cli/ui/result_panels.py
@@ -10,6 +10,7 @@ from rich.table import Table
from rich.text import Text
from repowise.cli.ui.brand import BRAND
+from repowise.cli.ui.mascot import EYES_HAPPY, mini
def build_analysis_summary_panel(
@@ -79,10 +80,12 @@ def build_completion_panel(
*,
next_steps: list[tuple[str, str]] | None = None,
) -> Panel:
- """Build a bordered summary panel.
+ """Build a bordered summary panel, titled with the happy owl.
*metrics* is a list of ``(label, value)`` pairs.
*next_steps* is an optional list of ``(command, description)`` pairs.
+ The mascot prefix lives here (not at call sites) so every completion
+ panel gets it consistently.
"""
table = Table(box=None, padding=(0, 2), show_header=False)
table.add_column("Metric", style="dim", min_width=20)
@@ -101,7 +104,7 @@ def build_completion_panel(
return Panel(
Group(*parts),
- title=f"[bold]{title}[/bold]",
+ title=f"[bold]{mini(EYES_HAPPY)} {title}[/bold]",
border_style=BRAND,
padding=(1, 1),
)
diff --git a/packages/core/alembic/versions/0030_agent_provenance.py b/packages/core/alembic/versions/0030_agent_provenance.py
new file mode 100644
index 00000000..a11e79e9
--- /dev/null
+++ b/packages/core/alembic/versions/0030_agent_provenance.py
@@ -0,0 +1,55 @@
+"""Agent-provenance columns on git_commits + git_metadata.
+
+Per-commit: which coding agent (if any) authored the commit, its autonomy tier
+(1 near-autonomous bot account · 2 human-driven agent · 3 assisted), the
+attribution channel, and a confidence band. Per-file: the rollup share of
+agent-attributed commits and the tier breakdown. Classification is
+deterministic local-git pattern matching (identity fields, message footers,
+co-author trailers) riding the existing commit-index walk — no extra git pass,
+no network. All columns are nullable / defaulted and back-populated on the
+next index.
+
+Revision ID: 0030
+Revises: 0029
+Create Date: 2026-06-04
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers
+revision: str = "0030"
+down_revision: str | None = "0029"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ op.add_column("git_commits", sa.Column("agent_name", sa.String(32), nullable=True))
+ op.add_column("git_commits", sa.Column("agent_autonomy_tier", sa.Integer, nullable=True))
+ op.add_column("git_commits", sa.Column("agent_channel", sa.String(32), nullable=True))
+ op.add_column("git_commits", sa.Column("agent_confidence", sa.String(8), nullable=True))
+
+ op.add_column(
+ "git_metadata",
+ sa.Column("agent_commit_count", sa.Integer, nullable=False, server_default="0"),
+ )
+ op.add_column("git_metadata", sa.Column("agent_authored_pct", sa.Float, nullable=True))
+ op.add_column(
+ "git_metadata",
+ sa.Column("agent_tier_counts_json", sa.Text, nullable=False, server_default="{}"),
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("git_metadata", "agent_tier_counts_json")
+ op.drop_column("git_metadata", "agent_authored_pct")
+ op.drop_column("git_metadata", "agent_commit_count")
+ op.drop_column("git_commits", "agent_confidence")
+ op.drop_column("git_commits", "agent_channel")
+ op.drop_column("git_commits", "agent_autonomy_tier")
+ op.drop_column("git_commits", "agent_name")
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index de9bb014..e87adeb0 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -12,7 +12,10 @@ this package.
from __future__ import annotations
import fnmatch
+import os
+import re
from datetime import UTC, datetime
+from functools import lru_cache
from pathlib import Path
from typing import Any
@@ -321,6 +324,20 @@ def _is_synthetic_node(node: str) -> bool:
return node.startswith("external:") or node.startswith("framework:")
+@lru_cache(maxsize=8)
+def _never_flag_regex(patterns: tuple[str, ...]) -> re.Pattern[str]:
+ """Compile *patterns* into one alternation regex equivalent to fnmatch.
+
+ ``fnmatch.fnmatch(path, p)`` normcases both sides and matches the
+ translated glob; doing that per (node x pattern) costs ~540 fnmatch
+ calls per node and dominated the whole dead-code pass (measured: 50s of
+ a 51s analyze() on a 13k-node graph, mostly Windows ``normcase``).
+ One pre-normcased alternation keeps the exact same match semantics at
+ one regex match per node.
+ """
+ return re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in patterns))
+
+
class DeadCodeAnalyzer:
"""Detects unreachable files, unused exports, unused internals, and
zombie packages using the dependency graph and git metadata.
@@ -1138,9 +1155,8 @@ class DeadCodeAnalyzer:
"""Return True if path should never be flagged as dead."""
if path in whitelist:
return True
- for pattern in _NEVER_FLAG_PATTERNS:
- if fnmatch.fnmatch(path, pattern):
- return True
+ if _never_flag_regex(_NEVER_FLAG_PATTERNS).match(os.path.normcase(path)):
+ return True
# Workspace-driven never-flag — set by language warmups that read
# the build manifest (Gradle non-``main`` source sets, Cargo
# ``[[example]]`` / ``[[bench]]`` targets, …). Lets each language
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
index 7396669a..dfaaedb2 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
@@ -34,7 +34,7 @@ from repowise.core.cancellation import check_cancelled
from .limits import DuplicationDiagnostics, DuplicationLimits, looks_minified
from .rabin_karp import WindowHash, index_by_hash, rolling_hashes
-from .tokenizer import Token, tokenize_file
+from .tokenizer import tokenize_file
log = structlog.get_logger(__name__)
@@ -132,19 +132,21 @@ def _co_change_score(
def _tokens_equal(
- a_tokens: list[Token],
- b_tokens: list[Token],
+ a_kinds: list[str],
+ b_kinds: list[str],
a_start: int,
b_start: int,
window: int,
) -> bool:
- """Verify hash-collision by comparing token ``kind`` sequences."""
- if a_start + window > len(a_tokens) or b_start + window > len(b_tokens):
+ """Verify hash-collision by comparing token ``kind`` sequences.
+
+ Operates on the per-file kind list (all the verifier ever compared of
+ the full ``Token`` records) so cached token streams round-trip without
+ rebuilding Token objects.
+ """
+ if a_start + window > len(a_kinds) or b_start + window > len(b_kinds):
return False
- for offset in range(window):
- if a_tokens[a_start + offset].kind != b_tokens[b_start + offset].kind:
- return False
- return True
+ return a_kinds[a_start : a_start + window] == b_kinds[b_start : b_start + window]
def _merge_adjacent_pairs(raw: list[ClonePair]) -> list[ClonePair]:
@@ -195,6 +197,7 @@ def detect_clones(
window_tokens: int = DEFAULT_WINDOW_TOKENS,
min_lines: int = DEFAULT_MIN_LINES,
limits: DuplicationLimits | None = None,
+ cache_dir: Path | None = None,
) -> DuplicationReport:
"""Run the duplication pipeline over the supplied parsed files.
@@ -216,14 +219,28 @@ def detect_clones(
lim = limits or DuplicationLimits()
diag = DuplicationDiagnostics()
- per_file_tokens, per_file_nloc, all_windows = _collect_windows(
- parsed_files, window_tokens, lim, diag
+ cache = None
+ if cache_dir is not None:
+ from .token_cache import DuplicationTokenCache
+
+ cache = DuplicationTokenCache(cache_dir, window_tokens)
+ cache.load()
+
+ per_file_kinds, per_file_nloc, all_windows = _collect_windows(
+ parsed_files, window_tokens, lim, diag, cache
)
+ if cache is not None:
+ cache.save()
+ log.debug(
+ "duplication_token_cache",
+ hits=cache.hits,
+ misses=cache.misses,
+ )
if not all_windows:
return DuplicationReport(diagnostics=diag.as_log_fields())
bucket = index_by_hash(all_windows)
- raw_pairs = _pairs_from_buckets(bucket, per_file_tokens, window_tokens, lim, diag)
+ raw_pairs = _pairs_from_buckets(bucket, per_file_kinds, window_tokens, lim, diag)
final = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)
pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)
@@ -246,15 +263,23 @@ def _collect_windows(
window_tokens: int,
limits: DuplicationLimits,
diag: DuplicationDiagnostics,
-) -> tuple[dict[str, list[Token]], dict[str, int], list[WindowHash]]:
+ cache: Any | None = None,
+) -> tuple[dict[str, list[str]], dict[str, int], list[WindowHash]]:
"""Tokenize each file once and emit its rolling-hash windows.
Files are dropped (and counted in *diag*) when they are unreadable,
minified/generated, shorter than one window, or exceed the per-file
token cap. Collection stops cleanly once the repo-wide window budget
is reached so peak memory stays bounded.
+
+ When a :class:`~.token_cache.DuplicationTokenCache` is supplied,
+ unchanged files (by content hash) skip the tokenize + rolling-hash
+ work and replay their cached kind sequence and window tuples; every
+ gate above still re-evaluates live against the cached lengths.
"""
- per_file_tokens: dict[str, list[Token]] = {}
+ import hashlib
+
+ per_file_kinds: dict[str, list[str]] = {}
per_file_nloc: dict[str, int] = {}
all_windows: list[WindowHash] = []
@@ -272,25 +297,57 @@ def _collect_windows(
diag.skipped_minified += 1
continue
- toks = tokenize_file(language, source)
- if len(toks) < window_tokens:
+ cached = None
+ content_hash = ""
+ if cache is not None:
+ content_hash = hashlib.sha256(source).hexdigest()
+ cached = cache.get(content_hash)
+
+ if cached is not None:
+ kinds, nloc, window_tuples = cached
+ windows = [
+ WindowHash(
+ file_path=path,
+ hash_value=h,
+ start_index=si,
+ start_line=sl,
+ end_line=el,
+ )
+ for h, si, sl, el in window_tuples
+ ]
+ else:
+ toks = tokenize_file(language, source)
+ if len(toks) > limits.max_tokens_per_file:
+ diag.skipped_token_cap += 1
+ continue
+ kinds = [t.kind for t in toks]
+ nloc = _nloc(source)
+ windows = rolling_hashes(path, toks, window_tokens)
+ if cache is not None:
+ cache.put(
+ content_hash,
+ kinds,
+ nloc,
+ [(w.hash_value, w.start_index, w.start_line, w.end_line) for w in windows],
+ )
+
+ if len(kinds) < window_tokens:
continue
- if len(toks) > limits.max_tokens_per_file:
+ if len(kinds) > limits.max_tokens_per_file:
diag.skipped_token_cap += 1
continue
- windows = rolling_hashes(path, toks, window_tokens)
if len(all_windows) + len(windows) > limits.max_total_windows:
diag.window_budget_hit = True
break
- per_file_tokens[path] = toks
- per_file_nloc[path] = _nloc(source)
+ per_file_kinds[path] = kinds
+ per_file_nloc[path] = nloc
all_windows.extend(windows)
diag.files_tokenized += 1
diag.total_windows = len(all_windows)
- return per_file_tokens, per_file_nloc, all_windows
+ return per_file_kinds, per_file_nloc, all_windows
# ---------------------------------------------------------------------------
@@ -300,7 +357,7 @@ def _collect_windows(
def _pairs_from_buckets(
bucket: dict[int, list[WindowHash]],
- per_file_tokens: dict[str, list[Token]],
+ per_file_kinds: dict[str, list[str]],
window_tokens: int,
limits: DuplicationLimits,
diag: DuplicationDiagnostics,
@@ -329,14 +386,14 @@ def _pairs_from_buckets(
if deadline is not None and (i & 0x3FF) == 0 and time.monotonic() > deadline:
diag.timed_out = True
break
- _verify_bucket(windows, per_file_tokens, window_tokens, seen, raw_pairs)
+ _verify_bucket(windows, per_file_kinds, window_tokens, seen, raw_pairs)
return raw_pairs
def _verify_bucket(
windows: list[WindowHash],
- per_file_tokens: dict[str, list[Token]],
+ per_file_kinds: dict[str, list[str]],
window_tokens: int,
seen: set[tuple[str, int, str, int]],
out: list[ClonePair],
@@ -357,8 +414,8 @@ def _verify_bucket(
continue
seen.add(key)
if not _tokens_equal(
- per_file_tokens[a.file_path],
- per_file_tokens[b.file_path],
+ per_file_kinds[a.file_path],
+ per_file_kinds[b.file_path],
a.start_index,
b.start_index,
window_tokens,
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py
new file mode 100644
index 00000000..ec2a9226
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py
@@ -0,0 +1,115 @@
+"""Content-hash keyed cache for duplication token streams + windows.
+
+Tokenizing every file (a full tree-sitter re-parse plus a pure-Python
+leaf walk) and re-rolling 1M+ window hashes dominates the duplication
+pass — and an incremental ``repowise update`` re-pays it for the whole
+repo when a single file changed. Both outputs are pure functions of the
+file *bytes* (given fixed window size and pinned hash constants), so they
+cache safely by content hash.
+
+Cached per file: the normalized token-kind sequence (all the verifier
+ever compares), the non-blank line count, and the rolling-hash windows
+as plain tuples (``WindowHash`` is rebuilt with the file's *current*
+path, which makes hits rename-proof). The minified/token-cap/window-
+budget gates in the detector stay live — they re-evaluate against the
+cached lengths, so config changes apply to cached entries too.
+
+Best-effort by design: any load/save error degrades to a full
+re-tokenize. The cache file is rewritten with only the current file set
+each run, so deleted files age out naturally.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import pickle
+import sys
+import tempfile
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+_CACHE_VERSION = 1
+_CACHE_FILENAME = "duplication_cache.pkl"
+
+
+class DuplicationTokenCache:
+ """Pickle-backed ``content_hash -> (kinds, nloc, windows)`` store."""
+
+ def __init__(self, cache_dir: Path, window_tokens: int) -> None:
+ self._path = Path(cache_dir) / _CACHE_FILENAME
+ self._window_tokens = window_tokens
+ self._entries: dict[str, tuple[list[str], int, list[tuple[int, int, int, int]]]] = {}
+ self._fresh: dict[str, tuple[list[str], int, list[tuple[int, int, int, int]]]] = {}
+ self.hits = 0
+ self.misses = 0
+
+ # -- lifecycle ---------------------------------------------------------
+
+ def load(self) -> None:
+ try:
+ with self._path.open("rb") as fh:
+ payload = pickle.load(fh)
+ if (
+ payload.get("version") != _CACHE_VERSION
+ or payload.get("window_tokens") != self._window_tokens
+ ):
+ return
+ self._entries = payload.get("files", {})
+ except FileNotFoundError:
+ return
+ except Exception as exc: # corrupt / unreadable cache -> full tokenize
+ log.debug("duplication_cache_load_failed", error=str(exc))
+
+ def save(self) -> None:
+ """Atomically persist the entries used or created this run."""
+ try:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ payload = {
+ "version": _CACHE_VERSION,
+ "window_tokens": self._window_tokens,
+ "files": self._fresh,
+ }
+ fd, tmp_name = tempfile.mkstemp(
+ dir=str(self._path.parent), prefix=_CACHE_FILENAME, suffix=".tmp"
+ )
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ pickle.dump(payload, fh, protocol=pickle.HIGHEST_PROTOCOL)
+ os.replace(tmp_name, self._path)
+ except BaseException:
+ with contextlib.suppress(OSError):
+ os.unlink(tmp_name)
+ raise
+ except Exception as exc:
+ log.debug("duplication_cache_save_failed", error=str(exc))
+
+ # -- access ------------------------------------------------------------
+
+ def get(
+ self, content_hash: str
+ ) -> tuple[list[str], int, list[tuple[int, int, int, int]]] | None:
+ entry = self._entries.get(content_hash)
+ if entry is None:
+ self.misses += 1
+ return None
+ self.hits += 1
+ # Entries touched this run survive into the rewritten cache file.
+ self._fresh[content_hash] = entry
+ return entry
+
+ def put(
+ self,
+ content_hash: str,
+ kinds: list[str],
+ nloc: int,
+ windows: list[tuple[int, int, int, int]],
+ ) -> None:
+ # Interned kinds keep the pickle compact (each distinct kind is
+ # memoized once by identity) and make later equality checks cheap.
+ entry = ([sys.intern(k) for k in kinds], nloc, windows)
+ self._entries[content_hash] = entry
+ self._fresh[content_hash] = entry
diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.py
index d0e27ef5..24ab7d59 100644
--- a/packages/core/src/repowise/core/analysis/health/engine.py
+++ b/packages/core/src/repowise/core/analysis/health/engine.py
@@ -256,6 +256,7 @@ class HealthAnalyzer:
parsed_files: list[Any] | None = None,
coverage_map: dict[str, dict[str, Any]] | None = None,
module_map: dict[str, str] | None = None,
+ duplication_cache_dir: Any | None = None,
) -> None:
self.graph = graph
self.git_meta_map = git_meta_map or {}
@@ -271,6 +272,10 @@ class HealthAnalyzer:
# module rollups still group sensibly on small repos that didn't
# produce community labels.
self.module_map = module_map or {}
+ # Directory for the duplication token/window cache (typically the
+ # repo's ``.repowise``). None disables caching — the duplication
+ # pass then re-tokenizes everything, exactly as before.
+ self.duplication_cache_dir = duplication_cache_dir
def analyze(
self,
@@ -309,7 +314,11 @@ class HealthAnalyzer:
dup_report = DuplicationReport()
else:
try:
- dup_report = detect_clones(self.parsed_files, self.git_meta_map)
+ dup_report = detect_clones(
+ self.parsed_files,
+ self.git_meta_map,
+ cache_dir=self.duplication_cache_dir,
+ )
_log_duplication_diagnostics(dup_report)
except Exception as exc:
log.debug("health_duplication_failed", error=str(exc))
@@ -418,7 +427,10 @@ class HealthAnalyzer:
else:
try:
dup_report = await asyncio.to_thread(
- detect_clones, self.parsed_files, self.git_meta_map
+ detect_clones,
+ self.parsed_files,
+ self.git_meta_map,
+ cache_dir=self.duplication_cache_dir,
)
_log_duplication_diagnostics(dup_report)
except Exception as exc:
diff --git a/packages/core/src/repowise/core/generation/__init__.py b/packages/core/src/repowise/core/generation/__init__.py
index db00b92d..0f891ca5 100644
--- a/packages/core/src/repowise/core/generation/__init__.py
+++ b/packages/core/src/repowise/core/generation/__init__.py
@@ -5,105 +5,79 @@ Jinja2-templated prompts and BaseProvider.generate().
Import direction (strictly one-way):
ingestion.models ← generation.models ← context_assembler ← page_generator
+
+Exports resolve lazily (PEP 562): importing a light leaf module such as
+``generation.report`` from the index-only update path no longer pays for
+``context_assembler``/``page_generator``/``editor_files`` — they load on first
+attribute access instead.
"""
-from .context_assembler import (
- ApiContractContext,
- ArchitectureDiagramContext,
- ContextAssembler,
- FilePageContext,
- InfraPageContext,
- ModulePageContext,
- RepoOverviewContext,
- SccPageContext,
- SymbolSpotlightContext,
-)
-from .editor_files import (
- ClaudeMdGenerator,
- DecisionSummary,
- EditorFileData,
- EditorFileDataFetcher,
- HotspotFile,
- KeyModule,
- TechStackItem,
-)
-from .job_system import Checkpoint, JobStatus, JobSystem
-from .models import (
- GENERATION_LEVELS,
- ConfidenceDecayResult,
- DeadCodeConfig,
- FreshnessStatus,
- GeneratedPage,
- GenerationConfig,
- GitConfig,
- PageType,
- compute_confidence_decay_with_git,
- compute_freshness,
- compute_page_id,
- compute_source_hash,
- decay_confidence,
-)
-from .api_contract_detector import detect_code_api_contracts
-from .interlinking import (
- LinkIndex,
- WikiLink,
- attach_wiki_links_and_backlinks,
- resolve_wiki_links,
-)
-from .page_generator import SYSTEM_PROMPTS, PageGenerator
-from .selection import (
- BucketAllocation,
- ModuleGroup,
- Selection,
- SelectionInputs,
- select_pages,
- summarize_selection,
-)
+from importlib import import_module
+from typing import Any
+
+# Public name → defining submodule. Names import lazily on first access so
+# light consumers (e.g. the index-only update path importing ``.report``)
+# don't pull the heavy assembler/generator stack via this ``__init__``.
+_EXPORTS = {
+ "ApiContractContext": ".context_assembler",
+ "ArchitectureDiagramContext": ".context_assembler",
+ "ContextAssembler": ".context_assembler",
+ "FilePageContext": ".context_assembler",
+ "InfraPageContext": ".context_assembler",
+ "ModulePageContext": ".context_assembler",
+ "RepoOverviewContext": ".context_assembler",
+ "SccPageContext": ".context_assembler",
+ "SymbolSpotlightContext": ".context_assembler",
+ "ClaudeMdGenerator": ".editor_files",
+ "DecisionSummary": ".editor_files",
+ "EditorFileData": ".editor_files",
+ "EditorFileDataFetcher": ".editor_files",
+ "HotspotFile": ".editor_files",
+ "KeyModule": ".editor_files",
+ "TechStackItem": ".editor_files",
+ "Checkpoint": ".job_system",
+ "JobStatus": ".job_system",
+ "JobSystem": ".job_system",
+ "GENERATION_LEVELS": ".models",
+ "ConfidenceDecayResult": ".models",
+ "DeadCodeConfig": ".models",
+ "FreshnessStatus": ".models",
+ "GeneratedPage": ".models",
+ "GenerationConfig": ".models",
+ "GitConfig": ".models",
+ "PageType": ".models",
+ "compute_confidence_decay_with_git": ".models",
+ "compute_freshness": ".models",
+ "compute_page_id": ".models",
+ "compute_source_hash": ".models",
+ "decay_confidence": ".models",
+ "detect_code_api_contracts": ".api_contract_detector",
+ "LinkIndex": ".interlinking",
+ "WikiLink": ".interlinking",
+ "attach_wiki_links_and_backlinks": ".interlinking",
+ "resolve_wiki_links": ".interlinking",
+ "SYSTEM_PROMPTS": ".page_generator",
+ "PageGenerator": ".page_generator",
+ "BucketAllocation": ".selection",
+ "ModuleGroup": ".selection",
+ "Selection": ".selection",
+ "SelectionInputs": ".selection",
+ "select_pages": ".selection",
+ "summarize_selection": ".selection",
+}
+
+__all__ = sorted(_EXPORTS)
+
+
+def __getattr__(name: str) -> Any:
+ try:
+ module = _EXPORTS[name]
+ except KeyError:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+ value = getattr(import_module(module, __name__), name)
+ globals()[name] = value # cache: subsequent access skips __getattr__
+ return value
+
-__all__ = [
- "BucketAllocation",
- "GENERATION_LEVELS",
- "LinkIndex",
- "ModuleGroup",
- "SYSTEM_PROMPTS",
- "Selection",
- "SelectionInputs",
- "WikiLink",
- "attach_wiki_links_and_backlinks",
- "resolve_wiki_links",
- "select_pages",
- "summarize_selection",
- "ApiContractContext",
- "ArchitectureDiagramContext",
- "Checkpoint",
- "ClaudeMdGenerator",
- "ConfidenceDecayResult",
- "ContextAssembler",
- "DeadCodeConfig",
- "DecisionSummary",
- "EditorFileData",
- "EditorFileDataFetcher",
- "FilePageContext",
- "FreshnessStatus",
- "GeneratedPage",
- "GenerationConfig",
- "GitConfig",
- "HotspotFile",
- "InfraPageContext",
- "JobStatus",
- "JobSystem",
- "KeyModule",
- "ModulePageContext",
- "PageGenerator",
- "PageType",
- "RepoOverviewContext",
- "SccPageContext",
- "SymbolSpotlightContext",
- "TechStackItem",
- "compute_confidence_decay_with_git",
- "compute_freshness",
- "compute_page_id",
- "compute_source_hash",
- "decay_confidence",
- "detect_code_api_contracts",
-]
+def __dir__() -> list[str]:
+ return sorted(set(globals()) | set(_EXPORTS))
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index 1911fb07..75192e73 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -90,8 +90,14 @@ class ContextAssembler:
page_summaries: dict[str, str] | None = None,
decision_records: list[dict] | None = None,
kg_context: Any | None = None,
+ symbol_index: dict[str, list[tuple[Any, dict]]] | None = None,
) -> FilePageContext:
- """Assemble context for the file_page template."""
+ """Assemble context for the file_page template.
+
+ *symbol_index* (see :func:`build_symbol_index`) makes the call-graph /
+ heritage extraction a dict lookup instead of a full graph-node scan —
+ pass it when assembling context for many files against one graph.
+ """
path = parsed.file_info.path
budget = self._config.token_budget
used = 0
@@ -158,8 +164,8 @@ class ContextAssembler:
depth = self._select_generation_depth(path, git_meta, pagerank.get(path, 0.0))
# Graph intelligence: call graph, heritage, community metadata
- call_graph_entries = extract_call_graph(path, graph)
- heritage_entries = extract_heritage(path, graph)
+ call_graph_entries = extract_call_graph(path, graph, symbol_index)
+ heritage_entries = extract_heritage(path, graph, symbol_index)
community_label, community_cohesion = extract_community_meta(path, graph)
return FilePageContext(
diff --git a/packages/core/src/repowise/core/generation/context/graph_intelligence.py b/packages/core/src/repowise/core/generation/context/graph_intelligence.py
index a6fa3048..a9e69af5 100644
--- a/packages/core/src/repowise/core/generation/context/graph_intelligence.py
+++ b/packages/core/src/repowise/core/generation/context/graph_intelligence.py
@@ -9,15 +9,50 @@ from __future__ import annotations
from typing import Any
-def extract_call_graph(file_path: str, graph: Any) -> list[dict]:
- """Extract symbol-level call edges for symbols defined in this file."""
- entries: list[dict] = []
+def build_symbol_index(graph: Any) -> dict[str, list[tuple[Any, dict]]]:
+ """Bucket symbol nodes by ``file_path`` in one pass over the graph.
+
+ ``extract_call_graph`` / ``extract_heritage`` historically scanned every
+ graph node per file — O(files x nodes) across a generation run. Callers
+ that assemble context for many files build this index once and pass it
+ in; per-file extraction becomes a dict lookup. Buckets preserve the
+ graph's node iteration order, so results are byte-identical to the scan.
+ """
+ index: dict[str, list[tuple[Any, dict]]] = {}
try:
for node, data in graph.nodes(data=True):
if data.get("node_type") != "symbol":
continue
- if data.get("file_path") != file_path:
- continue
+ fp = data.get("file_path")
+ if fp:
+ index.setdefault(fp, []).append((node, data))
+ except Exception:
+ return {}
+ return index
+
+
+def _file_symbol_nodes(
+ file_path: str, graph: Any, symbol_index: dict[str, list[tuple[Any, dict]]] | None
+) -> Any:
+ """This file's symbol nodes — from the index when supplied, else a scan."""
+ if symbol_index is not None:
+ return symbol_index.get(file_path, [])
+ return (
+ (node, data)
+ for node, data in graph.nodes(data=True)
+ if data.get("node_type") == "symbol" and data.get("file_path") == file_path
+ )
+
+
+def extract_call_graph(
+ file_path: str,
+ graph: Any,
+ symbol_index: dict[str, list[tuple[Any, dict]]] | None = None,
+) -> list[dict]:
+ """Extract symbol-level call edges for symbols defined in this file."""
+ entries: list[dict] = []
+ try:
+ for node, data in _file_symbol_nodes(file_path, graph, symbol_index):
# Outgoing calls from this symbol
for _, target, edata in graph.out_edges(node, data=True):
if edata.get("edge_type") == "calls":
@@ -56,15 +91,15 @@ def extract_call_graph(file_path: str, graph: Any) -> list[dict]:
return unique[:15]
-def extract_heritage(file_path: str, graph: Any) -> list[dict]:
+def extract_heritage(
+ file_path: str,
+ graph: Any,
+ symbol_index: dict[str, list[tuple[Any, dict]]] | None = None,
+) -> list[dict]:
"""Extract extends/implements edges for symbols in this file."""
entries: list[dict] = []
try:
- for node, data in graph.nodes(data=True):
- if data.get("node_type") != "symbol":
- continue
- if data.get("file_path") != file_path:
- continue
+ for node, data in _file_symbol_nodes(file_path, graph, symbol_index):
for _, target, edata in graph.out_edges(node, data=True):
etype = edata.get("edge_type", "")
if etype in ("extends", "implements"):
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index 67458fb4..8a4f46f6 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -208,15 +208,26 @@ class PageGenerator(PerTypeGenerationMixin):
self,
parsed: ParsedFile,
ctx: FilePageContext,
+ rag_prefetched: bool = False,
) -> GeneratedPage:
- """Generate a file_page from a pre-assembled context (avoids double-assembly)."""
+ """Generate a file_page from a pre-assembled context (avoids double-assembly).
+
+ *rag_prefetched* is set by the level-2 builder when RAG context was
+ already resolved for the whole level in one batched search (see
+ ``levels._prefetch_rag_context``) — the per-page search below would
+ re-fetch identical results inside the LLM semaphore, so it is skipped.
+ """
# RAG context: query vector store for related pages (B1).
# Gated by two short-circuits so we don't burn an embedder
# round-trip on every page when the result wouldn't help:
# 1. ``enable_rag_context`` config flag (off → fully skip).
# 2. ``rag_min_store_size`` — early pages run against an empty
# or near-empty store and the search returns nothing useful.
- if self._vector_store is not None and getattr(self._config, "enable_rag_context", True):
+ if (
+ not rag_prefetched
+ and self._vector_store is not None
+ and getattr(self._config, "enable_rag_context", True)
+ ):
min_store_size = max(0, int(getattr(self._config, "rag_min_store_size", 10) or 0))
store_ok = True
if min_store_size > 0:
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index ff088069..80ea7ce8 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -130,6 +130,59 @@ async def _prefetch_dependency_summaries(run: _GenerationRun) -> None:
log.debug("rag.batch_dep_prefetch_failed", error=str(exc))
+async def _prefetch_rag_context(
+ run: _GenerationRun, items: list[tuple[Any, FilePageContext]]
+) -> bool:
+ """Resolve RAG context for all tier-1 file pages in one batched search.
+
+ Replicates the per-page gating in ``_generate_file_page_from_ctx`` (flag,
+ store size, query-term derivation, self-exclusion) but runs BEFORE the
+ level starts, outside the LLM semaphore, with all queries embedded in a
+ single embedder call via :meth:`VectorStore.search_many`. The store is
+ static for the whole level (pages embed in one batch at level end), so
+ prefetched results are identical to what each page would have fetched.
+
+ Returns True when the per-page search can be skipped (results resolved,
+ or the per-page gate would have skipped every search anyway); False on
+ batch failure so pages fall back to the per-page path.
+ """
+ if run.vector_store is None or not getattr(run.config, "enable_rag_context", True):
+ return False
+ min_store_size = max(0, int(getattr(run.config, "rag_min_store_size", 10) or 0))
+ if min_store_size > 0:
+ try:
+ current_ids = await run.vector_store.list_page_ids()
+ if len(current_ids) < min_store_size:
+ # Store too small for useful RAG — the per-page gate would
+ # skip every search this level, so there is nothing to fetch.
+ return True
+ except Exception:
+ pass # same as the per-page gate: fall through to the search
+ queries: list[str] = []
+ targets: list[tuple[Any, FilePageContext]] = []
+ for p, ctx in items:
+ query_terms = p.exports or [
+ s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"
+ ]
+ if not query_terms:
+ continue
+ queries.append(", ".join(query_terms[:5]))
+ targets.append((p, ctx))
+ if not queries:
+ return True
+ try:
+ all_results = await run.vector_store.search_many(queries, limit=3)
+ except Exception as exc:
+ log.debug("rag.batch_search_failed", error=str(exc))
+ return False
+ for (p, ctx), results in zip(targets, all_results, strict=False):
+ self_id = f"file_page:{p.file_info.path}"
+ ctx.rag_context = [
+ f"[{r.page_id}]\n{r.snippet}" for r in results if r.page_id != self_id
+ ]
+ return True
+
+
async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
"""Level 2 (file_page): topo-ordered context assembly + tier routing.
@@ -141,7 +194,13 @@ async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
_topo_order_code_files(run)
await _prefetch_dependency_summaries(run)
- coros: list[tuple[str, Any]] = []
+ # One pass over the graph's symbol nodes feeds every file's call-graph /
+ # heritage extraction (instead of a full node scan per file).
+ from ..context.graph_intelligence import build_symbol_index
+
+ symbol_index = build_symbol_index(run.graph)
+
+ items: list[tuple[Any, FilePageContext]] = []
for p in run.code_files:
kg_file_ctx = run.kg_ctx.get_file_context(p.file_info.path) if run.kg_ctx.available else None
ctx: FilePageContext = gen._assembler.assemble_file_page(
@@ -156,13 +215,32 @@ async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
dead_code_findings=run.dead_code_by_file.get(p.file_info.path),
decision_records=run.decisions_by_file.get(p.file_info.path),
kg_context=kg_file_ctx,
+ symbol_index=symbol_index,
)
run.file_page_contexts[p.file_info.path] = ctx
+ items.append((p, ctx))
+
+ def _emits_tier1(p: Any) -> bool:
+ path = p.file_info.path
+ return (
+ path in run.sel_file_paths
+ and path in run.tier1_paths
+ and compute_page_id("file_page", path) not in run.completed_ids
+ )
+
+ rag_prefetched = await _prefetch_rag_context(
+ run, [(p, ctx) for p, ctx in items if _emits_tier1(p)]
+ )
+
+ coros: list[tuple[str, Any]] = []
+ for p, ctx in items:
path = p.file_info.path
pid = compute_page_id("file_page", path)
if path in run.sel_file_paths and pid not in run.completed_ids:
if path in run.tier1_paths:
- coros.append((pid, gen._generate_file_page_from_ctx(p, ctx)))
+ coros.append(
+ (pid, gen._generate_file_page_from_ctx(p, ctx, rag_prefetched=rag_prefetched))
+ )
else:
coros.append((pid, gen._generate_file_page_tier2(p, ctx)))
return coros
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 264a6f6f..344bfda4 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -365,6 +365,16 @@ class _GenerationRun:
error=str(exc),
)
return exc # return as value so gather works
+ except BaseException:
+ # Cancellation (Ctrl+C teardown): CancelledError is a
+ # BaseException, so it skips the handler above. If the cancel
+ # landed while this page was still queued on the semaphore,
+ # ``coro`` was never started — close it so interpreter
+ # shutdown doesn't spray one "coroutine ... was never
+ # awaited" RuntimeWarning per pending page (issue #358).
+ # close() is a no-op on a coroutine that already ran.
+ coro.close()
+ raise
tasks = [guarded_named(pid, c) for pid, c in named_coros]
results = await asyncio.gather(*tasks)
diff --git a/packages/core/src/repowise/core/ingestion/call_resolver.py b/packages/core/src/repowise/core/ingestion/call_resolver.py
index 8d3577d9..392c553a 100644
--- a/packages/core/src/repowise/core/ingestion/call_resolver.py
+++ b/packages/core/src/repowise/core/ingestion/call_resolver.py
@@ -64,6 +64,7 @@ class CallResolver:
import_targets: dict[str, set[str]],
*,
repo_path: str | None = None,
+ import_maps: Any | None = None,
) -> None:
# Per-file symbol index: {file_path: {symbol_name: symbol_id}}
self._file_symbols: dict[str, dict[str, str]] = {}
@@ -71,6 +72,11 @@ class CallResolver:
# Per-file method index: {file_path: {(class_name, method_name): symbol_id}}
self._file_methods: dict[str, dict[tuple[str, str], str]] = {}
+ # Global method index: {(class_name, method_name): [(file_path, symbol_id)]}
+ # in file-insertion order — replaces the trait-dispatch scan over
+ # every file's method dict with one short-list lookup.
+ self._global_methods: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list)
+
# Global symbol index: {name: [symbol_ids]} — for Tier 3
self._global_symbols: dict[str, list[str]] = defaultdict(list)
@@ -80,14 +86,27 @@ class CallResolver:
# Import graph: {file_path: set of imported file paths}
self._import_targets = import_targets
- # Import name mapping: {file_path: {local_name: source_file}}
- self._import_names: dict[str, dict[str, str]] = defaultdict(dict)
+ # Shared import-name maps (built once per GraphBuilder.build() and
+ # injected; standalone construction builds them locally).
+ if import_maps is None:
+ from .import_index import build_import_name_maps
+ import_maps = build_import_name_maps(parsed_files)
+ # Import name mapping: {file_path: {local_name: source_file}}
+ self._import_names: dict[str, dict[str, str]] = import_maps.import_names
# Full binding data: {file_path: {local_name: NamedBinding}}
- self._import_bindings: dict[str, dict[str, NamedBinding]] = defaultdict(dict)
-
+ self._import_bindings: dict[str, dict[str, NamedBinding]] = import_maps.import_bindings
# Module alias mapping: {file_path: {alias: source_file}}
- self._module_aliases: dict[str, dict[str, str]] = defaultdict(dict)
+ self._module_aliases: dict[str, dict[str, str]] = import_maps.module_aliases
+
+ # Lazy per-file merged views of every imported file's symbol /
+ # method tables — turns the Tier-2b "scan each imported file"
+ # loops into single dict lookups. Built on first miss per file;
+ # merge order is sorted(import paths) with first-wins so shadowed
+ # names resolve deterministically (the old set-iteration order was
+ # hash-randomized per process).
+ self._merged_import_symbols: dict[str, dict[str, str]] = {}
+ self._merged_import_methods: dict[str, dict[tuple[str, str], str]] = {}
# Barrel re-export origins: {barrel_file: {name: origin_file}}
self._barrel_origins: dict[str, dict[str, str]] = defaultdict(dict)
@@ -366,7 +385,10 @@ class CallResolver:
return None
def _build_indices(self, parsed_files: dict[str, ParsedFile]) -> None:
- """Build all lookup indices from parsed file data."""
+ """Build symbol lookup indices from parsed file data.
+
+ (Import-name maps are shared — see ``import_index.build_import_name_maps``.)
+ """
for path, parsed in parsed_files.items():
file_syms: dict[str, str] = {}
file_methods: dict[tuple[str, str], str] = {}
@@ -377,7 +399,9 @@ class CallResolver:
# Method index: (class_name, method_name) → symbol_id
if sym.parent_name:
- file_methods[(sym.parent_name, sym.name)] = sym.id
+ key = (sym.parent_name, sym.name)
+ file_methods[key] = sym.id
+ self._global_methods[key].append((path, sym.id))
# Global indices
self._global_symbols[sym.name].append(sym.id)
@@ -387,25 +411,35 @@ class CallResolver:
self._file_symbols[path] = file_syms
self._file_methods[path] = file_methods
- # Build import-name mapping using per-import resolved_file
- for imp in parsed.imports:
- if imp.resolved_file is None:
+ def _merged_symbols_for(self, file_path: str) -> dict[str, str]:
+ """Merged ``{name → symbol_id}`` across every file *file_path* imports.
+
+ Sorted-path merge order with first-wins gives deterministic
+ precedence for names exported by multiple imports.
+ """
+ merged = self._merged_import_symbols.get(file_path)
+ if merged is None:
+ merged = {}
+ for imported_file in sorted(self._import_targets.get(file_path, ())):
+ if imported_file.startswith("external:"):
continue
- resolved = imp.resolved_file
- if imp.bindings:
- for binding in imp.bindings:
- if binding.local_name == "*":
- continue
- binding.source_file = resolved
- self._import_names[path][binding.local_name] = resolved
- self._import_bindings[path][binding.local_name] = binding
- if binding.is_module_alias:
- self._module_aliases[path][binding.local_name] = resolved
- else:
- # Fallback for imports without binding data
- for name in imp.imported_names:
- if name != "*":
- self._import_names[path][name] = resolved
+ for name, sym_id in self._file_symbols.get(imported_file, {}).items():
+ merged.setdefault(name, sym_id)
+ self._merged_import_symbols[file_path] = merged
+ return merged
+
+ def _merged_methods_for(self, file_path: str) -> dict[tuple[str, str], str]:
+ """Merged ``{(class, method) → symbol_id}`` across imports (see above)."""
+ merged = self._merged_import_methods.get(file_path)
+ if merged is None:
+ merged = {}
+ for imported_file in sorted(self._import_targets.get(file_path, ())):
+ if imported_file.startswith("external:"):
+ continue
+ for key, sym_id in self._file_methods.get(imported_file, {}).items():
+ merged.setdefault(key, sym_id)
+ self._merged_import_methods[file_path] = merged
+ return merged
def resolve_file(self, file_path: str, calls: list[CallSite]) -> list[ResolvedCall]:
"""Resolve all calls in a single file to symbol-level edges."""
@@ -504,13 +538,10 @@ class CallResolver:
if target_name in source_syms:
return ResolvedCall(caller_id, source_syms[target_name], 0.90, call.line)
- # 2b: Check all imported files for the symbol
- for imported_file in self._import_targets.get(file_path, set()):
- if imported_file.startswith("external:"):
- continue
- imported_syms = self._file_symbols.get(imported_file, {})
- if target_name in imported_syms:
- return ResolvedCall(caller_id, imported_syms[target_name], 0.85, call.line)
+ # 2b: Check all imported files for the symbol (pre-merged lookup)
+ merged_syms = self._merged_symbols_for(file_path)
+ if target_name in merged_syms:
+ return ResolvedCall(caller_id, merged_syms[target_name], 0.85, call.line)
# Tier 3: global unique match — only within the same language
candidates = self._global_symbols.get(target_name, [])
@@ -591,22 +622,20 @@ class CallResolver:
if key in file_methods:
return ResolvedCall(caller_id, file_methods[key], 0.93, call.line)
- # Check imported files for (class, method) pairs
- for imported_file in self._import_targets.get(file_path, set()):
- if imported_file.startswith("external:"):
- continue
- imp_methods = self._file_methods.get(imported_file, {})
- if key in imp_methods:
- return ResolvedCall(caller_id, imp_methods[key], 0.88, call.line)
+ # Check imported files for (class, method) pairs (pre-merged lookup)
+ merged_methods = self._merged_methods_for(file_path)
+ if key in merged_methods:
+ return ResolvedCall(caller_id, merged_methods[key], 0.88, call.line)
# Strategy 2b: trait method dispatch — receiver is a type that
# implements a trait; the method may be defined on the trait's
- # impl block in another file.
- for _path, methods in self._file_methods.items():
+ # impl block in another file. The global index preserves the old
+ # file-insertion match order; entries from the caller's own file
+ # are skipped exactly as before.
+ for _path, sym_id in self._global_methods.get(key, ()):
if _path == file_path:
continue
- if key in methods:
- return ResolvedCall(caller_id, methods[key], 0.75, call.line)
+ return ResolvedCall(caller_id, sym_id, 0.75, call.line)
# Strategy 3: receiver is "self" or "this" — look in same class
if receiver_name in ("self", "this"):
diff --git a/packages/core/src/repowise/core/ingestion/extractors/heritage/__init__.py b/packages/core/src/repowise/core/ingestion/extractors/heritage/__init__.py
index aa464034..cff6d725 100644
--- a/packages/core/src/repowise/core/ingestion/extractors/heritage/__init__.py
+++ b/packages/core/src/repowise/core/ingestion/extractors/heritage/__init__.py
@@ -60,22 +60,18 @@ HERITAGE_EXTRACTORS: dict[str, Callable[..., None]] = {
def extract_heritage(
- tree: object,
- query: object,
+ matches: list[dict],
config: object,
file_info: object,
src: str,
- *,
- run_query: Callable,
) -> list[HeritageRelation]:
"""Extract inheritance/implementation relationships from class definitions.
- Walks the same @symbol.def captures used by _extract_symbols, extracting
- superclass/interface/trait information from the definition AST nodes.
+ Walks the same @symbol.def captures used by _extract_symbols (the parser
+ executes the compiled query once per file and shares the capture dicts
+ across all extraction passes), extracting superclass/interface/trait
+ information from the definition AST nodes.
"""
- if query is None:
- return []
-
lang = file_info.language # type: ignore[attr-defined]
heritage_types = heritage_node_types_for(lang)
if not heritage_types:
@@ -88,7 +84,7 @@ def extract_heritage(
relations: list[HeritageRelation] = []
seen: set[tuple[int, str]] = set()
- for capture_dict in run_query(query, tree.root_node): # type: ignore[attr-defined]
+ for capture_dict in matches:
def_nodes = capture_dict.get("symbol.def", [])
name_nodes = capture_dict.get("symbol.name", [])
diff --git a/packages/core/src/repowise/core/ingestion/git_commit_index.py b/packages/core/src/repowise/core/ingestion/git_commit_index.py
index 01719a3f..5fb248ab 100644
--- a/packages/core/src/repowise/core/ingestion/git_commit_index.py
+++ b/packages/core/src/repowise/core/ingestion/git_commit_index.py
@@ -36,6 +36,7 @@ def load_commit_index(
*,
commit_sink: list[dict] | None = None,
since_ts: int | None = None,
+ provenance_classifier: object | None = None,
) -> dict[str, list[_CommitRec]]:
"""Bucket every commit in the recent history by the files it touched.
@@ -66,6 +67,13 @@ def load_commit_index(
without re-walking the full window. Default (``None``) processes the whole
depth.
+ *provenance_classifier* (an
+ :class:`~.git_indexer.agent_provenance.AgentProvenanceClassifier`) labels
+ each commit's agent provenance once per commit — pure in-memory regex on
+ the already-parsed record, no extra git pass. ``None`` uses the built-in
+ pattern registry; callers with repo-local pattern extensions pass the
+ config-aware instance instead.
+
Failures (git unavailable, corrupt log output, etc.) return an
empty dict so the caller can fall back to per-file indexing.
"""
@@ -78,6 +86,10 @@ def load_commit_index(
_extract_rename_paths,
_parse_commit_record,
)
+ from .git_indexer.agent_provenance import AgentProvenanceClassifier
+
+ if provenance_classifier is None:
+ provenance_classifier = AgentProvenanceClassifier()
try:
raw = repo.git.log( # type: ignore[attr-defined]
@@ -114,6 +126,16 @@ def load_commit_index(
continue
commits_parsed += 1
+ # Agent provenance — classified ONCE per commit here (not per touched
+ # file): the per-file records below share the result by reference.
+ prov = provenance_classifier.classify( # type: ignore[attr-defined]
+ header["author_name"],
+ header["author_email"],
+ header["committer_name"],
+ header["committer_email"],
+ f"{header['subject']}\n{header['body']}",
+ )
+
# Full change footprint of this commit (every file, not just the
# indexable subset) — only accumulated when a sink is requested so the
# default path pays nothing.
@@ -162,6 +184,8 @@ def load_commit_index(
body=header["body"],
added=added,
deleted=deleted,
+ agent=prov.agent,
+ agent_tier=prov.autonomy_tier,
)
)
@@ -174,6 +198,10 @@ def load_commit_index(
"ts": header["ts"],
"subject": header["subject"],
"changes": commit_changes,
+ "agent_name": prov.agent,
+ "agent_autonomy_tier": prov.autonomy_tier,
+ "agent_channel": prov.channel,
+ "agent_confidence": prov.confidence,
}
)
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/README.md b/packages/core/src/repowise/core/ingestion/git_indexer/README.md
index c9fefa60..775a9604 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/README.md
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/README.md
@@ -17,7 +17,9 @@ layers consume:
- co-change partners and Hassan **change entropy** (History Complexity Metric);
- per-function blame (modification counts, line age) for function-level health;
- **prior-defect history** — bug-fix commits touching each file in the trailing
- window, the most cost-effective defect predictor.
+ window, the most cost-effective defect predictor;
+- **agent provenance** — which coding agent (if any) authored each commit, at
+ what autonomy tier, plus per-file `agent_authored_pct` rollups.
## Window anchoring (`REPOWISE_GIT_WINDOW_ANCHOR`)
@@ -94,6 +96,41 @@ live `repowise risk` path) is reconstructed **in memory** here: walking commits
oldest→newest, each author's prior-commit count is the running tally. Empty in
rename-tracking mode (which uses the per-file walk, not the batched index).
+## Agent provenance
+
+`agent_provenance.py` labels every walked commit
+`{agent, autonomy_tier, channel, confidence}` from the attribution channels
+present in **local git history** — identity fields (bot accounts / service
+e-mails), exact message footers ("Generated with Claude Code", Codex,
+opencode, aider), and `Co-authored-by:` trailers anchored to service
+identities. Tiers: **1** near-autonomous (an agent service account authored
+the commit) · **2** human-driven agent (footer, or a service identity as
+*committer* over a human author) · **3** assisted (co-author trailer only).
+
+Design rules:
+
+- **Precision-first.** Every pattern is anchored to a service identity; a bare
+ name never matches (a human contributor named "Devin" must not become the
+ Devin agent). A false "agent-authored" label on a human commit is worse
+ than a miss.
+- **Local channels only.** PR-level evidence (bot PR authors, agent branch
+ prefixes, PR-body footers) needs the forge API and is out of scope — squash
+ merges that strip trailers are a known *recall* loss, stated rather than
+ patched with network calls at index time.
+- **Zero extra cost.** Classification rides the existing commit-index walk —
+ pure in-memory regex on already-parsed records, ~20 µs/commit (≈10 ms at the
+ default commit depth), no additional `git` pass.
+- **Config-driven.** Repos can extend the registry from
+ `.repowise/config.yaml` under an `agent_provenance:` block
+ (`service_emails`, `footer_patterns`, `coauthor_patterns`) — additive on top
+ of the built-ins, malformed entries skipped with a warning.
+
+Persisted as four nullable columns on `git_commits`
+(`agent_name`/`agent_autonomy_tier`/`agent_channel`/`agent_confidence`) and a
+per-file rollup on `git_metadata` (`agent_commit_count`, `agent_authored_pct`,
+`agent_tier_counts_json`), surfaced in the `/git-metadata` file view and the
+MCP `get_context` ownership block.
+
## Internal layout
| Module | Contents |
@@ -102,6 +139,7 @@ rename-tracking mode (which uses the per-file walk, not the batched index).
| `_constants.py` | commit-depth defaults, decay half-lives, the bug-fix keyword classifier (`is_fix_commit`, mirrors the benchmark) + `PRIOR_DEFECT_WINDOW_DAYS`, skip heuristics, GitPython noise patch |
| `records.py` | `_CommitRec`, `GitIndexSummary` (carries `commit_rows`), the `git log` record format + rename/skip path helpers |
| `commit_rows.py` | `build_commit_rows` — per-commit Kamei features + change-risk from the walk's sunk commits (pure, in-memory author experience) |
+| `agent_provenance.py` | `AgentProvenanceClassifier` — deterministic per-commit agent attribution (local channels, config-extensible) |
| `file_history.py` | `index_file` — per-file parse + base metrics (blame gated by tier) |
| `enrich.py` | blame ownership, commit significance, rename detection, percentiles |
| `function_blame.py` | per-line blame index → per-function modification counts + line age (FULL tier; feeds `function_hotspot` / `code_age_volatility`) |
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py b/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py
index 3a361f9a..2546ec4e 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py
@@ -20,6 +20,11 @@ This package was split out of a single 1k-line module; the public import path
from __future__ import annotations
from ._constants import HOTSPOT_HALFLIFE_DAYS, is_fix_commit
+from .agent_provenance import (
+ AgentProvenance,
+ AgentProvenanceClassifier,
+ classifier_from_repo_config,
+)
from .backfill import BACKFILL_PHASE, backfill_full_tier
from .co_change import compute_co_changes, compute_co_changes_and_entropy
from .enrich import (
@@ -52,6 +57,8 @@ __all__ = [
"_FIELD_SEP",
"_LOG_FORMAT",
"_RECORD_SEP",
+ "AgentProvenance",
+ "AgentProvenanceClassifier",
"GitIndexSummary",
"GitIndexTier",
"GitIndexer",
@@ -60,6 +67,7 @@ __all__ = [
"_parse_commit_record",
"_should_skip_index",
"backfill_full_tier",
+ "classifier_from_repo_config",
"compute_co_changes",
"compute_co_changes_and_entropy",
"compute_percentiles",
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/agent_provenance.py b/packages/core/src/repowise/core/ingestion/git_indexer/agent_provenance.py
new file mode 100644
index 00000000..1286985c
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/agent_provenance.py
@@ -0,0 +1,255 @@
+"""Per-commit agent-provenance classification — deterministic, precision-first.
+
+Labels a commit ``{agent, autonomy_tier, channel, confidence}`` from the
+attribution channels available in **local git history** (identity fields,
+message footers, ``Co-authored-by:`` trailers — no network, no API calls, no
+LLM), in strip-resistance order:
+
+ Tier 1 (near-autonomous — an agent service account authored the commit):
+ * commit AUTHOR is a known agent bot account / service e-mail
+ (``copilot-swe-agent[bot]``, ``devin-ai-integration[bot]``,
+ ``[email protected]``, ...)
+ Tier 2 (human-driven agent — the agent wrote the change):
+ * commit-message footer (exact service phrases: "Generated with Claude
+ Code", opencode, Codex) or the aider author-name suffix ``... (aider)``
+ * a service identity as COMMITTER over a human author (the agent
+ pushed/amended a human-driven change — deliberately NOT tier 1)
+ Tier 3 (assisted — an agent contributed, a human authored):
+ * ``Co-authored-by:`` trailer naming a known agent service identity
+
+Precedence is tier order (1 > 2 > 3); first match wins. A commit with no
+match is human (``None``). Every pattern is anchored to a *service identity*
+(bot login, service e-mail, exact footer phrase) — never a bare name. A loose
+name pattern is how a human contributor named "Devin" gets misclassified as
+the Devin agent; a false "agent-authored" label on a human commit is worse
+than a miss.
+
+PR-level channels (bot PR author, agent branch prefixes, PR-body footers)
+require the forge API and are intentionally out of scope here: squash merges
+that strip commit trailers are therefore a known recall loss, not a precision
+loss. ``confidence`` is carried per channel so future, weaker channels can be
+filtered by consumers; every local channel is ``"high"``.
+
+The default registry below is data, not code: repos can extend it from
+``.repowise/config.yaml`` (see :func:`classifier_from_repo_config`) without
+touching this module, e.g.::
+
+ agent_provenance:
+ service_emails:
+ [email protected]: {agent: mybot, tier: 1}
+ footer_patterns:
+ - {pattern: "Generated by MyBot", agent: mybot}
+ coauthor_patterns:
+ - {pattern: "^co-authored-by: mybot <[^>]*@example\\.com>", agent: mybot}
+
+Classification is pure in-memory regex over already-parsed commit records and
+rides the existing commit-index walk — it adds no git subprocess and no I/O.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+__all__ = [
+ "AgentProvenance",
+ "AgentProvenanceClassifier",
+ "classifier_from_repo_config",
+]
+
+
+# ------------------------------------------------------------------ registry --
+
+# Bot login (lowercase, without the ``[bot]`` suffix) → agent name, for
+# ``<login>[bot]@users.noreply.github.com`` author/committer e-mails (GitHub's
+# canonical bot identity, also re-injected on squash merges).
+_BOT_LOGINS: dict[str, str] = {
+ "copilot-swe-agent": "copilot",
+ "copilot": "copilot",
+ "devin-ai-integration": "devin",
+ "cursor": "cursor",
+ "cursoragent": "cursor",
+ "google-labs-jules": "jules",
+ "codegen-sh": "codegen",
+ "openhands-ai": "openhands",
+ "sweep-ai": "sweep",
+ "claude": "claude",
+}
+
+# Exact service e-mails → (agent, tier) for commit identity fields.
+_SERVICE_EMAILS: dict[str, tuple[str, int]] = {
+ "[email protected]": ("cursor", 1),
+ "devin-ai-integration[bot]@users.noreply.github.com": ("devin", 1),
+}
+
+# Commit-message footers → agent (exact service phrases only).
+_FOOTER_PATTERNS: list[tuple[str, str]] = [
+ (r"generated with \[?claude code\]?", "claude"),
+ (r"generated with \[?opencode\]?", "opencode"),
+ (r"generated with \[?(?:openai )?codex\]?", "codex"),
+ (r"^\s*aider wrote this", "aider"),
+]
+
+# ``Co-authored-by:`` trailers → agent. Anchored to the trailer position AND
+# the agent's service identity — a name alone is not enough (see module doc).
+_COAUTHOR_PATTERNS: list[tuple[str, str]] = [
+ (
+ r"^co-authored-by:\s*claude(?:\s+(?:opus|sonnet|haiku)[^<]*)?\s*<[^>]*@anthropic\.com>",
+ "claude",
+ ),
+ (r"^co-authored-by:\s*copilot\s*<\d+\+copilot@users\.noreply\.github\.com>", "copilot"),
+ (
+ r"^co-authored-by:\s*cursor(?:\s*agent)?\s*<(?:cursoragent@cursor\.com|[^>]*cursor[^>]*)>",
+ "cursor",
+ ),
+ (r"^co-authored-by:\s*opencode\s*<", "opencode"),
+ (r"^co-authored-by:\s*devin[^<]*<[^>]*devin-ai-integration[^>]*>", "devin"),
+ (r"^co-authored-by:\s*aider[^<]*<[^>]*aider[^>]*>", "aider"),
+]
+
+_AIDER_NAME_RE = re.compile(r"\(aider\)\s*$")
+
+
+@dataclass(frozen=True)
+class AgentProvenance:
+ """One commit's agent attribution. ``agent is None`` means human."""
+
+ agent: str | None = None
+ autonomy_tier: int | None = None # 1 near-autonomous · 2 human-driven · 3 assisted
+ channel: str | None = None
+ confidence: str | None = None # "high" | "medium" (local channels: always high)
+
+
+_HUMAN = AgentProvenance()
+
+
+class AgentProvenanceClassifier:
+ """Compiled pattern registry + the classification rule.
+
+ Construct once per index run (compilation is the only non-trivial cost);
+ :meth:`classify` is then a handful of anchored regex searches per commit.
+ Extra patterns merge on top of the built-in registry; they never replace
+ it (precision of the defaults is validated, user patterns are additive).
+ """
+
+ def __init__(
+ self,
+ *,
+ extra_service_emails: dict[str, tuple[str, int]] | None = None,
+ extra_footer_patterns: list[tuple[str, str]] | None = None,
+ extra_coauthor_patterns: list[tuple[str, str]] | None = None,
+ ) -> None:
+ self._service_emails = {**_SERVICE_EMAILS, **(extra_service_emails or {})}
+ # ``[bot]`` is optional: GitHub squash-merges authored by the Copilot
+ # coding agent use ``<id>[email protected]`` (no
+ # suffix). Safe because each login is a unique, vendor-owned GitHub
+ # account — no human can register them.
+ self._bot_email_re = re.compile(
+ r"^(?:\d+\+)?("
+ + "|".join(re.escape(b) for b in _BOT_LOGINS)
+ + r")(?:\[bot\])?@users\.noreply\.github\.com$",
+ re.IGNORECASE,
+ )
+ self._footers = [
+ (re.compile(p, re.IGNORECASE | re.MULTILINE), agent)
+ for p, agent in [*_FOOTER_PATTERNS, *(extra_footer_patterns or [])]
+ ]
+ self._coauthors = [
+ (re.compile(p, re.IGNORECASE | re.MULTILINE), agent)
+ for p, agent in [*_COAUTHOR_PATTERNS, *(extra_coauthor_patterns or [])]
+ ]
+
+ def _identity_match(self, email: str) -> tuple[str, int] | None:
+ e = (email or "").strip().lower()
+ if not e:
+ return None
+ hit = self._service_emails.get(e)
+ if hit:
+ return hit
+ m = self._bot_email_re.match(e)
+ if m:
+ return (_BOT_LOGINS[m.group(1).lower()], 1)
+ return None
+
+ def classify(
+ self,
+ author_name: str,
+ author_email: str,
+ committer_name: str,
+ committer_email: str,
+ message: str,
+ ) -> AgentProvenance:
+ """Classify one commit from local metadata. *message* is the full
+ commit message (subject + body) — trailers and footers live in the
+ body, so passing the subject alone silently disables two channels."""
+ # T1 — agent service identity AUTHORED the commit.
+ hit = self._identity_match(author_email)
+ if hit:
+ return AgentProvenance(hit[0], hit[1], "service_email", "high")
+ # T2 — message footer / aider author-name suffix.
+ for pat, agent in self._footers:
+ if pat.search(message):
+ return AgentProvenance(agent, 2, "message_footer", "high")
+ if _AIDER_NAME_RE.search(author_name or ""):
+ return AgentProvenance("aider", 2, "author_name_aider", "high")
+ # T2 — service identity as COMMITTER over a human author: the agent
+ # pushed/amended a human-driven change. Deliberately tier 2, not 1.
+ hit = self._identity_match(committer_email)
+ if hit:
+ return AgentProvenance(hit[0], max(hit[1], 2), "service_committer", "high")
+ # T3 — co-author trailer naming an agent service identity.
+ for pat, agent in self._coauthors:
+ if pat.search(message):
+ return AgentProvenance(agent, 3, "coauthor_trailer", "high")
+ return _HUMAN
+
+
+def classifier_from_repo_config(repo_path: object) -> AgentProvenanceClassifier:
+ """Build a classifier with any repo-local pattern extensions.
+
+ Reads the optional ``agent_provenance`` block of ``.repowise/config.yaml``
+ (see module docstring for the shape). Malformed entries are skipped with a
+ warning — a broken user pattern must never take down the git index.
+ """
+ extra_emails: dict[str, tuple[str, int]] = {}
+ extra_footers: list[tuple[str, str]] = []
+ extra_coauthors: list[tuple[str, str]] = []
+ try:
+ from ...repo_config import load_repo_config
+
+ block = load_repo_config(repo_path).get("agent_provenance") or {} # type: ignore[arg-type]
+ if isinstance(block, dict):
+ for email, spec in (block.get("service_emails") or {}).items():
+ if isinstance(spec, dict) and spec.get("agent"):
+ tier = max(1, min(3, int(spec.get("tier", 1))))
+ extra_emails[str(email).lower()] = (str(spec["agent"]), tier)
+ for key, sink in (
+ ("footer_patterns", extra_footers),
+ ("coauthor_patterns", extra_coauthors),
+ ):
+ for entry in block.get(key) or []:
+ if not (
+ isinstance(entry, dict) and entry.get("pattern") and entry.get("agent")
+ ):
+ continue
+ try:
+ re.compile(entry["pattern"])
+ except re.error as exc:
+ logger.warning(
+ "agent_provenance_pattern_invalid",
+ pattern=entry["pattern"],
+ error=str(exc),
+ )
+ continue
+ sink.append((entry["pattern"], str(entry["agent"])))
+ except Exception as exc: # config parsing must never break indexing
+ logger.warning("agent_provenance_config_failed", error=str(exc))
+ return AgentProvenanceClassifier(
+ extra_service_emails=extra_emails,
+ extra_footer_patterns=extra_footers,
+ extra_coauthor_patterns=extra_coauthors,
+ )
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py b/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py
index d49a2b7d..1ce3cedb 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py
@@ -98,6 +98,12 @@ def build_commit_rows(parsed_commits: list[dict]) -> list[dict]:
"author_experience": exp,
"change_risk_score": risk.score,
"change_risk_level": risk.level,
+ # Agent provenance — classified during the walk (commit-index
+ # sink); pure passthrough here so this transform stays git-free.
+ "agent_name": c.get("agent_name"),
+ "agent_autonomy_tier": c.get("agent_autonomy_tier"),
+ "agent_channel": c.get("agent_channel"),
+ "agent_confidence": c.get("agent_confidence"),
}
)
return rows
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py b/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py
index 59e81574..938ef0a4 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py
@@ -208,15 +208,18 @@ def compute_percentiles(metadata_list: list[dict]) -> None:
total = len(metadata_list)
for rank, idx in enumerate(sorted_by_churn):
- metadata_list[idx]["churn_percentile"] = rank / total if total > 0 else 0.0
-
- # Hotspot: top 25% churn (churn_percentile >= 0.75) AND the absolute
- # activity floors — without the floors, a quiet repo's top quartile is
- # just "every file touched in 90 days" (issue #361).
- for meta in metadata_list:
- churn_pct = meta.get("churn_percentile", 0.0)
+ meta = metadata_list[idx]
+ churn_pct = rank / total if total > 0 else 0.0
+ meta["churn_percentile"] = churn_pct
+ # Hotspot: top 25% churn (churn_percentile >= 0.75) AND the absolute
+ # activity floors — without the floors, a quiet repo's top quartile is
+ # just "every file touched in 90 days" (issue #361). Folded into the
+ # rank loop so the list is scanned once.
if churn_pct >= 0.75 and meets_hotspot_floors(meta):
meta["is_hotspot"] = True
+ # change_entropy percentile default — overwritten below for files
+ # carrying a positive entropy signal.
+ meta.setdefault("change_entropy_pct", 0.0)
# change_entropy percentile (mirrors churn_percentile). Rank ONLY files
# that carry a positive entropy signal; files with zero entropy — every
@@ -224,8 +227,6 @@ def compute_percentiles(metadata_list: list[dict]) -> None:
# alone — keep pct 0.0 so the change_entropy biomarker stays silent. (A
# naive rank-everything would hand the topmost zero-entropy file a high
# percentile when most files are zero.)
- for meta in metadata_list:
- meta.setdefault("change_entropy_pct", 0.0)
entropy_idxs = [
i for i in range(total) if (metadata_list[i].get("change_entropy") or 0.0) > 0.0
]
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py b/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py
index 9f50dd7f..3fe4c8ca 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py
@@ -110,6 +110,12 @@ def new_meta(file_path: str) -> dict[str, Any]:
# ``prior_defect`` health biomarker; mirrors the benchmark's
# prior-defects baseline definition (product == benchmark).
"prior_defect_count": 0,
+ # Agent provenance rollup: how much of this file's indexed history is
+ # agent-attributed (local channels only — see agent_provenance module).
+ # agent_authored_pct stays None when the file has no commits at all.
+ "agent_commit_count": 0,
+ "agent_authored_pct": None,
+ "agent_tier_counts_json": "{}",
# Temporal hotspot score (exponentially decayed churn)
"temporal_hotspot_score": 0.0,
# Change entropy (Hassan HCM) — populated repo-wide by the co-change
@@ -126,6 +132,7 @@ def _parse_per_file_log(
*,
commit_limit: int,
follow_renames: bool,
+ provenance_classifier: Any | None = None,
) -> tuple[list[_CommitRec], str | None]:
"""Run a per-file ``git log --numstat`` and parse it into commit records.
@@ -176,6 +183,16 @@ def _parse_per_file_log(
subject=header["subject"],
body=header["body"],
)
+ if provenance_classifier is not None:
+ prov = provenance_classifier.classify(
+ header["author_name"],
+ header["author_email"],
+ header["committer_name"],
+ header["committer_email"],
+ f"{header['subject']}\n{header['body']}",
+ )
+ current.agent = prov.agent
+ current.agent_tier = prov.autonomy_tier
commits.append(current)
for line in numstat_lines:
numstat_parts = line.split("\t")
@@ -205,6 +222,7 @@ def index_file(
include_blame: bool = True,
precomputed_commits: list[_CommitRec] | None = None,
as_of_ts: float | None = None,
+ provenance_classifier: Any | None = None,
) -> dict:
"""Index a single file's git history. Runs in executor.
@@ -239,6 +257,7 @@ def index_file(
file_path,
commit_limit=commit_limit,
follow_renames=follow_renames,
+ provenance_classifier=provenance_classifier,
)
if not commits:
@@ -291,6 +310,18 @@ def index_file(
total_churn = meta["lines_added_90d"] + meta["lines_deleted_90d"]
meta["avg_commit_size"] = total_churn / c90 if c90 > 0 else 0.0
+ # Agent-provenance rollup: share of this file's indexed commits that
+ # are agent-attributed, plus the per-autonomy-tier breakdown. The
+ # per-commit labels were classified once during the walk; this is a
+ # pure count over the already-carried records.
+ agent_commits = [c for c in commits if getattr(c, "agent", None)]
+ meta["agent_commit_count"] = len(agent_commits)
+ meta["agent_authored_pct"] = len(agent_commits) / len(commits)
+ tier_counts: Counter[str] = Counter(
+ str(c.agent_tier) for c in agent_commits if c.agent_tier
+ )
+ meta["agent_tier_counts_json"] = json.dumps(dict(tier_counts))
+
# Temporal hotspot score: exponentially decayed per-commit churn.
_ln2 = math.log(2)
temporal_score = 0.0
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
index 0cd84ed1..58ed967f 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -12,6 +12,7 @@ import asyncio
import contextlib
import json
import os
+import threading
import time
from collections.abc import Callable
from pathlib import Path
@@ -120,6 +121,7 @@ class GitIndexer:
# ``git log -- <file>`` — turning O(files) process spawns into O(1).
commit_index: dict[str, list[_CommitRec]] = {}
commit_sink: list[dict] = []
+ prov_clf = self._provenance_classifier()
if not self.follow_renames:
from ..git_commit_index import load_commit_index
@@ -128,31 +130,35 @@ class GitIndexer:
self.commit_limit,
set(indexable_files),
commit_sink=commit_sink,
+ provenance_classifier=prov_clf,
)
include_blame = self.tier.includes_blame
as_of_ts = self._resolve_as_of_ts(repo, commit_index)
+ get_thread_repo, close_thread_repos = self._thread_repo_pool()
def _index_one_sync(file_path: str) -> dict:
- """Use a per-thread Repo to avoid shared-handle issues on Windows."""
- try:
- import git as gitpython
+ """Use a per-thread Repo to avoid shared-handle issues on Windows.
- thread_repo = gitpython.Repo(self.repo_path, search_parent_directories=True)
- try:
- precomputed = commit_index.get(file_path) if commit_index else None
- return index_file(
- thread_repo,
- file_path,
- repo_path=self.repo_path,
- commit_limit=self.commit_limit,
- follow_renames=self.follow_renames,
- include_blame=include_blame,
- precomputed_commits=precomputed,
- as_of_ts=as_of_ts,
- )
- finally:
- thread_repo.close()
+ The Repo is created once per worker thread and reused across files
+ — constructing a fresh ``gitpython.Repo`` per file costs ~40 ms
+ each (config + ref resolution), which dominated the git phase on
+ repos with thousands of files.
+ """
+ try:
+ thread_repo = get_thread_repo()
+ precomputed = commit_index.get(file_path) if commit_index else None
+ return index_file(
+ thread_repo,
+ file_path,
+ repo_path=self.repo_path,
+ commit_limit=self.commit_limit,
+ follow_renames=self.follow_renames,
+ include_blame=include_blame,
+ precomputed_commits=precomputed,
+ as_of_ts=as_of_ts,
+ provenance_classifier=prov_clf,
+ )
except Exception:
return {"file_path": file_path}
@@ -212,6 +218,7 @@ class GitIndexer:
# Abandon any timed-out threads immediately instead of letting
# asyncio.run() block for minutes during default-executor cleanup.
executor.shutdown(wait=False, cancel_futures=True)
+ close_thread_repos()
results: list[dict] = []
for r in metadata_list:
@@ -279,7 +286,14 @@ class GitIndexer:
return summary, results
async def index_changed_files(self, changed_file_paths: list[str]) -> list[dict]:
- """Incremental update: re-index only changed files."""
+ """Incremental update: re-index only changed files.
+
+ Mirrors ``index_repo``'s batched shape: one repo-wide commit-index
+ walk feeds every per-file worker (instead of one ``git log -- <file>``
+ subprocess per changed file), so the metadata an update produces is
+ identical to what a fresh full index would produce — including the
+ agent-provenance rollup, which the old per-file path never classified.
+ """
repo = self._get_repo()
if repo is None:
return []
@@ -287,25 +301,36 @@ class GitIndexer:
loop = asyncio.get_event_loop()
semaphore = asyncio.Semaphore(20)
include_blame = self.tier.includes_blame
- as_of_ts = self._resolve_as_of_ts(repo)
+
+ prov_clf = self._provenance_classifier()
+ commit_index: dict[str, list[_CommitRec]] = {}
+ if not self.follow_renames:
+ from ..git_commit_index import load_commit_index
+
+ commit_index = load_commit_index(
+ repo,
+ self.commit_limit,
+ set(changed_file_paths),
+ provenance_classifier=prov_clf,
+ )
+ as_of_ts = self._resolve_as_of_ts(repo, commit_index)
+ get_thread_repo, close_thread_repos = self._thread_repo_pool()
def _index_one_sync(file_path: str) -> dict:
try:
- import git as gitpython
-
- thread_repo = gitpython.Repo(self.repo_path, search_parent_directories=True)
- try:
- return index_file(
- thread_repo,
- file_path,
- repo_path=self.repo_path,
- commit_limit=self.commit_limit,
- follow_renames=self.follow_renames,
- include_blame=include_blame,
- as_of_ts=as_of_ts,
- )
- finally:
- thread_repo.close()
+ thread_repo = get_thread_repo()
+ precomputed = commit_index.get(file_path) if commit_index else None
+ return index_file(
+ thread_repo,
+ file_path,
+ repo_path=self.repo_path,
+ commit_limit=self.commit_limit,
+ follow_renames=self.follow_renames,
+ include_blame=include_blame,
+ precomputed_commits=precomputed,
+ as_of_ts=as_of_ts,
+ provenance_classifier=prov_clf,
+ )
except Exception:
return {"file_path": file_path}
@@ -325,7 +350,10 @@ class GitIndexer:
return {"file_path": file_path}
tasks = [index_one(fp) for fp in changed_file_paths]
- results_raw = await asyncio.gather(*tasks, return_exceptions=True)
+ try:
+ results_raw = await asyncio.gather(*tasks, return_exceptions=True)
+ finally:
+ close_thread_repos()
results: list[dict] = []
for r in results_raw:
@@ -373,7 +401,12 @@ class GitIndexer:
# Empty indexable set: we only want the full-footprint sink, not the
# per-file bucket (which the incremental file pass already rebuilt).
load_commit_index(
- repo, self.commit_limit, set(), commit_sink=sink, since_ts=since_ts
+ repo,
+ self.commit_limit,
+ set(),
+ commit_sink=sink,
+ since_ts=since_ts,
+ provenance_classifier=self._provenance_classifier(),
)
return build_commit_rows(sink)
except Exception as exc:
@@ -386,6 +419,22 @@ class GitIndexer:
# Internal methods
# ------------------------------------------------------------------
+ def _provenance_classifier(self) -> Any:
+ """Config-aware agent-provenance classifier, built once per index run.
+
+ Failure-isolated: a malformed repo config falls back to the built-in
+ pattern registry rather than breaking the git phase.
+ """
+ try:
+ from .agent_provenance import classifier_from_repo_config
+
+ return classifier_from_repo_config(self.repo_path)
+ except Exception as exc:
+ logger.debug("agent_provenance_classifier_failed", error=str(exc))
+ from .agent_provenance import AgentProvenanceClassifier
+
+ return AgentProvenanceClassifier()
+
def _resolve_as_of_ts(
self, repo: Any, commit_index: dict[str, list[_CommitRec]] | None = None
) -> float | None:
@@ -417,6 +466,42 @@ class GitIndexer:
except Exception:
return None
+ def _thread_repo_pool(self) -> tuple[Callable[[], Any], Callable[[], None]]:
+ """Return ``(get_thread_repo, close_all)`` for per-thread Repo reuse.
+
+ gitpython ``Repo`` handles can't be shared across threads on Windows,
+ but constructing one per *file* costs ~40 ms each. This pool hands
+ each worker thread its own lazily-created ``Repo`` and reuses it for
+ every file that thread processes. ``close_all()`` closes every repo
+ the pool created; a worker thread abandoned mid-file (timeout) will
+ see its repo closed underneath it and fall into the existing
+ per-file exception path, which is the same partial-data outcome the
+ old per-file construction produced on failure.
+ """
+ tls = threading.local()
+ created: list[Any] = []
+ lock = threading.Lock()
+
+ def get_thread_repo() -> Any:
+ repo = getattr(tls, "repo", None)
+ if repo is None:
+ import git as gitpython
+
+ repo = gitpython.Repo(self.repo_path, search_parent_directories=True)
+ tls.repo = repo
+ with lock:
+ created.append(repo)
+ return repo
+
+ def close_all() -> None:
+ with lock:
+ repos, created[:] = list(created), []
+ for repo in repos:
+ with contextlib.suppress(Exception):
+ repo.close()
+
+ return get_thread_repo, close_all
+
def _get_repo(self) -> Any | None:
try:
import git as gitpython
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/records.py b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
index 420d3fda..b6050f70 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/records.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
@@ -38,8 +38,10 @@ _FIELD_SEP = "\x1f"
# ``%an``/``%ae``) so a repo's ``.mailmap`` folds a contributor's multiple
# names/emails — personal vs. GitHub ``noreply``, a second machine's git config —
# into one identity. Without this, ownership and "last touched" split one human
-# across several contributor buckets.
-_LOG_FORMAT = "%x00%H%x1f%aN%x1f%aE%x1f%ct%x1f%P%x1f%s%x1f%b"
+# across several contributor buckets. Committer name/email (``%cN``/``%cE``)
+# ride the same record for agent-provenance classification: a service identity
+# as committer over a human author means an agent pushed/amended the change.
+_LOG_FORMAT = "%x00%H%x1f%aN%x1f%aE%x1f%cN%x1f%cE%x1f%ct%x1f%P%x1f%s%x1f%b"
# A git ``--numstat`` line: ``<added>\t<deleted>\t<path>`` where added/deleted
# are decimal counts or ``-`` for binary files. Used to split the trailing
@@ -60,6 +62,10 @@ class _CommitRec:
body: str = ""
added: int = 0
deleted: int = 0
+ # Agent provenance (classified once per commit during the walk; carried on
+ # the record so per-file rollups don't re-run the regex per touched file).
+ agent: str | None = None
+ agent_tier: int | None = None
def _parse_commit_record(record: str) -> tuple[dict, list[str]] | None:
@@ -67,7 +73,7 @@ def _parse_commit_record(record: str) -> tuple[dict, list[str]] | None:
A record (after the leading record separator is stripped) looks like::
- <sha>\\x1f<an>\\x1f<ae>\\x1f<ct>\\x1f<parents>\\x1f<subject>\\x1f<body>\\n<numstat...>
+ <sha>\\x1f<an>\\x1f<ae>\\x1f<cn>\\x1f<ce>\\x1f<ct>\\x1f<parents>\\x1f<subject>\\x1f<body>\\n<numstat...>
The body is multi-line, so the trailing ``--numstat`` block is identified
by the first line matching :data:`_NUMSTAT_RE`; everything before it is
@@ -76,12 +82,12 @@ def _parse_commit_record(record: str) -> tuple[dict, list[str]] | None:
fields except ``added``/``deleted`` (the caller attributes churn).
"""
parts = record.split(_FIELD_SEP)
- if len(parts) < 7:
+ if len(parts) < 9:
return None
- sha, an, ae, ct, parents, subject = parts[:6]
+ sha, an, ae, cn, ce, ct, parents, subject = parts[:8]
# %b cannot contain the field separator, so the body + numstat tail is the
- # 7th field exactly (join is defensive against an unexpected stray sep).
- body_and_numstat = _FIELD_SEP.join(parts[6:])
+ # 9th field exactly (join is defensive against an unexpected stray sep).
+ body_and_numstat = _FIELD_SEP.join(parts[8:])
body_lines: list[str] = []
numstat_lines: list[str] = []
@@ -103,6 +109,8 @@ def _parse_commit_record(record: str) -> tuple[dict, list[str]] | None:
"sha": sha,
"author_name": an or "unknown",
"author_email": ae,
+ "committer_name": cn or "",
+ "committer_email": ce or "",
"ts": ts,
"is_merge": len(parents.split()) > 1,
"subject": subject,
diff --git a/packages/core/src/repowise/core/ingestion/graph/_betweenness.py b/packages/core/src/repowise/core/ingestion/graph/_betweenness.py
new file mode 100644
index 00000000..e2585c9e
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_betweenness.py
@@ -0,0 +1,155 @@
+"""Parallel exact betweenness centrality (Brandes) over a process pool.
+
+``nx.betweenness_centrality`` is pure-Python Brandes: O(V·E) per run, single
+threaded, and it does not release the GIL — on a ~11k-node symbol subgraph it
+is by far the most expensive metric kernel (~54s isolated, measured on the
+repowise repo itself). Brandes is embarrassingly parallel over *source*
+nodes: each source's dependency accumulation is independent and the final
+score is the plain sum of the per-source partials. This module fans the
+source set out across a process pool and sums the partials, producing the
+same values as the sequential call (the only difference is floating-point
+summation order, bounded at ~1e-15 relative).
+
+Workers receive the graph once (pool initializer) as an integer edge list —
+node attributes are irrelevant to betweenness, so the pickled payload stays
+small even for graphs whose node ids are long symbol strings.
+
+Falls back to ``nx.betweenness_centrality`` whenever the graph is small
+enough that pool startup would dominate, or the pool / NetworkX internals
+are unavailable.
+"""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import ProcessPoolExecutor
+
+import networkx as nx
+import structlog
+
+log = structlog.get_logger(__name__)
+
+# Parallelize only when the estimated Brandes cost (~nodes x edges) is large
+# enough to amortize process startup (~2-3s for pool spawn + imports).
+# Measured rates on the repowise graph: file subgraph (1.8k x 5.2k ~ 1e7)
+# runs in ~0.4s sequentially; the symbol subgraph (11.3k x 12k ~ 1.4e8)
+# takes ~54s. The threshold sits between the two.
+_PARALLEL_COST_THRESHOLD = 4e7
+
+# ---------------------------------------------------------------------------
+# Worker side (must be module-level + picklable for Windows spawn)
+# ---------------------------------------------------------------------------
+
+_WORKER_GRAPH: nx.DiGraph | None = None
+
+
+def _init_worker(n: int, edges: list[tuple[int, int]], directed: bool) -> None:
+ """Pool initializer: build the (attribute-free) int-labeled graph once."""
+ global _WORKER_GRAPH
+ g: nx.DiGraph = nx.DiGraph() if directed else nx.Graph()
+ g.add_nodes_from(range(n))
+ g.add_edges_from(edges)
+ _WORKER_GRAPH = g
+
+
+def _partial_betweenness(sources: list[int]) -> dict[int, float]:
+ """Accumulate Brandes dependencies for a chunk of source nodes.
+
+ Mirrors the per-source loop of ``nx.betweenness_centrality`` exactly
+ (unweighted BFS + basic accumulation, no endpoints). Summing the
+ partials over all chunks reproduces the sequential pre-rescale values.
+ """
+ from networkx.algorithms.centrality.betweenness import (
+ _accumulate_basic,
+ _single_source_shortest_path_basic,
+ )
+
+ g = _WORKER_GRAPH
+ assert g is not None, "worker pool initializer did not run"
+ betweenness: dict[int, float] = dict.fromkeys(g, 0.0)
+ for s in sources:
+ S, P, sigma, _ = _single_source_shortest_path_basic(g, s) # noqa: N806 - NX naming
+ betweenness, _ = _accumulate_basic(betweenness, S, P, sigma, s)
+ # Only non-zero entries cross the process boundary.
+ return {v: b for v, b in betweenness.items() if b != 0.0}
+
+
+# ---------------------------------------------------------------------------
+# Parent side
+# ---------------------------------------------------------------------------
+
+
+def _rescale(
+ betweenness: dict[int, float], n: int, *, normalized: bool, directed: bool
+) -> dict[int, float]:
+ """Rescale raw accumulations exactly like NetworkX (k=None, no endpoints)."""
+ if normalized:
+ scale = None if n <= 2 else 1 / ((n - 1) * (n - 2))
+ else:
+ scale = None if directed else 0.5
+ if scale is not None:
+ for v in betweenness:
+ betweenness[v] *= scale
+ return betweenness
+
+
+def betweenness_centrality_fast(
+ g: nx.DiGraph,
+ *,
+ normalized: bool = True,
+ max_workers: int | None = None,
+) -> dict[str, float]:
+ """Exact betweenness centrality, parallelized over sources when worthwhile.
+
+ Drop-in equivalent of ``nx.betweenness_centrality(g, normalized=...)``
+ for unweighted graphs without endpoint counting. Small graphs (or any
+ pool failure) take the sequential NetworkX path, so callers never need
+ a fallback of their own.
+ """
+ n = g.number_of_nodes()
+ e = g.number_of_edges()
+ if n == 0:
+ return {}
+ workers = max_workers or os.cpu_count() or 1
+ if n * e < _PARALLEL_COST_THRESHOLD or workers < 2:
+ return nx.betweenness_centrality(g, normalized=normalized)
+
+ try:
+ # Verify the NetworkX internals the workers rely on exist in this
+ # version before paying for pool spawn.
+ from networkx.algorithms.centrality.betweenness import ( # noqa: F401
+ _accumulate_basic,
+ _single_source_shortest_path_basic,
+ )
+ except ImportError: # pragma: no cover - depends on networkx version
+ log.warning("betweenness_parallel_unavailable", reason="networkx internals moved")
+ return nx.betweenness_centrality(g, normalized=normalized)
+
+ nodes = list(g)
+ index = {node: i for i, node in enumerate(nodes)}
+ edges = [(index[u], index[v]) for u, v in g.edges()]
+
+ # ~3 chunks per worker for load balancing without excessive IPC.
+ chunk_count = max(1, workers * 3)
+ chunk_size = max(1, (n + chunk_count - 1) // chunk_count)
+ chunks = [list(range(i, min(i + chunk_size, n))) for i in range(0, n, chunk_size)]
+
+ totals = [0.0] * n
+ try:
+ with ProcessPoolExecutor(
+ max_workers=workers,
+ initializer=_init_worker,
+ initargs=(n, edges, g.is_directed()),
+ ) as pool:
+ # ``map`` preserves chunk order, keeping float summation
+ # deterministic for a given worker count.
+ for partial in pool.map(_partial_betweenness, chunks):
+ for v, b in partial.items():
+ totals[v] += b
+ except Exception as exc: # pragma: no cover - depends on host environment
+ log.warning("betweenness_parallel_failed_falling_back", error=str(exc))
+ return nx.betweenness_centrality(g, normalized=normalized)
+
+ raw = dict(enumerate(totals))
+ _rescale(raw, n, normalized=normalized, directed=g.is_directed())
+ return {nodes[i]: score for i, score in raw.items()}
diff --git a/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py b/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
new file mode 100644
index 00000000..4d64ce54
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
@@ -0,0 +1,113 @@
+"""Structure-keyed disk cache for betweenness centrality.
+
+Exact betweenness (Brandes) is the most expensive metric kernel on every
+ingest — and an incremental ``repowise update`` recomputes it for the whole
+graph even when the docstring-only edit it processed didn't change a single
+edge. Betweenness is a pure function of the *structure* of the (unweighted)
+subgraph it runs on, so values cache safely keyed by a signature over the
+sorted node and edge sets: content edits that don't move call/heritage/import
+edges hit; any structural change misses and recomputes.
+
+One entry per kind (``file`` / ``symbol``) — the cache answers "is the graph
+still the one I last scored?", not a history. Thread-safe because the init
+pipeline computes both kinds concurrently. Best-effort by design: any
+load/save error degrades to a fresh computation.
+
+Note for the >30k-node sampled path (``nx.betweenness_centrality(k=...)``):
+sampling is seedless, so recomputing the same graph yields slightly
+different values run to run. A cache hit returns the previously sampled
+values — deterministic where the status quo was not.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import os
+import pickle
+import tempfile
+import threading
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+_CACHE_VERSION = 1
+_CACHE_FILENAME = "centrality_cache.pkl"
+
+__all__ = ["CentralityCache", "subgraph_signature"]
+
+
+def subgraph_signature(graph) -> str:
+ """Hash the structure (sorted nodes + edges) of *graph*.
+
+ Attributes are deliberately excluded — betweenness on these subgraphs is
+ unweighted, and the subgraph constructors already filtered edges by type.
+ """
+ h = hashlib.sha256()
+ for node in sorted(graph.nodes()):
+ h.update(node.encode("utf-8", "replace"))
+ h.update(b"\x00")
+ h.update(b"\x01")
+ for u, v in sorted(graph.edges()):
+ h.update(u.encode("utf-8", "replace"))
+ h.update(b"\x00")
+ h.update(v.encode("utf-8", "replace"))
+ h.update(b"\x00")
+ return h.hexdigest()
+
+
+class CentralityCache:
+ """Pickle-backed ``kind -> (signature, values)`` store."""
+
+ def __init__(self, cache_dir: Path | str) -> None:
+ self._path = Path(cache_dir) / _CACHE_FILENAME
+ self._entries: dict[str, tuple[str, dict[str, float]]] = {}
+ self._lock = threading.Lock()
+ self._loaded = False
+
+ def _ensure_loaded(self) -> None:
+ if self._loaded:
+ return
+ self._loaded = True
+ try:
+ with self._path.open("rb") as fh:
+ payload = pickle.load(fh)
+ if payload.get("version") != _CACHE_VERSION:
+ return
+ self._entries = payload.get("entries", {})
+ except FileNotFoundError:
+ return
+ except Exception as exc: # corrupt / unreadable cache -> recompute
+ log.debug("centrality_cache_load_failed", error=str(exc))
+
+ def get(self, kind: str, signature: str) -> dict[str, float] | None:
+ with self._lock:
+ self._ensure_loaded()
+ entry = self._entries.get(kind)
+ if entry is None or entry[0] != signature:
+ return None
+ return dict(entry[1])
+
+ def put(self, kind: str, signature: str, values: dict[str, float]) -> None:
+ """Store *values* for *kind* and persist atomically."""
+ with self._lock:
+ self._ensure_loaded()
+ self._entries[kind] = (signature, dict(values))
+ try:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ payload = {"version": _CACHE_VERSION, "entries": self._entries}
+ fd, tmp_name = tempfile.mkstemp(
+ dir=str(self._path.parent), prefix=_CACHE_FILENAME, suffix=".tmp"
+ )
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ pickle.dump(payload, fh, protocol=pickle.HIGHEST_PROTOCOL)
+ os.replace(tmp_name, self._path)
+ except BaseException:
+ with contextlib.suppress(OSError):
+ os.unlink(tmp_name)
+ raise
+ except Exception as exc:
+ log.debug("centrality_cache_save_failed", error=str(exc))
diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.py
index 73789287..06748cd3 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_edges.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py
@@ -62,6 +62,8 @@ class EdgesMixin:
count += 1
log.info("Co-change edges added", count=count)
+ if count:
+ self._invalidate_subgraph_caches()
return count
def update_co_change_edges(self, updated_meta: dict, min_count: int = 3) -> None:
@@ -72,6 +74,7 @@ class EdgesMixin:
edges_to_remove.append((u, v))
self._graph.remove_edges_from(edges_to_remove)
self.add_co_change_edges(updated_meta, min_count)
+ self._invalidate_subgraph_caches()
def add_dynamic_edges(self, edges: list) -> None:
"""Add dynamic-hint edges to the graph. Each edge is a DynamicEdge."""
@@ -97,6 +100,8 @@ class EdgesMixin:
# source file node as is_test so downstream consumers can filter it.
if e.hint_source and e.hint_source.endswith(":test"):
self._graph.nodes[e.source]["is_test"] = True
+ if edges:
+ self._invalidate_subgraph_caches()
def add_framework_edges(self, tech_stack: list[str] | None = None) -> int:
"""Add synthetic edges for framework-mediated relationships.
@@ -124,4 +129,5 @@ class EdgesMixin:
count = add_framework_edges(self._graph, self._parsed_files, ctx, tech_stack)
if count:
log.info("Framework edges added", count=count)
+ self._invalidate_subgraph_caches()
return count
diff --git a/packages/core/src/repowise/core/ingestion/graph/_metrics.py b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
index 052a8de7..377c52f1 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_metrics.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
@@ -76,17 +76,35 @@ class MetricsMixin:
# ------------------------------------------------------------------
def file_subgraph(self) -> nx.DiGraph:
- """Return a subgraph containing only file-level nodes and import edges."""
- g = self.graph()
- file_nodes = [
- n for n, d in g.nodes(data=True) if d.get("node_type", "file") in ("file", "external")
- ]
- sub = g.subgraph(file_nodes).copy()
- edges_to_remove = [
- (u, v) for u, v, d in sub.edges(data=True) if d.get("edge_type") in ("co_changes",)
- ]
- sub.remove_edges_from(edges_to_remove)
- return sub
+ """Return a subgraph containing only file-level nodes and import edges.
+
+ Cached per build — five metric kernels (SCC, PageRank, betweenness,
+ in/out degree) read it, and rebuilding the filtered copy per call is
+ O(V+E) each time. The cache is guarded by ``_subgraph_lock`` because
+ the init pipeline computes metrics concurrently via
+ ``asyncio.to_thread``. Callers must treat the result as read-only.
+ """
+ cached = self._file_subgraph_cache
+ if cached is not None:
+ return cached
+ with self._subgraph_lock:
+ if self._file_subgraph_cache is not None:
+ return self._file_subgraph_cache
+ g = self.graph()
+ file_nodes = [
+ n
+ for n, d in g.nodes(data=True)
+ if d.get("node_type", "file") in ("file", "external")
+ ]
+ sub = g.subgraph(file_nodes).copy()
+ edges_to_remove = [
+ (u, v)
+ for u, v, d in sub.edges(data=True)
+ if d.get("edge_type") in ("co_changes",)
+ ]
+ sub.remove_edges_from(edges_to_remove)
+ self._file_subgraph_cache = sub
+ return sub
def symbol_subgraph(self) -> nx.DiGraph:
"""Return a subgraph of symbol nodes connected by call + heritage edges.
@@ -94,17 +112,27 @@ class MetricsMixin:
File-to-symbol ``defines`` edges and class-to-method ``has_method``
ownership edges are dropped so that the resulting centrality
scores reflect call/heritage flow rather than containment.
+
+ Cached per build (see :meth:`file_subgraph` for the locking
+ rationale). Callers must treat the result as read-only.
"""
- g = self.graph()
- symbol_nodes = [n for n, d in g.nodes(data=True) if d.get("node_type") == "symbol"]
- sub = g.subgraph(symbol_nodes).copy()
- edges_to_remove = [
- (u, v)
- for u, v, d in sub.edges(data=True)
- if d.get("edge_type") not in ("calls", "extends", "implements")
- ]
- sub.remove_edges_from(edges_to_remove)
- return sub
+ cached = self._symbol_subgraph_cache
+ if cached is not None:
+ return cached
+ with self._subgraph_lock:
+ if self._symbol_subgraph_cache is not None:
+ return self._symbol_subgraph_cache
+ g = self.graph()
+ symbol_nodes = [n for n, d in g.nodes(data=True) if d.get("node_type") == "symbol"]
+ sub = g.subgraph(symbol_nodes).copy()
+ edges_to_remove = [
+ (u, v)
+ for u, v, d in sub.edges(data=True)
+ if d.get("edge_type") not in ("calls", "extends", "implements")
+ ]
+ sub.remove_edges_from(edges_to_remove)
+ self._symbol_subgraph_cache = sub
+ return sub
# ------------------------------------------------------------------
# File-level metrics
@@ -136,15 +164,10 @@ class MetricsMixin:
if self._betweenness_cache is not None:
return self._betweenness_cache
g = self.file_subgraph()
- n = g.number_of_nodes()
- if n == 0:
+ if g.number_of_nodes() == 0:
self._betweenness_cache = {}
return self._betweenness_cache
- if n > _LARGE_REPO_THRESHOLD:
- k = min(500, n)
- self._betweenness_cache = nx.betweenness_centrality(g, k=k, normalized=True)
- else:
- self._betweenness_cache = nx.betweenness_centrality(g, normalized=True)
+ self._betweenness_cache = self._betweenness_with_disk_cache("file", g)
return self._betweenness_cache
def in_degree(self) -> dict[str, int]:
@@ -239,16 +262,51 @@ class MetricsMixin:
if self._symbol_betweenness_cache is not None:
return self._symbol_betweenness_cache
sub = self.symbol_subgraph()
- n = sub.number_of_nodes()
- if n == 0:
+ if sub.number_of_nodes() == 0:
self._symbol_betweenness_cache = {}
return self._symbol_betweenness_cache
+ self._symbol_betweenness_cache = self._betweenness_with_disk_cache("symbol", sub)
+ return self._symbol_betweenness_cache
+
+ def _betweenness_with_disk_cache(self, kind: str, g: nx.DiGraph) -> dict[str, float]:
+ """Compute betweenness for *g*, consulting the structure-keyed disk cache.
+
+ With a cache attached (see ``GraphBuilder(centrality_cache_dir=...)``)
+ and an unchanged subgraph structure, the previous run's values are
+ returned without re-running Brandes — the dominant metric cost of an
+ incremental update whose change didn't move any edges. Structural
+ changes, cache errors, or no cache all fall through to the exact
+ computation used before.
+ """
+ cache = getattr(self, "_centrality_cache", None)
+ signature: str | None = None
+ if cache is not None:
+ try:
+ from ._centrality_cache import subgraph_signature
+
+ signature = subgraph_signature(g)
+ hit = cache.get(kind, signature)
+ if hit is not None:
+ log.info("betweenness_reused_from_cache", kind=kind, nodes=len(hit))
+ return hit
+ except Exception as exc:
+ log.debug("centrality_cache_lookup_failed", kind=kind, error=str(exc))
+ signature = None
+
+ n = g.number_of_nodes()
if n > _LARGE_REPO_THRESHOLD:
k = min(500, n)
- self._symbol_betweenness_cache = nx.betweenness_centrality(sub, k=k, normalized=True)
+ values = nx.betweenness_centrality(g, k=k, normalized=True)
else:
- self._symbol_betweenness_cache = nx.betweenness_centrality(sub, normalized=True)
- return self._symbol_betweenness_cache
+ from ._betweenness import betweenness_centrality_fast
+
+ values = betweenness_centrality_fast(g, normalized=True)
+ if cache is not None and signature is not None:
+ try:
+ cache.put(kind, signature, values)
+ except Exception as exc:
+ log.debug("centrality_cache_store_failed", kind=kind, error=str(exc))
+ return values
# ------------------------------------------------------------------
# Execution flows + bulk priming
diff --git a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
index b0580416..62c94ae5 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
@@ -16,6 +16,16 @@ log = structlog.get_logger(__name__)
class ResolveMixin:
"""Symbol-level edge resolution passes run during ``build()``."""
+ def _shared_import_maps(self) -> Any:
+ """Build the import-name maps once per build; both resolvers share them."""
+ maps = getattr(self, "_import_name_maps", None)
+ if maps is None:
+ from ..import_index import build_import_name_maps
+
+ maps = build_import_name_maps(self._parsed_files)
+ self._import_name_maps = maps
+ return maps
+
def _resolve_heritage(
self,
import_targets: dict[str, set[str]],
@@ -24,7 +34,9 @@ class ResolveMixin:
"""Resolve heritage relations and add EXTENDS/IMPLEMENTS edges."""
from ..heritage_resolver import HeritageResolver
- resolver = HeritageResolver(self._parsed_files, import_targets)
+ resolver = HeritageResolver(
+ self._parsed_files, import_targets, import_maps=self._shared_import_maps()
+ )
total_resolved = 0
files_with_heritage = [
@@ -135,7 +147,12 @@ class ResolveMixin:
"""Run three-tier call resolution and add CALLS edges to the graph."""
from ..call_resolver import CallResolver
- resolver = CallResolver(self._parsed_files, import_targets, repo_path=str(self._repo_path) if self._repo_path else None)
+ resolver = CallResolver(
+ self._parsed_files,
+ import_targets,
+ repo_path=str(self._repo_path) if self._repo_path else None,
+ import_maps=self._shared_import_maps(),
+ )
total_resolved = 0
files_with_calls = [
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
index 9f6223d4..862b5ef0 100644
--- a/packages/core/src/repowise/core/ingestion/graph/builder.py
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -45,12 +45,24 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
repo_path: Path | str | None = None,
*,
exclude_patterns: list[str] | None = None,
+ centrality_cache_dir: Path | str | None = None,
) -> None:
self._graph: nx.DiGraph = nx.DiGraph()
self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile
self._built = False
self._repo_path: Path | None = Path(repo_path) if repo_path else None
self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)
+ # Optional structure-keyed disk cache for betweenness (the most
+ # expensive metric kernel). Opt-in via *centrality_cache_dir* — the
+ # ingest paths pass ``<repo>/.repowise``; ad-hoc builders are unchanged.
+ self._centrality_cache: Any | None = None
+ if centrality_cache_dir is not None:
+ try:
+ from ._centrality_cache import CentralityCache
+
+ self._centrality_cache = CentralityCache(centrality_cache_dir)
+ except Exception:
+ self._centrality_cache = None
import pathspec
@@ -68,6 +80,16 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._symbol_pagerank_cache: dict[str, float] | None = None
self._symbol_betweenness_cache: dict[str, float] | None = None
self._execution_flow_cache: Any | None = None
+ # Filtered-subgraph caches — rebuilt lazily; guarded by a lock because
+ # the init pipeline computes metrics concurrently (asyncio.to_thread).
+ import threading
+
+ self._subgraph_lock = threading.Lock()
+ self._file_subgraph_cache: nx.DiGraph | None = None
+ self._symbol_subgraph_cache: nx.DiGraph | None = None
+ # Shared import-name maps (built once per build(), injected into the
+ # call + heritage resolvers; reset whenever files change).
+ self._import_name_maps: Any | None = None
def set_tsconfig_resolver(self, resolver: Any) -> None:
"""Attach a :class:`TsconfigResolver` for TS/JS path-alias resolution."""
@@ -86,6 +108,20 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._symbol_pagerank_cache = None
self._symbol_betweenness_cache = None
self._execution_flow_cache = None
+ self._file_subgraph_cache = None
+ self._symbol_subgraph_cache = None
+ self._import_name_maps = None
+
+ def _invalidate_subgraph_caches(self) -> None:
+ """Clear only the filtered-subgraph caches.
+
+ Called by graph mutations that historically did NOT clear the metric
+ caches (co-change edge refresh, framework edges) — those keep their
+ existing semantics, but a cached subgraph must never go stale
+ structurally.
+ """
+ self._file_subgraph_cache = None
+ self._symbol_subgraph_cache = None
def release_graph(self) -> None:
"""Drop the in-memory NetworkX object after metrics are materialized.
@@ -102,6 +138,7 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
"""
self._graph = nx.DiGraph()
self._built = True
+ self._invalidate_subgraph_caches() # they hold the old structure — free it
# ------------------------------------------------------------------
# Building
diff --git a/packages/core/src/repowise/core/ingestion/heritage_resolver.py b/packages/core/src/repowise/core/ingestion/heritage_resolver.py
index 6b08f17d..62b7bbde 100644
--- a/packages/core/src/repowise/core/ingestion/heritage_resolver.py
+++ b/packages/core/src/repowise/core/ingestion/heritage_resolver.py
@@ -23,6 +23,7 @@ from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
+from typing import Any
import structlog
@@ -75,6 +76,8 @@ class HeritageResolver:
self,
parsed_files: dict[str, ParsedFile],
import_targets: dict[str, set[str]],
+ *,
+ import_maps: Any | None = None,
) -> None:
# Per-file class/interface/trait index: {file: {name: symbol_id}}
self._file_types: dict[str, dict[str, str]] = {}
@@ -85,8 +88,19 @@ class HeritageResolver:
# Import graph
self._import_targets = import_targets
- # Import name mapping (reuses resolved_file from Import objects)
- self._import_names: dict[str, dict[str, str]] = defaultdict(dict)
+ # Import name mapping — shared with CallResolver (built once per
+ # GraphBuilder.build() and injected; standalone construction builds
+ # it locally).
+ if import_maps is None:
+ from .import_index import build_import_name_maps
+
+ import_maps = build_import_name_maps(parsed_files)
+ self._import_names: dict[str, dict[str, str]] = import_maps.import_names
+
+ # Lazy per-file merged view of imported files' type tables —
+ # Tier-2b lookup; sorted-path merge order, first-wins (see
+ # CallResolver for rationale).
+ self._merged_import_types: dict[str, dict[str, str]] = {}
# Keep reference for cross-language checks in Tier 3
self._parsed_files = parsed_files
@@ -105,18 +119,18 @@ class HeritageResolver:
self._file_types[path] = file_types
- # Build import name mapping using resolved_file
- for imp in parsed.imports:
- if imp.resolved_file is None:
+ def _merged_types_for(self, file_path: str) -> dict[str, str]:
+ """Merged ``{type_name → symbol_id}`` across every imported file."""
+ merged = self._merged_import_types.get(file_path)
+ if merged is None:
+ merged = {}
+ for imported_file in sorted(self._import_targets.get(file_path, ())):
+ if imported_file.startswith("external:"):
continue
- for binding in imp.bindings:
- if binding.local_name != "*":
- self._import_names[path][binding.local_name] = imp.resolved_file
- # Fallback for imports without bindings
- if not imp.bindings:
- for name in imp.imported_names:
- if name != "*":
- self._import_names[path][name] = imp.resolved_file
+ for name, sym_id in self._file_types.get(imported_file, {}).items():
+ merged.setdefault(name, sym_id)
+ self._merged_import_types[file_path] = merged
+ return merged
def resolve_file(
self, file_path: str, relations: list[HeritageRelation]
@@ -162,19 +176,16 @@ class HeritageResolver:
line=rel.line,
)
- # Tier 2b: scan all imported files
- for imported_file in self._import_targets.get(file_path, set()):
- if imported_file.startswith("external:"):
- continue
- imported_types = self._file_types.get(imported_file, {})
- if parent_name in imported_types:
- return ResolvedHeritage(
- child_id=child_id,
- parent_id=imported_types[parent_name],
- edge_type=edge_type,
- confidence=0.85,
- line=rel.line,
- )
+ # Tier 2b: all imported files (pre-merged lookup)
+ merged_types = self._merged_types_for(file_path)
+ if parent_name in merged_types:
+ return ResolvedHeritage(
+ child_id=child_id,
+ parent_id=merged_types[parent_name],
+ edge_type=edge_type,
+ confidence=0.85,
+ line=rel.line,
+ )
# Tier 3: global unique match — only within the same language
candidates = self._global_types.get(parent_name, [])
diff --git a/packages/core/src/repowise/core/ingestion/import_index.py b/packages/core/src/repowise/core/ingestion/import_index.py
new file mode 100644
index 00000000..89794c69
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/import_index.py
@@ -0,0 +1,65 @@
+"""Shared import-name index construction for the call and heritage resolvers.
+
+Both resolvers used to derive the same ``{file → {local_name → source_file}}``
+mapping independently from every file's imports. Build it once here and
+inject it into both (the GraphBuilder does this during ``build()``); the
+resolvers keep a backwards-compatible fallback that builds the maps
+themselves when none are supplied.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+from .models import NamedBinding, ParsedFile
+
+
+@dataclass(slots=True)
+class ImportNameMaps:
+ """Per-file import-name lookups shared across resolution passes.
+
+ ``import_names`` — {file: {local_name: source_file}}
+ ``import_bindings`` — {file: {local_name: NamedBinding}} (call resolver only)
+ ``module_aliases`` — {file: {alias: source_file}} (call resolver only)
+ """
+
+ import_names: dict[str, dict[str, str]] = field(
+ default_factory=lambda: defaultdict(dict)
+ )
+ import_bindings: dict[str, dict[str, NamedBinding]] = field(
+ default_factory=lambda: defaultdict(dict)
+ )
+ module_aliases: dict[str, dict[str, str]] = field(
+ default_factory=lambda: defaultdict(dict)
+ )
+
+
+def build_import_name_maps(parsed_files: dict[str, ParsedFile]) -> ImportNameMaps:
+ """Build the shared import-name maps from every file's resolved imports.
+
+ Mirrors the historical ``CallResolver._build_indices`` import loop,
+ including the ``binding.source_file`` back-fill side effect that
+ downstream barrel-origin chasing relies on.
+ """
+ maps = ImportNameMaps()
+ for path, parsed in parsed_files.items():
+ for imp in parsed.imports:
+ if imp.resolved_file is None:
+ continue
+ resolved = imp.resolved_file
+ if imp.bindings:
+ for binding in imp.bindings:
+ if binding.local_name == "*":
+ continue
+ binding.source_file = resolved
+ maps.import_names[path][binding.local_name] = resolved
+ maps.import_bindings[path][binding.local_name] = binding
+ if binding.is_module_alias:
+ maps.module_aliases[path][binding.local_name] = resolved
+ else:
+ # Fallback for imports without binding data
+ for name in imp.imported_names:
+ if name != "*":
+ maps.import_names[path][name] = resolved
+ return maps
diff --git a/packages/core/src/repowise/core/ingestion/parse_cache.py b/packages/core/src/repowise/core/ingestion/parse_cache.py
new file mode 100644
index 00000000..aec7d78f
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/parse_cache.py
@@ -0,0 +1,160 @@
+"""Content-hash keyed cache for ParsedFile results.
+
+Parsing is the repo-proportional term of every ingest: ``repowise update``
+re-parses the whole repo when one file changed, and large-repo inits pay
+minutes of tree-sitter time. A ParsedFile is a pure function of the file
+*bytes* and the parser itself, so it caches safely keyed by
+``(relative path, content hash)`` under a parser fingerprint (the compiled
+``.scm`` query sources + the package version) that invalidates the whole
+cache when extraction rules change.
+
+Entries are stored as pickle *bytes* snapshotted at parse time — downstream
+graph builds mutate ParsedFile in place (``Import.resolved_file``,
+``NamedBinding.source_file`` are back-filled during resolution), so a hit
+must deserialize a fresh object graph rather than share one. The path is
+part of the key because symbol IDs embed it; a rename is a miss and
+re-parses (producing IDs for the new path). On a hit the *current*
+traversal ``FileInfo`` is rebound so git hash / mtime stay fresh.
+
+Best-effort by design: any load/save/deserialize error degrades to a full
+parse. The cache file is rewritten with only the entries touched this run,
+so deleted files age out naturally.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import os
+import pickle
+import tempfile
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from .models import FileInfo, ParsedFile
+
+log = structlog.get_logger(__name__)
+
+_CACHE_VERSION = 1
+_CACHE_FILENAME = "parse_cache.pkl"
+
+__all__ = ["ParseCache", "parser_fingerprint"]
+
+
+@lru_cache(maxsize=1)
+def parser_fingerprint() -> str:
+ """Fingerprint of everything that determines a ParsedFile besides bytes.
+
+ Hashes every ``.scm`` query source (extraction rules) plus the package
+ version (parser/extractor code changes ship as releases). A mismatch
+ invalidates the whole cache — correctness over reuse.
+ """
+ h = hashlib.sha256()
+ h.update(f"cache-version:{_CACHE_VERSION}".encode())
+ try:
+ from repowise.core import __version__
+
+ h.update(f"version:{__version__}".encode())
+ except Exception:
+ h.update(b"version:unknown")
+ queries_dir = Path(__file__).parent / "queries"
+ try:
+ for scm in sorted(queries_dir.glob("*.scm")):
+ h.update(scm.name.encode())
+ h.update(scm.read_bytes())
+ except Exception:
+ h.update(b"queries:unreadable")
+ return h.hexdigest()
+
+
+class ParseCache:
+ """Pickle-backed ``(path, content_hash) -> ParsedFile-bytes`` store."""
+
+ def __init__(self, cache_dir: Path | str) -> None:
+ self._path = Path(cache_dir) / _CACHE_FILENAME
+ self._fingerprint = parser_fingerprint()
+ self._entries: dict[tuple[str, str], bytes] = {}
+ self._fresh: dict[tuple[str, str], bytes] = {}
+ self.hits = 0
+ self.misses = 0
+
+ # -- lifecycle ---------------------------------------------------------
+
+ def load(self) -> None:
+ try:
+ with self._path.open("rb") as fh:
+ payload = pickle.load(fh)
+ if (
+ payload.get("version") != _CACHE_VERSION
+ or payload.get("fingerprint") != self._fingerprint
+ ):
+ return
+ self._entries = payload.get("files", {})
+ except FileNotFoundError:
+ return
+ except Exception as exc: # corrupt / unreadable cache -> full parse
+ log.debug("parse_cache_load_failed", error=str(exc))
+
+ def save(self) -> None:
+ """Atomically persist the entries used or created this run."""
+ try:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ payload = {
+ "version": _CACHE_VERSION,
+ "fingerprint": self._fingerprint,
+ "files": self._fresh,
+ }
+ fd, tmp_name = tempfile.mkstemp(
+ dir=str(self._path.parent), prefix=_CACHE_FILENAME, suffix=".tmp"
+ )
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ pickle.dump(payload, fh, protocol=pickle.HIGHEST_PROTOCOL)
+ os.replace(tmp_name, self._path)
+ except BaseException:
+ with contextlib.suppress(OSError):
+ os.unlink(tmp_name)
+ raise
+ except Exception as exc:
+ log.debug("parse_cache_save_failed", error=str(exc))
+
+ # -- access ------------------------------------------------------------
+
+ def get(self, file_info: FileInfo, content_hash: str) -> ParsedFile | None:
+ """Return a fresh ParsedFile for *content_hash* or None on miss.
+
+ Hits deserialize a new object graph (downstream resolution mutates
+ ParsedFile in place) and rebind the current *file_info* so traversal
+ metadata (git hash, mtime) stays current. *content_hash* is
+ :func:`compute_content_hash` of the raw bytes — callers compute it
+ once and reuse it for the matching :meth:`put`.
+ """
+ key = (file_info.path, content_hash)
+ blob = self._entries.get(key)
+ if blob is None:
+ self.misses += 1
+ return None
+ try:
+ parsed: Any = pickle.loads(blob)
+ except Exception as exc: # corrupt entry -> re-parse this file
+ log.debug("parse_cache_entry_corrupt", path=file_info.path, error=str(exc))
+ self.misses += 1
+ return None
+ self.hits += 1
+ self._fresh[key] = blob
+ parsed.file_info = file_info
+ return parsed
+
+ def put(self, parsed: ParsedFile, content_hash: str) -> None:
+ """Snapshot *parsed* (call before any graph build mutates it)."""
+ try:
+ blob = pickle.dumps(parsed, protocol=pickle.HIGHEST_PROTOCOL)
+ except Exception as exc:
+ log.debug("parse_cache_put_failed", path=parsed.file_info.path, error=str(exc))
+ return
+ key = (parsed.file_info.path, content_hash)
+ self._entries[key] = blob
+ self._fresh[key] = blob
diff --git a/packages/core/src/repowise/core/ingestion/parser.py b/packages/core/src/repowise/core/ingestion/parser.py
index c49f403f..d827a4a8 100644
--- a/packages/core/src/repowise/core/ingestion/parser.py
+++ b/packages/core/src/repowise/core/ingestion/parser.py
@@ -257,7 +257,13 @@ class ASTParser:
parse_errors = _collect_error_nodes(root)
query = self._get_query(lang, language, grammar_tag)
- symbols = self._extract_symbols(tree, query, config, file_info, src)
+ # Execute the compiled query ONCE per file. The five extraction
+ # passes below all consume the same capture dicts read-only;
+ # re-running ``cursor.matches()`` per pass multiplied the most
+ # expensive part of parsing by five.
+ matches = _run_query(query, root) if query is not None else []
+
+ symbols = self._extract_symbols(matches, config, file_info, src)
# Per-language synthetic-symbol pass — recognises source-generator
# attributes (e.g. CommunityToolkit.Mvvm) and adds the symbols the
# generator would emit at compile time. No-op for languages
@@ -266,12 +272,12 @@ class ASTParser:
if synthetic:
existing_ids = {s.id for s in symbols}
symbols.extend(s for s in synthetic if s.id not in existing_ids)
- imports = self._extract_imports(tree, query, config, file_info, src)
- calls = self._extract_calls(tree, query, config, file_info, src, symbols)
- heritage = extract_heritage(tree, query, config, file_info, src, run_query=_run_query)
+ imports = self._extract_imports(matches, config, file_info, src)
+ calls = self._extract_calls(matches, config, file_info, src, symbols)
+ heritage = extract_heritage(matches, config, file_info, src)
exports = self._derive_exports(symbols, config, src)
docstring = extract_module_docstring(root, src, lang)
- type_refs = self._extract_type_refs(tree, query, src, lang)
+ type_refs = self._extract_type_refs(matches, src, lang)
if len(symbols) > _SYMBOL_COUNT_WARN_THRESHOLD:
log.warning(
@@ -310,19 +316,15 @@ class ASTParser:
def _extract_symbols(
self,
- tree: object,
- query: object,
+ matches: list[dict],
config: LanguageConfig,
file_info: FileInfo,
src: str,
) -> list[Symbol]:
- if query is None:
- return []
-
symbols: list[Symbol] = []
seen: set[tuple[int, str]] = set() # (start_line, name) — dedup decorated dupes
- for capture_dict in _run_query(query, tree.root_node): # type: ignore[attr-defined]
+ for capture_dict in matches:
def_nodes = capture_dict.get("symbol.def", [])
name_nodes = capture_dict.get("symbol.name", [])
params_nodes = capture_dict.get("symbol.params", [])
@@ -502,19 +504,15 @@ class ASTParser:
def _extract_imports(
self,
- tree: object,
- query: object,
+ matches: list[dict],
config: LanguageConfig,
file_info: FileInfo,
src: str,
) -> list[Import]:
- if query is None:
- return []
-
imports: list[Import] = []
seen_raws: set[str] = set()
- for capture_dict in _run_query(query, tree.root_node): # type: ignore[attr-defined]
+ for capture_dict in matches:
stmt_nodes = capture_dict.get("import.statement", [])
module_nodes = capture_dict.get("import.module", [])
@@ -588,17 +586,13 @@ class ASTParser:
def _extract_calls(
self,
- tree: object,
- query: object,
+ matches: list[dict],
config: LanguageConfig,
file_info: FileInfo,
src: str,
symbols: list[Symbol],
) -> list[CallSite]:
"""Extract function/method call sites from the AST."""
- if query is None:
- return []
-
from .language_data import get_builtin_calls
_call_builtins = get_builtin_calls(file_info.language)
@@ -611,7 +605,7 @@ class ASTParser:
calls: list[CallSite] = []
seen: set[tuple[int, str, str | None]] = set()
- for capture_dict in _run_query(query, tree.root_node): # type: ignore[attr-defined]
+ for capture_dict in matches:
site_nodes = capture_dict.get("call.site", [])
target_nodes = capture_dict.get("call.target", [])
arg_nodes = capture_dict.get("call.arguments", [])
@@ -676,8 +670,7 @@ class ASTParser:
def _extract_type_refs(
self,
- tree: object,
- query: object,
+ matches: list[dict],
src: str,
lang: str = "",
) -> list[TypeReference]:
@@ -697,15 +690,12 @@ class ASTParser:
``field_declaration`` → ``field_type``, ``composite_literal`` →
``composite_literal`` (Go).
"""
- if query is None:
- return []
-
head_of = TYPE_HEAD_EXTRACTORS.get(lang, _head_type_identifier)
refs: list[TypeReference] = []
seen: set[tuple[str, int]] = set()
- for capture_dict in _run_query(query, tree.root_node): # type: ignore[attr-defined]
+ for capture_dict in matches:
type_nodes = capture_dict.get("param.type", [])
if not type_nodes:
continue
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py
index 594db4d2..657e1ed8 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py
@@ -25,6 +25,7 @@ flattened to the first plausible source target. Packages without an
from __future__ import annotations
import json
+import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -33,6 +34,81 @@ if TYPE_CHECKING:
from .context import ResolverContext
+# ---------------------------------------------------------------------------
+# Single pruned filesystem scan
+# ---------------------------------------------------------------------------
+# The mdx / vitest-config / package.json finders below all need to locate
+# files by name across the repo. Doing that with per-finder ``rglob`` calls
+# walked the ENTIRE tree (including node_modules, virtualenvs, .git) once
+# per pattern — 12+ full walks, measured at ~6 min on a medium repo with a
+# checked-in .venv. One ``os.walk`` with directory pruning, memoized on the
+# resolver context, replaces all of them.
+#
+# The prune list is deliberately narrow: vendored/derived trees that can
+# never contain *our* manifests or docs. ``.github`` and other dot-dirs that
+# may hold real package.json files (custom actions) are NOT pruned wholesale.
+_SCAN_PRUNE_DIRS: frozenset[str] = frozenset({
+ "node_modules",
+ ".git",
+ ".hg",
+ ".svn",
+ ".venv",
+ "venv",
+ ".tox",
+ "__pycache__",
+ ".repowise",
+ ".repowise.prebench-bak",
+ ".next",
+ ".nuxt",
+ ".turbo",
+ ".cache",
+ ".parcel-cache",
+ ".yarn",
+ ".pnpm-store",
+})
+
+_VITEST_CONFIG_NAMES: frozenset[str] = frozenset({
+ "vitest.config.ts", "vitest.config.js", "vitest.config.mts",
+ "vitest.config.mjs", "vitest.config.cjs", "vitest.config.cts",
+ "vite.config.ts", "vite.config.js", "vite.config.mts",
+ "vite.config.mjs",
+})
+
+
+@dataclass
+class _RepoFileScan:
+ """File locations gathered by the single pruned walk."""
+
+ mdx_files: list[Path] = field(default_factory=list)
+ vitest_configs: list[Path] = field(default_factory=list)
+ package_jsons: list[Path] = field(default_factory=list)
+
+
+def _scan_repo_files(repo_path: Path) -> _RepoFileScan:
+ """One pruned ``os.walk`` collecting every finder's target files."""
+ scan = _RepoFileScan()
+ for dirpath, dirnames, filenames in os.walk(repo_path):
+ dirnames[:] = [d for d in dirnames if d not in _SCAN_PRUNE_DIRS]
+ for fname in filenames:
+ if fname.endswith(".mdx"):
+ scan.mdx_files.append(Path(dirpath) / fname)
+ elif fname in _VITEST_CONFIG_NAMES:
+ scan.vitest_configs.append(Path(dirpath) / fname)
+ elif fname == "package.json":
+ scan.package_jsons.append(Path(dirpath) / fname)
+ return scan
+
+
+def _get_repo_scan(ctx: "ResolverContext") -> _RepoFileScan:
+ """Memoized accessor — one walk per resolver context."""
+ cached = getattr(ctx, "_ts_repo_file_scan", None)
+ if cached is not None:
+ return cached
+ scan = _scan_repo_files(ctx.repo_path) if ctx.repo_path is not None else _RepoFileScan()
+ ctx._ts_repo_file_scan = scan # type: ignore[attr-defined]
+ return scan
+
+
# Order in which we collapse Node "conditional exports" objects down to
# a single target. Source-pointing conditions come first so a TS-aware
# static analyser sees the original ``.ts`` file rather than a built
@@ -453,7 +529,7 @@ def find_mdx_import_targets(ctx: "ResolverContext") -> set[str]:
from .typescript import resolve_ts_js_import
targets: set[str] = set()
- for mdx in ctx.repo_path.rglob("*.mdx"):
+ for mdx in _get_repo_scan(ctx).mdx_files:
try:
text = mdx.read_text(encoding="utf-8", errors="ignore")
except Exception:
@@ -537,34 +613,25 @@ def find_vitest_include_targets(ctx: "ResolverContext") -> set[str]:
return set()
targets: set[str] = set()
- config_globs = ("vitest.config.ts", "vitest.config.js", "vitest.config.mts",
- "vitest.config.mjs", "vitest.config.cjs", "vitest.config.cts",
- "vite.config.ts", "vite.config.js", "vite.config.mts",
- "vite.config.mjs")
- seen: set[Path] = set()
- for pattern in config_globs:
- for cfg in ctx.repo_path.rglob(pattern):
- if cfg in seen:
- continue
- seen.add(cfg)
- try:
- text = cfg.read_text(encoding="utf-8", errors="ignore")
- except Exception:
- continue
- cfg_dir = cfg.parent
- for inc_match in _VITEST_INCLUDE_RE.finditer(text):
- for str_match in _VITEST_STRING_RE.finditer(inc_match.group(1)):
- glob_pat = str_match.group(1)
- # Resolve glob relative to the config file's directory.
- base = (cfg_dir / glob_pat).as_posix()
- try:
- rel_glob = Path(base).relative_to(ctx.repo_path).as_posix()
- except ValueError:
- continue
- regex = _vitest_glob_to_regex(rel_glob)
- for candidate in ctx.path_set:
- if regex.match(candidate):
- targets.add(candidate)
+ for cfg in _get_repo_scan(ctx).vitest_configs:
+ try:
+ text = cfg.read_text(encoding="utf-8", errors="ignore")
+ except Exception:
+ continue
+ cfg_dir = cfg.parent
+ for inc_match in _VITEST_INCLUDE_RE.finditer(text):
+ for str_match in _VITEST_STRING_RE.finditer(inc_match.group(1)):
+ glob_pat = str_match.group(1)
+ # Resolve glob relative to the config file's directory.
+ base = (cfg_dir / glob_pat).as_posix()
+ try:
+ rel_glob = Path(base).relative_to(ctx.repo_path).as_posix()
+ except ValueError:
+ continue
+ regex = _vitest_glob_to_regex(rel_glob)
+ for candidate in ctx.path_set:
+ if regex.match(candidate):
+ targets.add(candidate)
return targets
@@ -640,7 +707,10 @@ def find_npm_script_entry_targets(ctx: "ResolverContext") -> set[str]:
if ctx.repo_path is None:
return set()
- repo_root = ctx.repo_path.resolve()
+ # NOTE: not .resolve()d — the shared scan walks ctx.repo_path as given,
+ # so relative_to() below must use the same base or it silently drops
+ # every manifest.
+ repo_root = ctx.repo_path
path_set = ctx.path_set
targets: set[str] = set()
@@ -656,10 +726,8 @@ def find_npm_script_entry_targets(ctx: "ResolverContext") -> set[str]:
dirs_in_repo.setdefault(p[:slash], []).append(p)
idx = slash + 1
- for pkg_file in repo_root.rglob("package.json"):
- # Skip node_modules — those are dependency manifests, not ours.
- if "node_modules" in pkg_file.parts:
- continue
+ for pkg_file in _get_repo_scan(ctx).package_jsons:
+ # node_modules manifests are already pruned by the shared scan.
try:
data = json.loads(pkg_file.read_text(encoding="utf-8", errors="ignore"))
except Exception:
diff --git a/packages/core/src/repowise/core/persistence/crud/_shared.py b/packages/core/src/repowise/core/persistence/crud/_shared.py
index 617afcfe..0e291191 100644
--- a/packages/core/src/repowise/core/persistence/crud/_shared.py
+++ b/packages/core/src/repowise/core/persistence/crud/_shared.py
@@ -1,8 +1,9 @@
"""Shared internals for the ``crud`` package.
Holds the datetime parser, batch-size constant, the job-status whitelist, and
-the generic SELECT-then-INSERT/UPDATE loop (:func:`_batch_upsert`) that the
-per-domain batch upserts delegate to. Private to the package.
+the generic keyed SELECT-once-then-INSERT/UPDATE loop
+(:func:`_batch_upsert_keyed`) that the per-domain batch upserts delegate to.
+Private to the package.
"""
from __future__ import annotations
@@ -28,39 +29,53 @@ def _parse_dt(ts: str) -> datetime:
return dt
-async def _batch_upsert(
+async def _batch_upsert_keyed(
session: AsyncSession,
model: type[Any],
items: Iterable[Any],
*,
- key_fn: Callable[[Any], tuple[Any, ...]],
+ prefilter: tuple[Any, ...],
+ item_key_fn: Callable[[Any], Any],
+ row_key_fn: Callable[[Any], Any],
update_fn: Callable[[Any, Any], None],
insert_fn: Callable[[Any], Any],
batch_size: int | None = None,
) -> None:
- """Generic SELECT-then-INSERT/UPDATE loop shared by the batch upserts.
+ """Generic batch upsert with ONE existence query for the whole batch.
- For each item, look up the existing row with the filter expressions from
- ``key_fn(item)``; if found, mutate it via ``update_fn(existing, item)``,
- otherwise ``session.add(insert_fn(item))``.
+ The previous generic loop issued a SELECT round-trip per item; on bulk writes
+ (graph nodes/edges, symbols, git metadata) that is 10k-30k SELECTs per
+ persist and dominated the whole persist step (measured: 9.3s for a
+ 13k-node ``persist_graph_nodes``). Here all candidate rows matching
+ *prefilter* (typically ``repository_id == X``) are fetched once and
+ matched in memory: ``item_key_fn(item)`` must equal ``row_key_fn(row)``
+ exactly when the legacy ``key_fn`` filter would have found that row.
- Flushes once after the whole sequence, or once per ``batch_size`` chunk when
- *batch_size* is given (matching the SELECT-then-write semantics the callers
- relied on before this was extracted — including no flush at all on an empty
- batched sequence).
+ Within-batch duplicate keys keep the legacy outcome: the first item
+ inserts, later ones update the pending object (the per-item SELECT used
+ to see the autoflushed insert).
"""
+ materialized = list(items)
+ if not materialized:
+ if batch_size is None:
+ await session.flush()
+ return
+
+ existing_rows = (await session.execute(select(model).where(*prefilter))).scalars().all()
+ by_key: dict[Any, Any] = {row_key_fn(row): row for row in existing_rows}
+
if batch_size is None:
- chunks: list[list[Any]] = [list(items)]
+ chunks: list[list[Any]] = [materialized]
else:
- materialized = list(items)
chunks = [materialized[i : i + batch_size] for i in range(0, len(materialized), batch_size)]
for chunk in chunks:
for item in chunk:
- existing = (
- await session.execute(select(model).where(*key_fn(item)))
- ).scalar_one_or_none()
+ key = item_key_fn(item)
+ existing = by_key.get(key)
if existing is not None:
update_fn(existing, item)
else:
- session.add(insert_fn(item))
+ obj = insert_fn(item)
+ session.add(obj)
+ by_key[key] = obj
await session.flush()
diff --git a/packages/core/src/repowise/core/persistence/crud/external_systems.py b/packages/core/src/repowise/core/persistence/crud/external_systems.py
index 97107fda..5766e39e 100644
--- a/packages/core/src/repowise/core/persistence/crud/external_systems.py
+++ b/packages/core/src/repowise/core/persistence/crud/external_systems.py
@@ -18,7 +18,7 @@ from ..models import (
_new_uuid,
_now_utc,
)
-from ._shared import _batch_upsert
+from ._shared import _batch_upsert_keyed
# ---------------------------------------------------------------------------
# ExternalSystem CRUD
@@ -153,14 +153,13 @@ async def batch_upsert_symbols(
Accepts ingestion.models.Symbol dataclass instances (duck-typed).
"""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
WikiSymbol,
symbols,
- key_fn=lambda sym: (
- WikiSymbol.repository_id == repository_id,
- WikiSymbol.symbol_id == _symbol_id(sym),
- ),
+ prefilter=(WikiSymbol.repository_id == repository_id,),
+ item_key_fn=_symbol_id,
+ row_key_fn=lambda row: row.symbol_id,
update_fn=_update_wiki_symbol,
insert_fn=lambda sym: WikiSymbol(
id=_new_uuid(),
diff --git a/packages/core/src/repowise/core/persistence/crud/git.py b/packages/core/src/repowise/core/persistence/crud/git.py
index dc2ef020..04877177 100644
--- a/packages/core/src/repowise/core/persistence/crud/git.py
+++ b/packages/core/src/repowise/core/persistence/crud/git.py
@@ -16,7 +16,7 @@ from ..models import (
_new_uuid,
_now_utc,
)
-from ._shared import _BATCH_SIZE, _batch_upsert
+from ._shared import _BATCH_SIZE, _batch_upsert_keyed
# ---------------------------------------------------------------------------
# GitMetadata CRUD
@@ -106,14 +106,13 @@ async def upsert_git_metadata_bulk(
metadata_list: list[dict],
) -> None:
"""Bulk upsert git metadata rows in batches."""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
GitMetadata,
metadata_list,
- key_fn=lambda meta: (
- GitMetadata.repository_id == repository_id,
- GitMetadata.file_path == meta.get("file_path", ""),
- ),
+ prefilter=(GitMetadata.repository_id == repository_id,),
+ item_key_fn=lambda meta: meta.get("file_path", ""),
+ row_key_fn=lambda row: row.file_path,
update_fn=_update_git_metadata,
insert_fn=lambda meta: GitMetadata(
id=_new_uuid(),
@@ -219,14 +218,13 @@ async def upsert_git_commits_bulk(
commit_rows: list[dict],
) -> None:
"""Bulk upsert per-commit rows (keyed on ``repository_id`` + ``sha``)."""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
GitCommit,
commit_rows,
- key_fn=lambda row: (
- GitCommit.repository_id == repository_id,
- GitCommit.sha == row.get("sha", ""),
- ),
+ prefilter=(GitCommit.repository_id == repository_id,),
+ item_key_fn=lambda row: row.get("sha", ""),
+ row_key_fn=lambda row: row.sha,
update_fn=_update_git_commit,
insert_fn=lambda row: GitCommit(
id=_new_uuid(),
@@ -338,14 +336,13 @@ async def upsert_git_function_blame_bulk(
rows: list[dict],
) -> None:
"""Bulk upsert per-function blame rows (keyed ``repository_id`` + ``symbol_id``)."""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
GitFunctionBlame,
rows,
- key_fn=lambda row: (
- GitFunctionBlame.repository_id == repository_id,
- GitFunctionBlame.symbol_id == row.get("symbol_id", ""),
- ),
+ prefilter=(GitFunctionBlame.repository_id == repository_id,),
+ item_key_fn=lambda row: row.get("symbol_id", ""),
+ row_key_fn=lambda row: row.symbol_id,
update_fn=_update_git_function_blame,
insert_fn=lambda row: GitFunctionBlame(
id=_new_uuid(),
diff --git a/packages/core/src/repowise/core/persistence/crud/graph.py b/packages/core/src/repowise/core/persistence/crud/graph.py
index 977d429a..3ded527e 100644
--- a/packages/core/src/repowise/core/persistence/crud/graph.py
+++ b/packages/core/src/repowise/core/persistence/crud/graph.py
@@ -17,7 +17,7 @@ from ..models import (
GraphNode,
_new_uuid,
)
-from ._shared import _BATCH_SIZE, _batch_upsert
+from ._shared import _BATCH_SIZE, _batch_upsert_keyed
# ---------------------------------------------------------------------------
# Graph CRUD (batch)
@@ -59,14 +59,13 @@ async def batch_upsert_graph_nodes(
Uses SELECT-then-INSERT/UPDATE for dialect portability.
"""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
GraphNode,
nodes,
- key_fn=lambda n: (
- GraphNode.repository_id == repository_id,
- GraphNode.node_id == n.get("node_id", ""),
- ),
+ prefilter=(GraphNode.repository_id == repository_id,),
+ item_key_fn=lambda n: n.get("node_id", ""),
+ row_key_fn=lambda row: row.node_id,
update_fn=_update_graph_node,
insert_fn=lambda n: GraphNode(
id=_new_uuid(),
@@ -89,16 +88,17 @@ async def batch_upsert_graph_edges(
The unique constraint is (repository_id, source, target, edge_type),
allowing multiple edge types between the same pair of nodes.
"""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
GraphEdge,
edges,
- key_fn=lambda e: (
- GraphEdge.repository_id == repository_id,
- GraphEdge.source_node_id == e.get("source_node_id", ""),
- GraphEdge.target_node_id == e.get("target_node_id", ""),
- GraphEdge.edge_type == e.get("edge_type", "imports"),
+ prefilter=(GraphEdge.repository_id == repository_id,),
+ item_key_fn=lambda e: (
+ e.get("source_node_id", ""),
+ e.get("target_node_id", ""),
+ e.get("edge_type", "imports"),
),
+ row_key_fn=lambda row: (row.source_node_id, row.target_node_id, row.edge_type),
update_fn=_update_graph_edge,
insert_fn=lambda e: GraphEdge(
id=_new_uuid(),
@@ -125,14 +125,13 @@ async def batch_upsert_graph_metrics(
``GraphBuilder.load_metrics_from_sql`` on large repos. SELECT-then-write
for dialect portability (SQLite + Postgres).
"""
- await _batch_upsert(
+ await _batch_upsert_keyed(
session,
GraphMetric,
list(metrics.items()),
- key_fn=lambda kv: (
- GraphMetric.repository_id == repository_id,
- GraphMetric.node_id == kv[0],
- ),
+ prefilter=(GraphMetric.repository_id == repository_id,),
+ item_key_fn=lambda kv: kv[0],
+ row_key_fn=lambda row: row.node_id,
update_fn=lambda existing, kv: _update_graph_metric(existing, kv[1]),
insert_fn=lambda kv: GraphMetric(
id=_new_uuid(),
diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.py
index 69edaece..82dc0190 100644
--- a/packages/core/src/repowise/core/persistence/models.py
+++ b/packages/core/src/repowise/core/persistence/models.py
@@ -403,6 +403,16 @@ class GitMetadata(Base):
change_entropy: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
change_entropy_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+ # Agent-provenance rollup: how much of this file's indexed history is
+ # agent-attributed (deterministic local-channel classification — identity
+ # fields, message footers, co-author trailers; see
+ # ingestion.git_indexer.agent_provenance). agent_tier_counts_json maps
+ # autonomy tier ("1" near-autonomous / "2" human-driven / "3" assisted)
+ # to commit counts. agent_authored_pct stays NULL until the next reindex.
+ agent_commit_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ agent_authored_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+ agent_tier_counts_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=_now_utc
)
@@ -463,6 +473,16 @@ class GitCommit(Base):
change_risk_score: Mapped[float | None] = mapped_column(Float, nullable=True)
change_risk_level: Mapped[str | None] = mapped_column(String(16), nullable=True)
+ # Agent provenance: which coding agent (if any) authored this commit, at
+ # what autonomy tier (1 near-autonomous bot account · 2 human-driven agent
+ # · 3 assisted/co-authored), via which attribution channel, and with what
+ # confidence band. NULL throughout = human-authored (or pre-migration rows;
+ # back-populated on the next index). Deterministic local-git channels only.
+ agent_name: Mapped[str | None] = mapped_column(String(32), nullable=True)
+ agent_autonomy_tier: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ agent_channel: Mapped[str | None] = mapped_column(String(32), nullable=True)
+ agent_confidence: Mapped[str | None] = mapped_column(String(8), nullable=True)
+
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=_now_utc
)
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.py
index 95507eb2..e5d1d970 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py
@@ -52,6 +52,28 @@ class VectorStore(ABC):
"""Embed *query* and return the *limit* nearest pages."""
...
+ async def search_many(
+ self, queries: list[str], limit: int = 10
+ ) -> list[list[SearchResult]]:
+ """Batch variant of :meth:`search` — one result list per query, aligned
+ by index.
+
+ The default implementation fires the per-query searches concurrently
+ via ``asyncio.gather``; a failed query yields an empty list (matching
+ the caller-side behaviour of swallowing a single failed search).
+ Backends override this to embed *all* queries in a single embedder
+ call — the network round-trip dominates each search, so batching the
+ embedding turns N round-trips into 1.
+ """
+ import asyncio as _asyncio
+
+ if not queries:
+ return []
+ results = await _asyncio.gather(
+ *(self.search(q, limit=limit) for q in queries), return_exceptions=True
+ )
+ return [r if isinstance(r, list) else [] for r in results]
+
@abstractmethod
async def delete(self, page_id: str) -> None:
"""Remove the vector for *page_id* from the store."""
diff --git a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
index b97e0463..02188d18 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
@@ -34,12 +34,7 @@ class InMemoryVectorStore(VectorStore):
for (page_id, _text, metadata), vector in zip(items, vectors, strict=True):
self._store[page_id] = (vector, dict(metadata))
- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
- if not self._store:
- return []
- q_vecs = await self._embedder.embed([query])
- q_vec = q_vecs[0]
-
+ def _search_by_vector(self, q_vec: list[float], limit: int) -> list[SearchResult]:
scored: list[tuple[float, str, dict]] = []
for pid, (vec, meta) in self._store.items():
score = cosine_similarity(q_vec, vec)
@@ -63,6 +58,23 @@ class InMemoryVectorStore(VectorStore):
)
return results
+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+ if not self._store:
+ return []
+ q_vecs = await self._embedder.embed([query])
+ return self._search_by_vector(q_vecs[0], limit)
+
+ async def search_many(
+ self, queries: list[str], limit: int = 10
+ ) -> list[list[SearchResult]]:
+ """One embedder call for all queries, then local scoring per query."""
+ if not queries:
+ return []
+ if not self._store:
+ return [[] for _ in queries]
+ q_vecs = await self._embedder.embed(list(queries))
+ return [self._search_by_vector(q_vec, limit) for q_vec in q_vecs]
+
async def delete(self, page_id: str) -> None:
self._store.pop(page_id, None)
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
index 84f1f415..4d521c1b 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
@@ -10,6 +10,17 @@ from ._base import VectorStore
__all__ = ["LanceDBVectorStore"]
+def _paths_in_filter(paths: list[str]) -> str:
+ """Build an SQL-injection-safe ``target_path IN (...)`` LanceDB filter.
+
+ LanceDB's ``.where()`` takes a DataFusion SQL string with no bind
+ parameters, so each path is quoted with the same quote-doubling escape
+ the single-path lookup uses.
+ """
+ quoted = ", ".join("'" + p.replace("'", "''") + "'" for p in paths)
+ return f"target_path IN ({quoted})"
+
+
class LanceDBVectorStore(VectorStore):
"""Vector store backed by LanceDB (embedded, local file storage).
@@ -154,14 +165,7 @@ class LanceDBVectorStore(VectorStore):
]
await self._upsert_rows(rows)
- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
- await self._ensure_connected()
- if self._table is None:
- return []
-
- q_vecs = await self._embedder.embed([query])
- q_vec = [float(v) for v in q_vecs[0]]
-
+ async def _search_by_vector(self, q_vec: list[float], limit: int) -> list[SearchResult]:
# Query with explicit cosine distance so ``_distance`` is a cosine
# distance (1 - cos); we return ``1 - _distance`` = cosine similarity.
# This makes the score semantics match the other backends
@@ -185,6 +189,32 @@ class LanceDBVectorStore(VectorStore):
for r in raw
]
+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+ await self._ensure_connected()
+ if self._table is None:
+ return []
+
+ q_vecs = await self._embedder.embed([query])
+ return await self._search_by_vector([float(v) for v in q_vecs[0]], limit)
+
+ async def search_many(
+ self, queries: list[str], limit: int = 10
+ ) -> list[list[SearchResult]]:
+ """One embedder call for all queries; the vector lookups are local."""
+ if not queries:
+ return []
+ await self._ensure_connected()
+ if self._table is None:
+ return [[] for _ in queries]
+ q_vecs = await self._embedder.embed(list(queries))
+ out: list[list[SearchResult]] = []
+ for q_vec in q_vecs:
+ try:
+ out.append(await self._search_by_vector([float(v) for v in q_vec], limit))
+ except Exception:
+ out.append([])
+ return out
+
async def delete(self, page_id: str) -> None:
await self._ensure_connected()
if self._table is not None:
@@ -230,3 +260,35 @@ class LanceDBVectorStore(VectorStore):
summary = rows[0].get("content_snippet") or ""
return {"summary": str(summary), "key_exports": []}
+
+ async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:
+ """One ``IN``-filtered scan instead of one filtered query per path.
+
+ Mirrors the single-path semantics (first row per path wins, empty
+ summaries dropped, ``key_exports`` not stored in this schema).
+ """
+ if not paths:
+ return {}
+ await self._ensure_connected()
+ if self._table is None:
+ return {}
+
+ try:
+ rows = (
+ await self._table.query() # type: ignore[union-attr]
+ .where(_paths_in_filter(paths))
+ .select(["target_path", "content_snippet"])
+ .to_list()
+ )
+ except Exception:
+ return {}
+
+ out: dict[str, dict] = {}
+ for r in rows:
+ tp = str(r.get("target_path") or "")
+ if not tp or tp in out:
+ continue
+ summary = r.get("content_snippet") or ""
+ if summary:
+ out[tp] = {"summary": str(summary), "key_exports": []}
+ return out
diff --git a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
index 4a5f7f40..8c441ed3 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
@@ -20,6 +20,23 @@ def _encode(vector: list[float]) -> str:
return "[" + ",".join(str(v) for v in vector) + "]"
+def _summary_payload(content: object, metadata: object) -> dict:
+ """Build the ``{'summary', 'key_exports'}`` payload from a wiki_pages row."""
+ key_exports: list[str] = []
+ if metadata and isinstance(metadata, dict):
+ key_exports = list(metadata.get("exports", []))
+ elif metadata and isinstance(metadata, str):
+ import json
+
+ try:
+ meta = json.loads(metadata)
+ key_exports = list(meta.get("exports", []))
+ except (json.JSONDecodeError, AttributeError):
+ pass
+
+ return {"summary": str(content or "")[:500], "key_exports": key_exports}
+
+
class PgVectorStore(VectorStore):
"""Vector store that writes embeddings to the ``wiki_pages.embedding`` column.
@@ -71,8 +88,9 @@ class PgVectorStore(VectorStore):
"UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"
)
async with self._session_factory() as session:
- for p in params:
- await session.execute(stmt, p)
+ # executemany: one driver round-trip batch instead of one UPDATE
+ # round-trip per row.
+ await session.execute(stmt, params)
await session.commit()
async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
@@ -108,6 +126,51 @@ class PgVectorStore(VectorStore):
for r in raw
]
+ async def search_many(
+ self, queries: list[str], limit: int = 10
+ ) -> list[list[SearchResult]]:
+ """One embedder call for all queries; per-query SELECTs share a session."""
+ if not queries:
+ return []
+ q_vecs = await self._embedder.embed(list(queries))
+
+ from sqlalchemy.sql import text as sa_text
+
+ stmt = sa_text(
+ "SELECT id, title, content, page_type, target_path, "
+ " 1 - (embedding <=> CAST(:q AS vector)) AS score "
+ "FROM wiki_pages "
+ "WHERE embedding IS NOT NULL "
+ "ORDER BY embedding <=> CAST(:q AS vector) "
+ "LIMIT :lim"
+ )
+ out: list[list[SearchResult]] = []
+ async with self._session_factory() as session:
+ for q_vec in q_vecs:
+ try:
+ rows = await session.execute(
+ stmt, {"q": _encode(q_vec), "lim": limit}
+ )
+ raw = rows.fetchall()
+ except Exception:
+ out.append([])
+ continue
+ out.append(
+ [
+ SearchResult(
+ page_id=r[0],
+ title=r[1],
+ page_type=r[3],
+ target_path=r[4],
+ score=float(r[5]),
+ snippet=str(r[2])[:200].rstrip(),
+ search_type="vector",
+ )
+ for r in raw
+ ]
+ )
+ return out
+
async def delete(self, page_id: str) -> None:
from sqlalchemy.sql import text as sa_text
@@ -153,17 +216,36 @@ class PgVectorStore(VectorStore):
if row is None:
return None
- content = str(row[0] or "")[:500]
- key_exports: list[str] = []
- if row[1] and isinstance(row[1], dict):
- key_exports = list(row[1].get("exports", []))
- elif row[1] and isinstance(row[1], str):
- import json
+ return _summary_payload(row[0], row[1])
+
+ async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:
+ """One ``IN``-filtered SELECT instead of one query per path.
+
+ Like the single-path variant (``LIMIT 1`` with no ``ORDER BY``), when
+ several pages share a ``target_path`` an arbitrary one wins — here the
+ first row returned per path.
+ """
+ if not paths:
+ return {}
+
+ from sqlalchemy import bindparam
+ from sqlalchemy.sql import text as sa_text
+
+ stmt = sa_text(
+ "SELECT target_path, content, metadata FROM wiki_pages "
+ "WHERE target_path IN :paths"
+ ).bindparams(bindparam("paths", expanding=True))
- try:
- meta = json.loads(row[1])
- key_exports = list(meta.get("exports", []))
- except (json.JSONDecodeError, AttributeError):
- pass
+ async with self._session_factory() as session:
+ rows = await session.execute(stmt, {"paths": list(paths)})
+ raw = rows.fetchall()
- return {"summary": content, "key_exports": key_exports}
+ out: dict[str, dict] = {}
+ for r in raw:
+ tp = str(r[0])
+ if tp in out:
+ continue
+ payload = _summary_payload(r[1], r[2])
+ if payload.get("summary"):
+ out[tp] = payload
+ return out
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index 454dac6b..fc71d80d 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -108,6 +108,114 @@ async def persist_graph_nodes(
logger.warning("graph_metrics_materialize_skipped", error=str(exc))
+# Chunk size for IN (...) deletes — stays under SQLite's host-parameter limit.
+_PRUNE_CHUNK = 500
+
+
+async def _prune_stale_file_rows(
+ session: Any,
+ repo_id: str,
+ current_graph_file_paths: set[str],
+ current_git_file_paths: set[str],
+) -> None:
+ """Delete file-scoped rows for files absent from the latest full pipeline run.
+
+ The parser and git indexer disagree on the file set — a file can be
+ git-tracked yet absent from ``parsed_files`` (parse failure, unparsed
+ extension, skipped) — so the tables use different sources of truth.
+ *current_graph_file_paths* (from ``parsed_files``) governs graph/analysis
+ tables; *current_git_file_paths* (from ``git_metadata_list``) governs
+ ``git_metadata`` only. Each set independently no-ops when empty to avoid
+ wiping rows on a broken run. FULL persistence only — not incremental paths.
+ """
+ from sqlalchemy import delete, or_, select
+
+ from repowise.core.persistence.models import (
+ DeadCodeFinding,
+ GitMetadata,
+ GraphEdge,
+ GraphMetric,
+ GraphNode,
+ HealthFileMetric,
+ HealthFinding,
+ SecurityFinding,
+ WikiSymbol,
+ )
+
+ async def _delete_stale_by_paths(model: Any, column: Any, current: set[str]) -> None:
+ # Diff persisted paths against *current* in Python so the IN (...) is
+ # bounded by the stale set, not the whole repo (SQLite param limit).
+ if not current:
+ return
+ existing = set(
+ (await session.execute(select(column).where(model.repository_id == repo_id).distinct()))
+ .scalars()
+ .all()
+ )
+ stale = [p for p in existing if p not in current]
+ for i in range(0, len(stale), _PRUNE_CHUNK):
+ await session.execute(
+ delete(model).where(
+ model.repository_id == repo_id,
+ column.in_(stale[i : i + _PRUNE_CHUNK]),
+ )
+ )
+
+ # ---- Graph nodes + edges -------------------------------------------------
+ # File nodes key on node_id; symbol nodes on file_path. Delete edges before
+ # nodes (no FK cascade between the tables).
+ if current_graph_file_paths:
+ node_rows = (
+ await session.execute(
+ select(GraphNode.node_id, GraphNode.node_type, GraphNode.file_path).where(
+ GraphNode.repository_id == repo_id
+ )
+ )
+ ).all()
+ stale_node_ids = [
+ node_id
+ for node_id, node_type, file_path in node_rows
+ if (node_type == "file" and node_id not in current_graph_file_paths)
+ or (
+ node_type != "file"
+ and file_path is not None
+ and file_path not in current_graph_file_paths
+ )
+ ]
+ for i in range(0, len(stale_node_ids), _PRUNE_CHUNK):
+ batch = stale_node_ids[i : i + _PRUNE_CHUNK]
+ await session.execute(
+ delete(GraphEdge).where(
+ GraphEdge.repository_id == repo_id,
+ or_(
+ GraphEdge.source_node_id.in_(batch),
+ GraphEdge.target_node_id.in_(batch),
+ ),
+ )
+ )
+ await session.execute(
+ delete(GraphNode).where(
+ GraphNode.repository_id == repo_id,
+ GraphNode.node_id.in_(batch),
+ )
+ )
+
+ # GraphMetric is file-level only (node_id == path).
+ await _delete_stale_by_paths(GraphMetric, GraphMetric.node_id, current_graph_file_paths)
+ await _delete_stale_by_paths(WikiSymbol, WikiSymbol.file_path, current_graph_file_paths)
+ await _delete_stale_by_paths(
+ SecurityFinding, SecurityFinding.file_path, current_graph_file_paths
+ )
+ await _delete_stale_by_paths(
+ DeadCodeFinding, DeadCodeFinding.file_path, current_graph_file_paths
+ )
+ await _delete_stale_by_paths(
+ HealthFileMetric, HealthFileMetric.file_path, current_graph_file_paths
+ )
+ await _delete_stale_by_paths(HealthFinding, HealthFinding.file_path, current_graph_file_paths)
+ await _delete_stale_by_paths(GitMetadata, GitMetadata.file_path, current_git_file_paths)
+
+
async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
"""Persist ingestion-phase outputs: graph nodes/edges, external systems,
symbols, and the per-file security scan.
@@ -412,6 +520,18 @@ async def persist_pipeline_result(
This function mutates ``sym.file_path`` on parsed-file symbols that
lack one. Callers should treat *result* as consumed after this call.
"""
+ # Prune rows for files absent from this full result before the phase
+ # persisters re-upsert. graph/analysis tables key off parsed_files;
+ # git_metadata keys off the git indexer's set (a file can be git-tracked
+ # but unparsed). Runs only here, never in the reusable phase persisters.
+ current_graph_file_paths = {pf.file_info.path for pf in result.parsed_files}
+ current_git_file_paths = {
+ (gm if isinstance(gm, dict) else dataclasses.asdict(gm)).get("file_path", "")
+ for gm in result.git_metadata_list
+ }
+ current_git_file_paths.discard("")
+ await _prune_stale_file_rows(session, repo_id, current_graph_file_paths, current_git_file_paths)
+
symbol_count = await persist_ingestion(result, session, repo_id)
await persist_git(result, session, repo_id)
await persist_analysis(result, session, repo_id)
diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.py
index 8a099766..5e07fcaa 100644
--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py
+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py
@@ -106,6 +106,7 @@ async def _run_health_analysis(
git_meta_map=git_meta_map,
parsed_files=parsed_files,
module_map=module_map,
+ duplication_cache_dir=(repo_path / ".repowise") if repo_path is not None else None,
)
# Load per-file override rules from `.repowise/health-rules.json`.
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index 693acf59..525a5a36 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -7,6 +7,7 @@ orchestrator.py) imports these phase functions. No CLI/click/rich imports.
from __future__ import annotations
import asyncio
+import contextlib
import os
import time
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
@@ -85,6 +86,85 @@ def _parse_one(path_and_fi_and_bytes: tuple) -> Any:
return (fi.abs_path, str(exc))
+def _split_cached(
+ repo_path: Path,
+ fi_and_bytes: list[tuple],
+ progress: ProgressCallback | None,
+) -> tuple[Any, dict[int, Any], list[tuple[int, tuple, str]]]:
+ """Partition pre-read files into parse-cache hits and misses.
+
+ Returns ``(cache, hits, misses)`` where *hits* maps the position in
+ *fi_and_bytes* to a cached ParsedFile and *misses* keeps ``(position,
+ (FileInfo, bytes), content_hash)`` triples that still need a real parse
+ (the hash rides along so the post-parse ``put`` doesn't re-digest).
+ Positions let the caller reassemble results in traversal order. Hits
+ tick the parse progress bar here (misses tick when their worker
+ finishes). Cache failures degrade to all-miss.
+ """
+ from repowise.core.ingestion import compute_content_hash
+ from repowise.core.ingestion.parse_cache import ParseCache
+
+ cache = None
+ hits: dict[int, Any] = {}
+ misses: list[tuple[int, tuple, str]] = []
+ try:
+ cache = ParseCache(repo_path / ".repowise")
+ cache.load()
+ except Exception as exc:
+ logger.debug("parse_cache_init_failed", error=str(exc))
+ return None, hits, [(idx, item, "") for idx, item in enumerate(fi_and_bytes)]
+
+ for idx, (fi, source) in enumerate(fi_and_bytes):
+ content_hash = compute_content_hash(source)
+ parsed = cache.get(fi, content_hash)
+ if parsed is not None:
+ hits[idx] = parsed
+ if progress:
+ progress.on_item_done("parse")
+ else:
+ misses.append((idx, (fi, source), content_hash))
+ if hits:
+ logger.info("parse_cache_used", hits=cache.hits, misses=cache.misses)
+ return cache, hits, misses
+
+
+def _cache_parsed(cache: Any, parsed: Any, content_hash: str) -> None:
+ """Best-effort put of a freshly parsed file into the parse cache."""
+ if cache is None or not content_hash:
+ return
+ # Cache failures must never fail the parse phase.
+ with contextlib.suppress(Exception):
+ cache.put(parsed, content_hash)
+
+
+def _read_sources(
+ file_infos: list[Any],
+ progress: ProgressCallback | None,
+) -> list[tuple]:
+ """Read source bytes for *file_infos* with a thread pool (I/O-bound).
+
+ Returns ``(FileInfo, bytes)`` tuples in the same order as the input —
+ the parse aggregation loop indexes positionally. Files that fail to
+ read are dropped, ticking the parse bar once each (matching the old
+ sequential loop's behavior).
+ """
+
+ def _read_one(fi: Any) -> tuple | None:
+ try:
+ return (fi, Path(fi.abs_path).read_bytes())
+ except Exception:
+ if progress:
+ progress.on_item_done("parse")
+ return None
+
+ if not file_infos:
+ return []
+ workers = min(32, max(4, (os.cpu_count() or 4) * 2))
+ with ThreadPoolExecutor(max_workers=workers) as pool:
+ results = list(pool.map(_read_one, file_infos))
+ return [r for r in results if r is not None]
+
+
async def _run_ingestion(
repo_path: Path,
*,
@@ -156,70 +236,82 @@ async def _run_ingestion(
if progress:
progress.on_phase_start("parse", len(file_infos))
- # Read source bytes up front (I/O, sequential — fast enough; keeps worker
+ # Read source bytes up front in a thread pool (I/O-bound; keeps worker
# args small: FileInfo + bytes, both picklable plain dataclasses/bytes).
- fi_and_bytes: list[tuple] = []
- for fi in file_infos:
- try:
- source = Path(fi.abs_path).read_bytes()
- fi_and_bytes.append((fi, source))
- except Exception:
- if progress:
- progress.on_item_done("parse")
+ fi_and_bytes: list[tuple] = _read_sources(file_infos, progress)
parsed_files: list[Any] = []
source_map: dict[str, bytes] = {}
- graph_builder = GraphBuilder(repo_path=repo_path, exclude_patterns=exclude_patterns)
+ graph_builder = GraphBuilder(
+ repo_path=repo_path,
+ exclude_patterns=exclude_patterns,
+ centrality_cache_dir=repo_path / ".repowise",
+ )
loop = asyncio.get_running_loop()
- workers = max(1, os.cpu_count() or 4)
- _use_process_pool = True
+ # Content-hash parse cache: unchanged files skip the worker pool
+ # entirely (an incremental update re-ingests the whole repo for one
+ # changed file). Misses parse exactly as before.
+ parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)
+ workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))
+
parse_results: list[Any] = []
- try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
- tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]
- # Tick the parse-progress bar as each worker finishes —
- # ``asyncio.gather`` would otherwise hold every event back
- # until the last file is done, which on PowerToys-scale
- # repos looked like a hang at ``0/N`` for many minutes.
- # Per-task done-callbacks fire on the event loop thread and
- # preserve gather's ordered results, so the aggregation
- # loop below still indexes ``fi_and_bytes`` correctly.
- if progress is not None:
- _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
- for fut in tasks:
- fut.add_done_callback(_parse_tick)
- parse_results = await asyncio.gather(*tasks, return_exceptions=True)
- except Exception as pool_exc:
- logger.warning(
- "process_pool_parse_failed_falling_back",
- error=str(pool_exc),
- )
- _use_process_pool = False
- # Fallback: in-process sequential parse
- _fallback_parser = ASTParser()
- for i, (fi, source) in enumerate(fi_and_bytes):
- try:
- result = _fallback_parser.parse_file(fi, source)
- parse_results.append(result)
- except Exception as exc:
- parse_results.append((fi.abs_path, str(exc)))
- if progress:
- progress.on_item_done("parse")
- if i % 50 == 49:
- await asyncio.sleep(0)
+ if to_parse:
+ try:
+ with ProcessPoolExecutor(max_workers=workers) as pool:
+ tasks = [
+ loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse
+ ]
+ # Tick the parse-progress bar as each worker finishes —
+ # ``asyncio.gather`` would otherwise hold every event back
+ # until the last file is done, which on PowerToys-scale
+ # repos looked like a hang at ``0/N`` for many minutes.
+ # Per-task done-callbacks fire on the event loop thread and
+ # preserve gather's ordered results, so the aggregation
+ # loop below still indexes ``to_parse`` correctly.
+ if progress is not None:
+ _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
+ for fut in tasks:
+ fut.add_done_callback(_parse_tick)
+ parse_results = await asyncio.gather(*tasks, return_exceptions=True)
+ except Exception as pool_exc:
+ logger.warning(
+ "process_pool_parse_failed_falling_back",
+ error=str(pool_exc),
+ )
+ # Fallback: in-process sequential parse
+ parse_results = []
+ _fallback_parser = ASTParser()
+ for i, (_idx, (fi, source), _h) in enumerate(to_parse):
+ try:
+ result = _fallback_parser.parse_file(fi, source)
+ parse_results.append(result)
+ except Exception as exc:
+ parse_results.append((fi.abs_path, str(exc)))
+ if progress:
+ progress.on_item_done("parse")
+ if i % 50 == 49:
+ await asyncio.sleep(0)
+
+ # Merge fresh parses back into traversal order alongside cache hits.
+ merged: dict[int, Any] = dict(cached_hits)
+ for (idx, _item, content_hash), result in zip(to_parse, parse_results, strict=True):
+ merged[idx] = result
+ if not isinstance(result, tuple | Exception):
+ _cache_parsed(parse_cache, result, content_hash)
# Aggregate results into GraphBuilder on the main loop (not thread-safe).
- for idx, result in enumerate(parse_results):
+ for idx in range(len(fi_and_bytes)):
fi, source = fi_and_bytes[idx]
+ result = merged.get(idx)
if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):
# Error tuple: (abs_path_str, error_str)
logger.debug("parse_error_in_worker", path=result[0], error=result[1])
elif isinstance(result, Exception):
logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))
- else:
+ elif result is not None:
parsed_files.append(result)
source_map[fi.path] = source
graph_builder.add_file(result)
@@ -227,6 +319,8 @@ async def _run_ingestion(
# attached above; only the fallback path ticks here (handled in
# its own loop). No tick needed in aggregation.
+ if parse_cache is not None:
+ parse_cache.save()
_phase_done(progress, "parse")
# ---- tsconfig path-alias resolver (before graph build) ------------------
@@ -434,51 +528,60 @@ async def reparse_for_resume(
if progress:
progress.on_phase_start("parse", len(file_infos))
- fi_and_bytes: list[tuple] = []
- for fi in file_infos:
- try:
- source = Path(fi.abs_path).read_bytes()
- fi_and_bytes.append((fi, source))
- except Exception:
- if progress:
- progress.on_item_done("parse")
+ fi_and_bytes: list[tuple] = _read_sources(file_infos, progress)
parsed_files: list[Any] = []
source_map: dict[str, bytes] = {}
loop = asyncio.get_running_loop()
- workers = max(1, os.cpu_count() or 4)
- parse_results: list[Any] = []
- try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
- tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]
- if progress is not None:
- _tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
- for fut in tasks:
- fut.add_done_callback(_tick)
- parse_results = await asyncio.gather(*tasks, return_exceptions=True)
- except Exception as pool_exc:
- logger.warning("resume_reparse_pool_failed_falling_back", error=str(pool_exc))
- _fallback_parser = ASTParser()
- for i, (fi, source) in enumerate(fi_and_bytes):
- try:
- parse_results.append(_fallback_parser.parse_file(fi, source))
- except Exception as exc:
- parse_results.append((fi.abs_path, str(exc)))
- if progress:
- progress.on_item_done("parse")
- if i % 50 == 49:
- await asyncio.sleep(0)
+ parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)
+ workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))
+ parse_results: list[Any] = []
- for idx, result in enumerate(parse_results):
+ if to_parse:
+ try:
+ with ProcessPoolExecutor(max_workers=workers) as pool:
+ tasks = [
+ loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse
+ ]
+ if progress is not None:
+ _tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
+ for fut in tasks:
+ fut.add_done_callback(_tick)
+ parse_results = await asyncio.gather(*tasks, return_exceptions=True)
+ except Exception as pool_exc:
+ logger.warning("resume_reparse_pool_failed_falling_back", error=str(pool_exc))
+ parse_results = []
+ _fallback_parser = ASTParser()
+ for i, (_idx, (fi, source), _h) in enumerate(to_parse):
+ try:
+ parse_results.append(_fallback_parser.parse_file(fi, source))
+ except Exception as exc:
+ parse_results.append((fi.abs_path, str(exc)))
+ if progress:
+ progress.on_item_done("parse")
+ if i % 50 == 49:
+ await asyncio.sleep(0)
+
+ merged: dict[int, Any] = dict(cached_hits)
+ for (idx, _item, content_hash), result in zip(to_parse, parse_results, strict=True):
+ merged[idx] = result
+ if not isinstance(result, tuple | Exception):
+ _cache_parsed(parse_cache, result, content_hash)
+
+ for idx in range(len(fi_and_bytes)):
fi, source = fi_and_bytes[idx]
+ result = merged.get(idx)
if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):
logger.debug("parse_error_in_worker", path=result[0], error=result[1])
elif isinstance(result, Exception):
logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))
- else:
+ elif result is not None:
parsed_files.append(result)
source_map[fi.path] = source
+
+ if parse_cache is not None:
+ parse_cache.save()
_phase_done(progress, "parse")
tech_items: list = []