Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(core): tiered doc generation + graph metrics in SQL for large repos (#220)

Raghav Chamadiya099d2a81d70a
diff --git a/.gitignore b/.gitignore
index 02eef4b0f..a43db9727 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,6 +83,9 @@ next-env.d.ts
.repowise-workspace/
.repowise-workspace.yaml
+# Benchmark output (large-repo scale runs) — keep the dir, ignore results
+bench/results/
+
# Database
*.db
*.db-shm
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd.py b/packages/cli/src/repowise/cli/commands/init_cmd.py
index dc166c4f2..043e9d17c 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd.py
@@ -948,6 +948,17 @@ def _workspace_init(
default=False,
help="Index files, git history, graph, and dead code — skip LLM page generation.",
)
+ "--mode",
+ "run_mode",
+ type=click.Choice(["standard", "fast"]),
+ default="standard",
+ help=(
+ "Pipeline depth. 'fast' indexes graph + essential git only (no per-file "
+ "blame, no co-change, no LLM docs) for a quick first pass on very large "
+ "repos; backfill the rest later. Default: standard."
+ ),
+)
@click.option(
"--exclude",
"-x",
@@ -1025,6 +1036,7 @@ def init_command(
reasoning: str | None,
test_run: bool,
index_only: bool,
+ run_mode: str,
exclude: tuple[str, ...],
commit_limit: int | None,
follow_renames: bool,
@@ -1038,7 +1050,13 @@ def init_command(
PATH defaults to the current directory.
Use --index-only to run ingestion (AST, graph, git, dead code) without LLM generation.
+ Use --mode fast for a quick graph + essential-git index of a very large repo.
"""
+ # --mode fast is a graph + essential-git index with no LLM work, so it
+ # implies index-only on the CLI side; the orchestrator mode below switches
+ # the git tier to ESSENTIAL.
+ if run_mode == "fast":
+ index_only = True
from repowise.cli.ui import (
BRAND,
RichProgressCallback,
@@ -1047,6 +1065,7 @@ def init_command(
build_contextual_next_steps,
format_elapsed,
interactive_advanced_config,
+ interactive_fast_mode_offer,
interactive_mode_select,
interactive_provider_select,
load_dotenv,
@@ -1055,6 +1074,7 @@ def init_command(
print_phase_header,
print_scan_summary,
quick_repo_scan,
+ should_offer_fast_mode,
)
start = time.monotonic()
@@ -1130,6 +1150,10 @@ def init_command(
# ---- Interactive mode (TTY, no explicit flags) ----
is_interactive = sys.stdin.isatty() and provider_name is None and not index_only
+ # Tiered doc generation cap (set in advanced mode); None = every selected
+ # file page is a full-LLM tier-1 page (unchanged behaviour).
+ tier1_top_n: int | None = None
+
# Pre-scan for interactive mode — fast stats to inform choices
scan_info = None
if is_interactive:
@@ -1141,9 +1165,18 @@ def init_command(
if mode == "index_only":
index_only = True
+ # On a large repo, an index-only run is exactly the case where the
+ # fast tier (essential git, no blame/co-change) pays off — offer it,
+ # defaulting to yes since docs are already opted out.
+ if (
+ run_mode != "fast"
+ and should_offer_fast_mode(scan_info)
+ and interactive_fast_mode_offer(console, scan_info, default_fast=True)
+ ):
+ run_mode = "fast"
elif mode == "advanced":
provider_name, model = interactive_provider_select(console, model, repo_path=repo_path)
- adv = interactive_advanced_config(console, scan=scan_info)
+ adv = interactive_advanced_config(console, scan=scan_info, allow_fast=True)
commit_limit = adv["commit_limit"]
follow_renames = adv["follow_renames"]
skip_tests = adv["skip_tests"]
@@ -1154,8 +1187,21 @@ def init_command(
test_run = adv["test_run"]
embedder_name = adv.get("embedder") or embedder_name
include_submodules = adv.get("include_submodules", include_submodules)
+ run_mode = adv.get("run_mode", run_mode)
+ tier1_top_n = adv.get("tier1_top_n")
+ if run_mode == "fast":
+ index_only = True
else:
provider_name, model = interactive_provider_select(console, model, repo_path=repo_path)
+ # Full mode picked, but on a large repo offer the quick path too.
+ # Default no here — the user explicitly asked for docs.
+ if (
+ run_mode != "fast"
+ and should_offer_fast_mode(scan_info)
+ and interactive_fast_mode_offer(console, scan_info, default_fast=False)
+ ):
+ run_mode = "fast"
+ index_only = True
editor_options = resolve_editor_setup_options(
console,
@@ -1267,6 +1313,11 @@ def init_command(
llm_client = provider if not index_only else decision_provider
from repowise.core.pipeline import PhaseTimingRecorder, run_pipeline
+ from repowise.core.pipeline.modes import OrchestratorMode
+
+ orchestrator_mode = (
+ OrchestratorMode.FAST if run_mode == "fast" else OrchestratorMode.STANDARD
+ )
with Progress(
SpinnerColumn(),
@@ -1298,6 +1349,7 @@ def init_command(
llm_client=llm_client,
concurrency=concurrency,
test_run=test_run,
+ mode=orchestrator_mode,
progress=callback,
)
)
@@ -1383,6 +1435,7 @@ def init_command(
language=language,
reasoning=resolved_reasoning,
enable_onboarding=onboarding,
+ tier1_top_n=tier1_top_n,
)
if sys.stdin.isatty() and coverage_pct is None and not yes:
options = compute_coverage_options(
@@ -1792,6 +1845,7 @@ def init_command(
next_steps = build_contextual_next_steps(
index_only=True,
+ fast_mode=(run_mode == "fast"),
dead_unreachable=_dc_unreachable,
dead_unused=_dc_unused,
hotspot_count=_hotspot_count_final,
diff --git a/packages/cli/src/repowise/cli/ui.py b/packages/cli/src/repowise/cli/ui.py
index 01f228294..f1505795a 100644
--- a/packages/cli/src/repowise/cli/ui.py
+++ b/packages/cli/src/repowise/cli/ui.py
@@ -406,6 +406,52 @@ def _ensure_gitignored(repo_path: Path) -> None:
# ---------------------------------------------------------------------------
+# A repo at or above this many files is "large" — large enough that a quick
+# fast-mode first index (graph + essential git, no LLM docs) is worth offering.
+LARGE_REPO_FILE_THRESHOLD = 5000
+
+
+def should_offer_fast_mode(scan: RepoScanInfo | None) -> bool:
+ """Whether to surface the fast-mode offer for this repo.
+
+ Fast mode only makes sense on large repos; small repos run full in seconds
+ so the offer would just be noise.
+ """
+ return scan is not None and scan.total_files > LARGE_REPO_FILE_THRESHOLD
+
+
+def interactive_fast_mode_offer(
+ console: Console,
+ scan: RepoScanInfo | None,
+ *,
+ default_fast: bool,
+) -> bool:
+ """Offer fast mode after a large repo is detected. Returns True to use it.
+
+ Shown only when :func:`should_offer_fast_mode` is true. Fast mode is a quick
+ first index (dependency graph + essential git history, metrics in SQL) with
+ no per-file blame, no co-change walk, and no LLM docs — backfillable later.
+ """
+ n = scan.total_files if scan else 0
+ body = Text()
+ body.append(" Large repository detected — ", style="bold")
+ body.append(f"{n:,} files.\n\n", style=BRAND_STYLE)
+ body.append(" Fast mode runs a quick first index:\n", style="bold")
+ body.append(" • dependency graph + essential git history\n")
+ body.append(" • graph metrics materialized to SQL\n")
+ body.append(" • no per-file blame, no co-change walk, no LLM docs\n\n")
+ body.append(" You can backfill full git history and generate docs later.\n", style="dim")
+ console.print(
+ Panel(
+ body,
+ title="[bold]Fast first index?[/bold]",
+ border_style=BRAND,
+ padding=(1, 2),
+ )
+ )
+ return click.confirm(" Use fast mode?", default=default_fast)
+
+
def interactive_mode_select(console: Console) -> str:
"""Let the user choose full / index-only / advanced.
@@ -616,6 +662,8 @@ def _prompt_api_key(
def interactive_advanced_config(
console: Console,
scan: RepoScanInfo | None = None,
+ *,
+ allow_fast: bool = False,
) -> dict[str, Any]:
"""Prompt for advanced init options, grouped into logical sections.
@@ -667,6 +715,26 @@ def interactive_advanced_config(
else:
result["include_submodules"] = False
+ # ── Run mode (large-repo scale) ───────────────────────────────────────
+ # Fast mode = quick graph + essential-git index, no LLM docs. Suggested
+ # by default on large repos; off otherwise. Only offered for single-repo
+ # init (allow_fast); the workspace path leaves this untouched.
+ is_large = bool(scan and scan.total_files > LARGE_REPO_FILE_THRESHOLD)
+ if allow_fast:
+ console.print()
+ console.print(f" [{BRAND}]Run mode[/]")
+ console.print(
+ " [dim]standard = full depth · fast = quick graph + essential git, "
+ "no LLM docs[/dim]"
+ )
+ result["run_mode"] = click.prompt(
+ " Run mode",
+ default="fast" if is_large else "standard",
+ type=click.Choice(["standard", "fast"]),
+ )
+ else:
+ result["run_mode"] = "standard"
+
# ── Exclude Patterns ──────────────────────────────────────────────────
console.print()
console.print(f" [{BRAND}]Exclude Patterns[/]")
@@ -766,6 +834,22 @@ def interactive_advanced_config(
default=False,
)
+ # Tiered doc generation: cap the number of full-LLM (tier-1) file pages on
+ # large repos. The long tail is rendered from a deterministic template +
+ # embedded for search (no LLM). 0 = no cap (every selected page is tier-1).
+ # Only meaningful when docs actually generate (standard mode).
+ result["tier1_top_n"] = None
+ if allow_fast and result["run_mode"] == "standard":
+ tier_default = 300 if is_large else 0
+ console.print()
+ console.print(" [dim]Tiered docs: cap full-LLM file pages; rest are template-only.[/dim]")
+ tier_val = click.prompt(
+ " Full-LLM file-page cap (tier-1, 0 = no cap)",
+ default=tier_default,
+ type=int,
+ )
+ result["tier1_top_n"] = tier_val if tier_val > 0 else None
+
# ── Summary ───────────────────────────────────────────────────────────
console.print()
summary = Table(box=None, padding=(0, 2), show_header=False)
@@ -780,6 +864,10 @@ def interactive_advanced_config(
summary.add_row("Concurrency", str(result["concurrency"]))
summary.add_row("Reasoning", result["reasoning"])
summary.add_row("Embedder", result["embedder"])
+ if allow_fast:
+ summary.add_row("Run mode", result["run_mode"])
+ if result.get("tier1_top_n"):
+ summary.add_row("Full-LLM page cap", str(result["tier1_top_n"]))
if patterns:
if len(patterns) <= 5:
summary.add_row("Exclude", ", ".join(patterns))
@@ -939,16 +1027,26 @@ def build_completion_panel(
def build_contextual_next_steps(
*,
index_only: bool,
+ fast_mode: bool = False,
dead_unreachable: int = 0,
dead_unused: int = 0,
hotspot_count: int = 0,
decision_count: int = 0,
top_hotspot: str = "",
) -> list[tuple[str, str]]:
- """Build next-step suggestions based on what the analysis actually found."""
+ """Build next-step suggestions based on what the analysis actually found.
+
+ When *fast_mode* is set, the index used the essential git tier and skipped
+ LLM docs, so the suggestions lead with how to upgrade to the full result.
+ """
steps: list[tuple[str, str]] = []
- if index_only:
+ if fast_mode:
+ # Fast index: graph + essential git, no docs. Tell them how to get the
+ # full thing — complete git history + generated wiki pages.
+ steps.append(("repowise init", "run full mode: complete git history + generate docs"))
+ steps.append(("repowise mcp .", "start MCP server for AI assistants now"))
+ elif index_only:
steps.append(("repowise mcp .", "start MCP server for AI assistants"))
steps.append(("repowise init --provider gemini", "generate full documentation"))
else:
diff --git a/packages/core/alembic/versions/0021_graph_metrics.py b/packages/core/alembic/versions/0021_graph_metrics.py
new file mode 100644
index 000000000..52b650118
--- /dev/null
+++ b/packages/core/alembic/versions/0021_graph_metrics.py
@@ -0,0 +1,57 @@
+"""Add graph_metrics table — materialized file-level graph metrics snapshot.
+
+Lets large-repo metric reads (PageRank / betweenness / community / in-out
+degree) be served from SQL instead of recomputing the NetworkX kernels.
+
+Revision ID: 0021
+Revises: 0020
+Create Date: 2026-05-23
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0021"
+down_revision: str | None = "0020"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "graph_metrics",
+ sa.Column("id", sa.String(32), primary_key=True),
+ sa.Column(
+ "repository_id",
+ sa.String(32),
+ sa.ForeignKey("repositories.id", ondelete="CASCADE"),
+ nullable=False,
+ ),
+ sa.Column("node_id", sa.Text, nullable=False),
+ sa.Column("pagerank", sa.Float, nullable=False, server_default="0"),
+ sa.Column("betweenness", sa.Float, nullable=False, server_default="0"),
+ sa.Column("community_id", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("in_degree", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("out_degree", sa.Integer, nullable=False, server_default="0"),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ nullable=False,
+ server_default=sa.func.now(),
+ ),
+ sa.UniqueConstraint("repository_id", "node_id", name="uq_graph_metric"),
+ )
+ op.create_index(
+ "ix_graph_metrics_repository_id",
+ "graph_metrics",
+ ["repository_id"],
+ )
+
+
+def downgrade() -> None:
+ op.drop_index("ix_graph_metrics_repository_id", table_name="graph_metrics")
+ op.drop_table("graph_metrics")
diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.py
index 7372f7964..f5a7a06e2 100644
--- a/packages/core/src/repowise/core/generation/models.py
+++ b/packages/core/src/repowise/core/generation/models.py
@@ -133,6 +133,15 @@ class GenerationConfig:
jobs_dir: str = ".repowise/jobs"
large_file_source_pct: float = 0.4 # use structural summary when source tokens > budget * this
language: str = "en"
+ # ---- Tiered doc generation (large-repo scale) ---------------------
+ # Caps the number of file pages that receive full LLM generation.
+ # The top ``tier1_top_n`` selected file pages by PageRank are
+ # generated by the LLM (tier-1, the existing path); the remaining
+ # selected file pages are rendered from a deterministic Jinja
+ # template and embedded for search — with no LLM call (tier-2).
+ # ``None`` (default) preserves the prior behaviour exactly: every
+ # selected file page is a full tier-1 LLM page.
+ tier1_top_n: int | None = None
def __post_init__(self) -> None:
if self.embed_concurrency is None:
@@ -186,7 +195,7 @@ class GeneratedPage:
confidence: float = 1.0
freshness_status: str = "fresh" # FreshnessStatus literal
metadata: dict[str, object] = field(default_factory=dict)
- # 1–3 sentence purpose blurb extracted from the rendered content. Used by
+ # 1-3 sentence purpose blurb extracted from the rendered content. Used by
# MCP get_context as the default narrative payload (content is gated behind
# include=["full_doc"]).
summary: str = ""
diff --git a/packages/core/src/repowise/core/generation/page_generator.py b/packages/core/src/repowise/core/generation/page_generator.py
deleted file mode 100644
index 7007282f2..000000000
--- a/packages/core/src/repowise/core/generation/page_generator.py
+++ /dev/null
@@ -1,1685 +0,0 @@
-"""Page generator — converts context dataclasses into GeneratedPage objects.
-
-PageGenerator is the main orchestration layer. It:
- 1. Calls ContextAssembler to build template context from ingestion data.
- 2. Renders the Jinja2 user-prompt template.
- 3. Calls the provider with the rendered prompt + system prompt constant.
- 4. Wraps the response in a GeneratedPage.
- 5. Manages concurrency (asyncio.Semaphore) and prompt caching (SHA256).
-
-System prompts are module-level constants — the same string per page type on
-every call. This enables Anthropic server-side prefix caching.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import hashlib
-import re
-import uuid
-from collections.abc import Callable
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from pathlib import Path
-from typing import Any
-
-import jinja2
-import structlog
-
-from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
-from repowise.core.ingestion.models import ParsedFile, RepoStructure
-from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse
-
-from . import onboarding as _onboarding
-from .context_assembler import ContextAssembler, FilePageContext
-from .models import (
- GENERATION_LEVELS,
- GeneratedPage,
- GenerationConfig,
- compute_page_id,
- compute_source_hash,
-)
-
-log = structlog.get_logger(__name__)
-
-
-@dataclass(frozen=True)
-class PriorPage:
- """Snapshot of a previously-generated page used for cross-run reuse.
-
- Lives in :class:`PageGenerator` keyed by ``page_id``. When the freshly
- rendered prompt produces a matching ``source_hash`` under the same
- ``model_name``, the LLM call is skipped and ``content`` is reused.
- """
-
- source_hash: str
- model_name: str
- content: str
- input_tokens: int = 0
- output_tokens: int = 0
- cached_tokens: int = 0
-
-_LANGUAGE_NAMES = {
- "en": "English",
- "ru": "Russian",
- "es": "Spanish",
- "fr": "French",
- "de": "German",
- "zh": "Chinese",
- "ja": "Japanese",
- "ko": "Korean",
- "it": "Italian",
- "pt": "Portuguese",
- "nl": "Dutch",
- "pl": "Polish",
- "tr": "Turkish",
- "ar": "Arabic",
- "hi": "Hindi",
-}
-
-# ---------------------------------------------------------------------------
-# System prompts — one per page type (constant strings for prefix caching)
-# ---------------------------------------------------------------------------
-
-SYSTEM_PROMPTS: dict[str, str] = {
- "file_page": (
- "You are repowise, an expert technical documentation generator. "
- "Your task is to produce comprehensive, accurate wiki pages from source code. "
- "Output markdown only. Do not include preamble or apologies. "
- "\n"
- "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "
- "state this file's job in one line. Use concrete architectural vocabulary "
- "(orchestrator, entry point, parser, adapter, dispatcher, factory, resolver, "
- "persistence layer, indexer, analyzer, etc.) and name what it produces or "
- "consumes. This sentence is what semantic search and grep both anchor on, "
- "so role keywords must appear at the very front — not buried in paragraph 3. "
- "Bad: 'This file contains the X class which is used by Y.' "
- "Good: 'X is the orchestrator for the indexing pipeline — it sequences "
- "traversal, parsing, graph analysis, git enrichment, and persistence.' "
- "If the user prompt lists Architectural signals (entry point, high PageRank, "
- "bridge), weave them into that first sentence rather than restating them. "
- "Required sections: ## Overview, ## Public API, ## Dependencies, ## Usage Notes."
- ),
- "symbol_spotlight": (
- "You are repowise, an expert technical documentation generator. "
- "Write a detailed spotlight page for a single code symbol. "
- "Output markdown only. "
- "Required sections: ## Purpose, ## Signature, ## Parameters, ## Returns, ## Example Usage."
- ),
- "module_page": (
- "You are repowise, an expert technical documentation generator. "
- "Write a module-level overview page summarising all files in the module. "
- "Output markdown only. "
- "\n"
- "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "
- "state the module's job in the larger system. Use architectural vocabulary "
- "(ingestion subsystem, generation pipeline, persistence layer, transport "
- "adapter, etc.) and name the inputs it consumes and the outputs it produces. "
- "Bad: 'The X module contains 15 files responsible for...' "
- "Good: 'The ingestion module is the entry stage of repowise's indexing "
- "pipeline — it traverses a repository, parses files into ASTs, extracts "
- "symbols, and yields ParsedFile objects for downstream analysis.' "
- "Required sections: ## Overview, ## Public API Summary, ## Architecture Notes."
- ),
- "scc_page": (
- "You are repowise, an expert technical documentation generator. "
- "Document this circular dependency cycle and provide actionable refactoring advice. "
- "Output markdown only. "
- "Required sections: ## Cycle Description, ## Files Involved, ## Why This Exists, "
- "## Refactoring Suggestions."
- ),
- "repo_overview": (
- "You are repowise, an expert technical documentation generator. "
- "Write a high-level repository overview suitable for onboarding new developers. "
- "Output markdown only. "
- "\n"
- "CRITICAL — lead-with-purpose rule: the FIRST sentence under ## Project "
- "Summary must answer 'what does this repository do, end-to-end?' in one "
- "concrete sentence. Name the inputs, the pipeline, and the outputs in "
- "architectural vocabulary. "
- "Bad: 'This repository implements a documentation tool with 15 packages.' "
- "Good: 'Repowise is a codebase documentation engine: it indexes a repository "
- "by traversing files, parsing code into ASTs, analyzing dependencies, and "
- "generating LLM-synthesised wiki pages served via MCP and a web UI.' "
- "Required sections: ## Project Summary, ## Technology Stack, ## Entry Points, ## Architecture."
- ),
- "architecture_diagram": (
- "You are repowise, an expert technical documentation generator. "
- "Generate an architecture overview with a Mermaid diagram. "
- "You MUST include a fenced mermaid block with graph TD showing key dependencies. "
- "Output markdown only."
- ),
- "api_contract": (
- "You are repowise, an expert technical documentation generator. "
- "Document this API contract file for developers integrating with the service. "
- "Output markdown only. "
- "Required sections: ## Overview, ## Endpoints, ## Schemas, ## Authentication, ## Examples."
- ),
- "infra_page": (
- "You are repowise, an expert technical documentation generator. "
- "Document this infrastructure file for DevOps and platform engineers. "
- "Output markdown only. "
- "Required sections: ## Purpose, ## Key Targets/Stages, ## Configuration, ## Operational Notes."
- ),
- "onboarding": (
- "You are repowise, an expert technical documentation generator producing "
- "a single page in a curated Onboarding collection that a new contributor "
- "or LLM agent reads first. "
- "Write concise, navigable prose grounded in the structured signals supplied. "
- "Do not invent file paths, symbol names, or rationale that is not in the context. "
- "Output markdown only — follow the exact section structure the user prompt prescribes."
- ),
-}
-
-_INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()
-_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})
-_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()
-
-
-def _now_iso() -> str:
- """Return current UTC time as ISO-8601 string."""
- return datetime.now(UTC).isoformat()
-
-
-class PageGenerator:
- """Generate wiki pages by rendering prompts and calling an LLM provider.
-
- Args:
- provider: Any BaseProvider implementation.
- assembler: ContextAssembler instance.
- config: GenerationConfig controlling budget, concurrency, caching.
- jinja_env: Optional Jinja2 Environment (defaults to FileSystemLoader
- pointing at the templates/ directory next to this file).
- """
-
- def __init__(
- self,
- provider: BaseProvider,
- assembler: ContextAssembler,
- config: GenerationConfig,
- jinja_env: jinja2.Environment | None = None,
- vector_store: Any | None = None,
- language: str = "en",
- prior_pages: dict[str, "PriorPage"] | None = None,
- ) -> None:
- self._provider = provider
- self._assembler = assembler
- self._config = config
- self._vector_store = vector_store
- self._language = language
- self._cache: dict[str, GeneratedResponse] = {}
- # Map of page_id → PriorPage from previous generation runs. When the
- # rendered prompt's source_hash matches the prior page's hash AND the
- # model is the same, the LLM call is skipped and the prior content is
- # reused. Wired by the orchestrator from the persisted wiki_pages
- # table.
- self._prior_pages: dict[str, PriorPage] = prior_pages or {}
- self._reuse_count: int = 0
-
- if jinja_env is None:
- templates_dir = Path(__file__).parent / "templates"
- loader = jinja2.FileSystemLoader(str(templates_dir))
- jinja_env = jinja2.Environment(
- loader=loader,
- undefined=jinja2.StrictUndefined,
- autoescape=False,
- )
- self._jinja_env = jinja_env
-
- # ------------------------------------------------------------------
- # Per-type generation methods
- # ------------------------------------------------------------------
-
- async def generate_file_page(
- self,
- parsed: ParsedFile,
- graph: Any,
- pagerank: dict[str, float],
- betweenness: dict[str, float],
- community: dict[str, int],
- source_bytes: bytes,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_file_page(
- parsed, graph, pagerank, betweenness, community, source_bytes
- )
- user_prompt = self._render("file_page.j2", ctx=ctx)
- response = await self._call_provider(
- "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
- )
- return self._build_generated_page(
- "file_page",
- parsed.file_info.path,
- f"File: {parsed.file_info.path}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["file_page"],
- )
-
- async def generate_symbol_spotlight(
- self,
- symbol: Any,
- parsed: ParsedFile,
- pagerank: dict[str, float],
- graph: Any,
- source_map: dict[str, bytes] | None = None,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_symbol_spotlight(
- symbol,
- parsed,
- pagerank,
- graph,
- source_bytes=(source_map or {}).get(parsed.file_info.path, b""),
- )
- user_prompt = self._render("symbol_spotlight.j2", ctx=ctx)
- target = f"{parsed.file_info.path}::{symbol.name}"
- response = await self._call_provider(
- "symbol_spotlight", user_prompt, str(uuid.uuid4()), target_path=target
- )
- return self._build_generated_page(
- "symbol_spotlight",
- f"{parsed.file_info.path}::{symbol.name}",
- f"Symbol: {symbol.qualified_name}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["symbol_spotlight"],
- )
-
- async def generate_module_page(
- self,
- module_path: str,
- language: str,
- file_contexts: list[FilePageContext],
- graph: Any,
- git_meta_map: dict[str, dict] | None = None,
- page_summaries: dict[str, str] | None = None,
- decision_records: list[dict] | None = None,
- dead_code_findings: list[dict] | None = None,
- external_systems: list[dict] | None = None,
- community_label: str | None = None,
- community_cohesion: float | None = None,
- target_path: str | None = None,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_module_page(
- module_path,
- language,
- file_contexts,
- graph,
- page_summaries=page_summaries,
- git_meta_map=git_meta_map,
- decision_records=decision_records,
- dead_code_findings=dead_code_findings,
- external_systems=external_systems,
- community_label=community_label,
- community_cohesion=community_cohesion,
- )
- module_git_summary = None
- if git_meta_map:
- from collections import Counter
-
- file_paths = [fc.file_path for fc in file_contexts]
- metas = [git_meta_map[f] for f in file_paths if f in git_meta_map]
- if metas:
- owner_counts = Counter(
- m.get("primary_owner_name") for m in metas if m.get("primary_owner_name")
- )
- most_active = max(metas, key=lambda m: m.get("commit_count_90d", 0))
- module_git_summary = {
- "top_owners": [
- {"name": n, "file_count": c} for n, c in owner_counts.most_common(3)
- ],
- "most_active_file": most_active.get("file_path", ""),
- "most_active_commits_90d": most_active.get("commit_count_90d", 0),
- }
- user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)
- page_target = target_path or module_path
- response = await self._call_provider(
- "module_page", user_prompt, str(uuid.uuid4()), target_path=page_target
- )
- return self._build_generated_page(
- "module_page",
- page_target,
- f"Module: {module_path}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["module_page"],
- )
-
- async def generate_scc_page(
- self,
- scc_id: str,
- scc_files: list[str],
- file_contexts: list[FilePageContext],
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_scc_page(scc_id, scc_files, file_contexts)
- user_prompt = self._render("scc_page.j2", ctx=ctx)
- response = await self._call_provider(
- "scc_page", user_prompt, str(uuid.uuid4()), target_path=scc_id
- )
- return self._build_generated_page(
- "scc_page",
- scc_id,
- f"Circular Dependency: {scc_id}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["scc_page"],
- )
-
- async def generate_repo_overview(
- self,
- repo_structure: RepoStructure,
- pagerank: dict[str, float],
- sccs: list[Any],
- community: dict[str, int],
- git_meta_map: dict[str, dict] | None = None,
- graph_builder: Any | None = None,
- repo_name: str | None = None,
- external_systems: list[dict] | None = None,
- decision_records: list[dict] | None = None,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_repo_overview(
- repo_structure,
- pagerank,
- sccs,
- community,
- graph_builder=graph_builder,
- external_systems=external_systems,
- decision_records=decision_records,
- )
- repo_git_summary = None
- if git_meta_map:
- metas = list(git_meta_map.values())
- top_churn = sorted(metas, key=lambda m: m.get("commit_count_90d", 0), reverse=True)[:3]
- oldest = min(
- (m for m in metas if m.get("first_commit_at")),
- key=lambda m: m["first_commit_at"],
- default=None,
- )
- repo_git_summary = {
- "hotspot_count": sum(1 for m in metas if m.get("is_hotspot")),
- "stable_count": sum(1 for m in metas if m.get("is_stable")),
- "top_churn_files": [m.get("file_path", "") for m in top_churn],
- "oldest_file": oldest.get("file_path", "") if oldest else "",
- "oldest_file_age_days": oldest.get("age_days", 0) if oldest else 0,
- }
- if not repo_name:
- repo_name = getattr(repo_structure, "name", None) or "repo"
- user_prompt = self._render("repo_overview.j2", ctx=ctx, repo_git_summary=repo_git_summary)
- response = await self._call_provider(
- "repo_overview", user_prompt, str(uuid.uuid4()), target_path=repo_name
- )
- return self._build_generated_page(
- "repo_overview",
- repo_name,
- f"Repository Overview: {repo_name}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["repo_overview"],
- )
-
- async def generate_architecture_diagram(
- self,
- graph: Any,
- pagerank: dict[str, float],
- community: dict[str, int],
- sccs: list[Any],
- repo_name: str,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_architecture_diagram(
- graph, pagerank, community, sccs, repo_name
- )
- user_prompt = self._render("architecture_diagram.j2", ctx=ctx)
- response = await self._call_provider(
- "architecture_diagram", user_prompt, str(uuid.uuid4()), target_path=repo_name
- )
- return self._build_generated_page(
- "architecture_diagram",
- repo_name,
- f"Architecture Diagram: {repo_name}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["architecture_diagram"],
- )
-
- async def generate_api_contract(
- self,
- parsed: ParsedFile,
- source_bytes: bytes,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_api_contract(parsed, source_bytes)
- user_prompt = self._render("api_contract.j2", ctx=ctx)
- response = await self._call_provider(
- "api_contract", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
- )
- return self._build_generated_page(
- "api_contract",
- parsed.file_info.path,
- f"API Contract: {parsed.file_info.path}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["api_contract"],
- )
-
- async def generate_onboarding_page(
- self,
- spec: "_onboarding.SubkindSpec",
- signals: "_onboarding.OnboardingSignals",
- ) -> GeneratedPage | None:
- """Generate one onboarding page from a registered subkind spec.
-
- Returns ``None`` when the subkind's gate fails (``build_context``
- returned ``None``) — the slot is silently skipped for this repo.
- """
- ctx = spec.build_context(signals)
- if ctx is None:
- log.debug("onboarding.gate_skipped", slot=spec.slot)
- return None
-
- template_name = f"onboarding/{spec.template}"
- user_prompt = self._render(template_name, ctx=ctx, slot=spec.slot)
- target = _onboarding.target_path(spec.slot)
- response = await self._call_provider(
- "onboarding", user_prompt, str(uuid.uuid4()), target_path=target
- )
- page = self._build_generated_page(
- "onboarding",
- target,
- spec.title,
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["onboarding"],
- )
- # Subkind discriminator lives in metadata; page_type alone is shared
- # across all six generated onboarding slots.
- page.metadata["subkind"] = spec.slot
- page.metadata["onboarding_slot"] = spec.slot
- return page
-
- @staticmethod
- def _tag_promoted_pages(pages: list[GeneratedPage]) -> None:
- """Tag repo_overview / architecture_diagram pages with their slot.
-
- Mutates each matching page's ``metadata["onboarding_slot"]`` so the
- UI groups them into the Onboarding folder without changing their
- underlying ``page_type``. Idempotent and tolerant of missing pages.
- """
- for page in pages:
- slot = _onboarding.PROMOTED_SLOTS.get(page.page_type)
- if slot is not None:
- page.metadata["onboarding_slot"] = slot
-
- async def generate_infra_page(
- self,
- parsed: ParsedFile,
- source_bytes: bytes,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_infra_page(parsed, source_bytes)
- user_prompt = self._render("infra_page.j2", ctx=ctx)
- response = await self._call_provider(
- "infra_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
- )
- return self._build_generated_page(
- "infra_page",
- parsed.file_info.path,
- f"Infrastructure: {parsed.file_info.path}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["infra_page"],
- )
-
- # ------------------------------------------------------------------
- # generate_all — orchestration
- # ------------------------------------------------------------------
-
- async def generate_all(
- self,
- parsed_files: list[ParsedFile],
- source_map: dict[str, bytes],
- graph_builder: Any, # GraphBuilder
- repo_structure: RepoStructure,
- repo_name: str,
- job_system: Any | None = None, # JobSystem | None
- on_page_done: Callable[[str], None] | None = None,
- on_total_known: Callable[[int], None] | None = None,
- on_subphase: Callable[[str, int | None], None] | None = None,
- git_meta_map: dict[str, dict] | None = None,
- resume: bool = False,
- repo_path: Path | str | None = None,
- dead_code_report: Any | None = None,
- decision_report: Any | None = None,
- external_systems: list[dict] | None = None,
- ) -> list[GeneratedPage]:
- """Generate all wiki pages for a repository.
-
- Runs generation in 8 ordered levels. Each level's pages are generated
- concurrently (up to config.max_concurrency). Failures within a level
- are logged but do not abort the remaining levels.
-
- Args:
- parsed_files: All ParsedFile objects from the ingestion pipeline.
- source_map: Raw file bytes keyed by relative path.
- graph_builder: Finalized GraphBuilder (build() already called).
- repo_structure: High-level repo metadata.
- repo_name: Human-readable repository name.
- job_system: Optional JobSystem for checkpoint persistence.
- on_page_done: Optional callback per completed page.
- git_meta_map: Optional dict of git metadata by file path.
-
- Returns:
- List of GeneratedPage objects in level order.
- """
- graph = graph_builder.graph()
- pagerank = graph_builder.pagerank()
- betweenness = graph_builder.betweenness_centrality()
- community = graph_builder.community_detection()
- sccs = graph_builder.strongly_connected_components()
-
- # ---- Build per-file lookup maps from phase-2 signals ----
- dead_code_by_file: dict[str, list[dict]] = {}
- if dead_code_report is not None and getattr(dead_code_report, "findings", None):
- for f in dead_code_report.findings:
- dead_code_by_file.setdefault(f.file_path, []).append(
- {
- "symbol_name": f.symbol_name,
- "symbol_kind": f.symbol_kind,
- "kind": str(f.kind),
- "reason": f.reason,
- "confidence": f.confidence,
- "safe_to_delete": f.safe_to_delete,
- }
- )
-
- decisions_by_file: dict[str, list[dict]] = {}
- decisions_all: list[dict] = []
- if decision_report is not None and getattr(decision_report, "decisions", None):
- for d in decision_report.decisions:
- payload = {
- "title": d.title,
- "decision": d.decision,
- "rationale": d.rationale,
- "source": d.source,
- "confidence": d.confidence,
- "evidence_file": d.evidence_file,
- }
- decisions_all.append(payload)
- for fp in d.affected_files or []:
- decisions_by_file.setdefault(fp, []).append(payload)
-
- external_systems = external_systems or []
-
- all_pages: list[GeneratedPage] = []
- semaphore = asyncio.Semaphore(self._config.max_concurrency)
- embed_semaphore = asyncio.Semaphore(self._config.embed_concurrency or 1)
- # Summaries of completed pages: target_path → brief summary text (for dep context)
- completed_page_summaries: dict[str, str] = {}
-
- def _extract_summary(content: str) -> str:
- if "## Overview" in content:
- start = content.index("## Overview") + len("## Overview")
- end = content.find("\n##", start)
- return content[start : end if end > 0 else start + 1600].strip()[:400]
- return content[:400]
-
- # Determine already-completed pages (for resume support)
- completed_ids: set[str] = set()
- job_id: str | None = None
- if job_system is not None:
- repo_path_str = (
- str(Path(repo_path).resolve())
- if repo_path
- else str(getattr(repo_structure, "root_path", "."))
- )
- # On resume, query the vector store directly — it is the ground truth
- if resume and self._vector_store is not None:
- completed_ids = await self._vector_store.list_page_ids()
- if completed_ids:
- log.info(
- "Resuming generation from vector store",
- already_completed=len(completed_ids),
- )
- job_id = job_system.create_job(
- repo_path_str,
- self._config,
- self._provider.provider_name,
- self._provider.model_name,
- )
-
- async def _embed_async(page: GeneratedPage) -> None:
- """Embed a finished page into the vector store. Safe to fire-and-forget.
-
- Errors are swallowed at debug level — embedding is a RAG
- enhancement, not load-bearing for the page itself.
- """
- try:
- summary = _extract_summary(page.content)
- async with embed_semaphore:
- await self._vector_store.embed_and_upsert(
- page.page_id,
- page.content,
- {
- "page_type": page.page_type,
- "target_path": page.target_path,
- "content": page.content[:600],
- "summary": summary,
- },
- )
- except Exception as e:
- log.debug("rag.embed_failed", page_id=page.page_id, error=str(e))
-
- async def run_level(named_coros: list[tuple[str, Any]], level: int) -> list[GeneratedPage]:
- if job_system is not None and job_id is not None:
- job_system.update_level(job_id, level)
-
- # Embed tasks spawned during this level. Drained after the
- # gather() so the next level's RAG search sees a fully-indexed
- # store, but never blocks the per-page wave.
- pending_embeds: list[asyncio.Task[None]] = []
-
- async def guarded_named(page_id: str, coro: Any) -> Any:
- try:
- async with semaphore:
- result = await coro
-
- if isinstance(result, GeneratedPage):
- # Summary capture is cheap (string ops) — keep
- # inline so the next page's context assembly sees
- # it immediately.
- completed_page_summaries[result.target_path] = _extract_summary(
- result.content
- )
- # Progress tick fires the moment the LLM call
- # returns — the user sees the page as done before
- # the embed completes.
- if on_page_done is not None:
- on_page_done(result.page_type)
- # Fire-and-forget the embed/upsert. Removes ~1
- # embedder round-trip from this task's critical
- # path so the LLM slot frees up immediately.
- if self._vector_store is not None:
- pending_embeds.append(
- asyncio.create_task(_embed_async(result))
- )
- return result
- except Exception as exc:
- if job_system is not None and job_id is not None:
- job_system.fail_page(job_id, page_id, str(exc))
- log.error(
- "page_generation_failed",
- page_id=page_id,
- level=level,
- error=str(exc),
- )
- return exc # return as value so gather works
-
- tasks = [guarded_named(pid, c) for pid, c in named_coros]
- results = await asyncio.gather(*tasks)
- # Drain pending embeds before declaring the level done — the
- # next level's RAG search depends on these landing in the store.
- if pending_embeds:
- await asyncio.gather(*pending_embeds, return_exceptions=True)
- pages = [r for r in results if isinstance(r, GeneratedPage)]
- if job_system is not None and job_id is not None:
- for r in pages:
- job_system.complete_page(job_id, r.page_id)
- return pages
-
- # ---- Page selection (single source of truth) ----
- # The selection subsystem scores every candidate, allocates the
- # global budget across page-type buckets, and returns an
- # allow-set. Each level's emit loop iterates only over members
- # of that allow-set — there are no bypass paths. The same
- # function is called by the cost estimator so the pre-run
- # estimate cannot drift from the actual run.
- code_files = [
- p
- for p in parsed_files
- if not p.file_info.is_api_contract
- and not _is_infra_file(p)
- and p.file_info.language in _CODE_LANGUAGES
- ]
-
- # Near-clone dedupe runs before scoring so clone losers never
- # consume scoring budget. Entry points are never dropped.
- if getattr(self._config, "dedupe_near_clones", True):
- drop_paths = _select_clone_representatives(code_files, pagerank)
- if drop_paths:
- log.info("page_selection.clone_dedupe", dropped=len(drop_paths))
- code_files = [p for p in code_files if p.file_info.path not in drop_paths]
- parsed_files_for_selection = [
- p for p in parsed_files if p.file_info.path not in drop_paths
- ]
- else:
- parsed_files_for_selection = parsed_files
- else:
- parsed_files_for_selection = parsed_files
-
- try:
- community_info_map = graph_builder.community_info() or {}
- except Exception:
- community_info_map = {}
-
- from .selection import SelectionInputs, select_pages
-
- selection = select_pages(
- SelectionInputs(
- parsed_files=parsed_files_for_selection,
- pagerank=pagerank,
- betweenness=betweenness,
- community=community,
- community_info=community_info_map,
- sccs=list(sccs),
- git_meta_map=git_meta_map,
- config=self._config,
- )
- )
-
- # Bucket allow-sets — O(1) membership checks in the level loops.
- sel_file_paths: set[str] = set(selection.file_page_paths)
- sel_symbol_keys: set[tuple[str, str]] = set(selection.symbol_spotlights)
- sel_api_paths: set[str] = set(selection.api_contract_paths)
- sel_infra_paths: set[str] = set(selection.infra_paths)
- sel_module_groups = list(selection.module_groups)
- sel_scc_groups = list(selection.scc_groups)
-
- # Sort code_files for stable level-2 ordering: selected files
- # first (so dep summaries land in the store earliest), then by
- # PageRank desc. The topo-sort below further refines this.
- code_files = sorted(
- code_files,
- key=lambda p: (
- p.file_info.path not in sel_file_paths,
- not p.file_info.is_entry_point,
- -pagerank.get(p.file_info.path, 0.0),
- ),
- )
-
- estimated_total = (
- selection.counts()["api_contract"]
- + selection.counts()["symbol_spotlight"]
- + selection.counts()["file_page"]
- + selection.counts()["scc_page"]
- + selection.counts()["module_page"]
- + int(selection.emit_repo_overview)
- + int(selection.emit_arch_diagram)
- + selection.counts()["infra_page"]
- )
- remaining_total = max(0, estimated_total - len(completed_ids))
- if on_total_known is not None:
- on_total_known(remaining_total)
- if job_system is not None and job_id is not None:
- job_system.start_job(job_id, estimated_total)
-
- # ---- Level 0: api_contract (allow-set filtered) ----
- api_files = [
- p
- for p in parsed_files
- if p.file_info.is_api_contract and p.file_info.path in sel_api_paths
- ]
- level0_coros = [
- (
- compute_page_id("api_contract", p.file_info.path),
- self.generate_api_contract(p, source_map.get(p.file_info.path, b"")),
- )
- for p in api_files
- if compute_page_id("api_contract", p.file_info.path) not in completed_ids
- ]
- # ---- Level 1: symbol_spotlight (allow-set filtered) ----
- # The selection layer already picked the top symbols by score;
- # here we just resolve them back to (Symbol, ParsedFile) pairs.
- parsed_by_path: dict[str, ParsedFile] = {
- p.file_info.path: p for p in parsed_files
- }
- top_symbols: list[tuple[Any, ParsedFile]] = []
- for file_path, sym_name in selection.symbol_spotlights:
- pf = parsed_by_path.get(file_path)
- if pf is None:
- continue
- sym = next((s for s in pf.symbols if s.name == sym_name), None)
- if sym is not None:
- top_symbols.append((sym, pf))
-
- level1_coros = [
- (
- compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}"),
- self.generate_symbol_spotlight(sym, pf, pagerank, graph, source_map=source_map),
- )
- for sym, pf in top_symbols
- if compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}")
- not in completed_ids
- ]
-
- # Levels 0 (api_contract) and 1 (symbol_spotlight) share no
- # data dependencies — both feed into nothing else upstream of
- # Level 2 — so they run in one merged batch instead of two
- # sequential barriers. ``run_level`` already bounds total
- # concurrency via ``self._config.max_concurrency`` so the merge
- # only removes idle slots, never over-saturates the provider.
- level01_pages = await run_level(level0_coros + level1_coros, 1)
- all_pages.extend(level01_pages)
-
- # ---- Level 2: file_page (significant code files only) ----
- # Context is assembled for ALL code files (module pages need it).
- # Pages are generated only for files that cross the significance bar.
- # page_summaries from level 0+1 are available here (B2).
- #
- # Topo-sort: process leaves (no internal out-edges) before roots so that
- # dependency summaries are available when assembling dependents' contexts.
- # Falls back to existing priority order if networkx is unavailable or graph
- # has cycles.
- code_file_paths = [p.file_info.path for p in code_files]
- try:
- import networkx as nx # type: ignore[import]
-
- # Build a subgraph of just the code files we are about to generate
- code_file_set = set(code_file_paths)
- dag = nx.DiGraph()
- dag.add_nodes_from(code_file_paths)
- for path_ in code_file_paths:
- if path_ in graph:
- for succ in graph.successors(path_):
- if succ in code_file_set:
- dag.add_edge(path_, succ) # path_ depends on succ
-
- if nx.is_directed_acyclic_graph(dag):
- # topological_sort yields nodes in an order where for each edge u→v,
- # u comes before v — i.e. dependents before dependencies.
- # We want leaves (dependencies) first, so reverse the order.
- topo_order = list(reversed(list(nx.topological_sort(dag))))
- else:
- # Cycle present: condense SCCs, topo-sort condensation, then expand.
- condensation = nx.condensation(dag)
- topo_order_scc = list(reversed(list(nx.topological_sort(condensation))))
- scc_members: dict[int, list[str]] = {
- n: list(condensation.nodes[n]["members"]) for n in condensation.nodes
- }
- topo_order = [node for scc_id in topo_order_scc for node in scc_members[scc_id]]
-
- # Preserve priority ordering within the topo-sort by mapping paths to
- # their original priority index.
- priority_index = {p: i for i, p in enumerate(code_file_paths)}
- topo_order = [p for p in topo_order if p in priority_index]
- # Re-sort code_files to match topo_order
- path_to_parsed = {p.file_info.path: p for p in code_files}
- code_files = [path_to_parsed[p] for p in topo_order if p in path_to_parsed]
- except Exception:
- pass # Keep existing priority order on any failure
-
- file_page_contexts: dict[str, FilePageContext] = {}
-
- # Batch-prefetch dependency summaries from the vector store in a
- # SINGLE call covering every code file's dependencies — replaces
- # the prior per-file serial loop that turned N×M awaits into a
- # measurable bottleneck on the level-2 critical path.
- if self._vector_store is not None:
- needed_deps: set[str] = set()
- for p in code_files:
- path_ = p.file_info.path
- if path_ not in graph:
- continue
- for dep in graph.successors(path_):
- if dep.startswith("external:"):
- continue
- if dep in completed_page_summaries:
- continue
- needed_deps.add(dep)
- if needed_deps:
- try:
- batch = await self._vector_store.get_page_summaries_by_paths(
- list(needed_deps)
- )
- for dep_path, payload in batch.items():
- summary = payload.get("summary") if payload else None
- if summary:
- completed_page_summaries[dep_path] = summary
- except Exception as exc:
- log.debug("rag.batch_dep_prefetch_failed", error=str(exc))
-
- level2_coros: list[tuple[str, Any]] = []
- for p in code_files:
- ctx = self._assembler.assemble_file_page(
- p,
- graph,
- pagerank,
- betweenness,
- community,
- source_map.get(p.file_info.path, b""),
- git_meta=git_meta_map.get(p.file_info.path) if git_meta_map else None,
- page_summaries=completed_page_summaries,
- dead_code_findings=dead_code_by_file.get(p.file_info.path),
- decision_records=decisions_by_file.get(p.file_info.path),
- )
- file_page_contexts[p.file_info.path] = ctx
- pid = compute_page_id("file_page", p.file_info.path)
- if p.file_info.path in sel_file_paths and pid not in completed_ids:
- level2_coros.append((pid, self._generate_file_page_from_ctx(p, ctx)))
-
- level2_pages = await run_level(level2_coros, 2)
- all_pages.extend(level2_pages)
-
- # ---- Level 3: scc_page (allow-set filtered) ----
- scc_coros: list[tuple[str, Any]] = []
- for scc_id, scc_files in sel_scc_groups:
- fc_list = [file_page_contexts[f] for f in scc_files if f in file_page_contexts]
- pid = compute_page_id("scc_page", scc_id)
- if pid not in completed_ids:
- scc_coros.append((pid, self.generate_scc_page(scc_id, scc_files, fc_list)))
- level3_pages = await run_level(scc_coros, 3)
- all_pages.extend(level3_pages)
-
- # ---- Level 4: module_page (allow-set filtered) ----
- # Module groups come from the selection layer; we only need to
- # resolve FilePageContext objects for the files we already
- # built contexts for in Level 2.
- level4_coros: list[tuple[str, Any]] = []
- for mg in sel_module_groups:
- fcs = [
- file_page_contexts[fp]
- for fp in mg.file_paths
- if fp in file_page_contexts
- ]
- if not fcs:
- continue
- page_id = compute_page_id("module_page", mg.key)
- if page_id in completed_ids:
- continue
- level4_coros.append(
- (
- page_id,
- self.generate_module_page(
- mg.display,
- mg.language,
- fcs,
- graph,
- git_meta_map=git_meta_map,
- page_summaries=completed_page_summaries,
- decision_records=decisions_all,
- dead_code_findings=[
- d for fc in fcs for d in dead_code_by_file.get(fc.file_path, [])
- ],
- external_systems=external_systems,
- community_label=mg.label,
- community_cohesion=mg.cohesion,
- target_path=mg.key,
- ),
- )
- )
- level4_pages = await run_level(level4_coros, 4)
- all_pages.extend(level4_pages)
-
- # ---- Level 6: repo_overview + architecture_diagram ----
- level6_coros: list[tuple[str, Any]] = []
- if compute_page_id("repo_overview", repo_name) not in completed_ids:
- level6_coros.append(
- (
- compute_page_id("repo_overview", repo_name),
- self.generate_repo_overview(
- repo_structure,
- pagerank,
- sccs,
- community,
- git_meta_map=git_meta_map,
- graph_builder=graph_builder,
- repo_name=repo_name,
- external_systems=external_systems,
- decision_records=decisions_all[:10],
- ),
- )
- )
- if compute_page_id("architecture_diagram", repo_name) not in completed_ids:
- level6_coros.append(
- (
- compute_page_id("architecture_diagram", repo_name),
- self.generate_architecture_diagram(graph, pagerank, community, sccs, repo_name),
- )
- )
- # ---- Level 7: infra_page (allow-set filtered) ----
- infra_files = [
- p
- for p in parsed_files
- if _is_infra_file(p) and p.file_info.path in sel_infra_paths
- ]
- level7_coros: list[tuple[str, Any]] = [
- (
- compute_page_id("infra_page", p.file_info.path),
- self.generate_infra_page(p, source_map.get(p.file_info.path, b"")),
- )
- for p in infra_files
- if compute_page_id("infra_page", p.file_info.path) not in completed_ids
- ]
-
- # ---- Level 8: onboarding (curated collection) ----
- # Each subkind defines its own gate inside build_context — slots
- # whose gates fail return None and are skipped entirely. Promoted
- # slots (project_overview / architecture_guide) are tagged onto the
- # level-6 pages and don't appear here.
- level8_coros: list[tuple[str, Any]] = []
- if getattr(self._config, "enable_onboarding", True):
- specs = _onboarding.iter_specs()
- if specs:
- if on_subphase is not None:
- try:
- on_subphase("onboarding", len(specs))
- except Exception:
- pass
- signals = _onboarding.OnboardingSignals(
- repo_name=repo_name,
- repo_structure=repo_structure,
- parsed_files=tuple(parsed_files),
- source_map=source_map,
- graph_builder=graph_builder,
- pagerank=pagerank,
- betweenness=betweenness,
- community=community,
- sccs=tuple(sccs),
- git_meta_map=git_meta_map,
- dead_code_by_file=dead_code_by_file,
- decisions_all=tuple(decisions_all),
- external_systems=tuple(external_systems),
- completed_page_summaries=dict(completed_page_summaries),
- )
- for spec in specs:
- page_id = compute_page_id(
- "onboarding", _onboarding.target_path(spec.slot)
- )
- if page_id in completed_ids:
- continue
- level8_coros.append(
- (page_id, self.generate_onboarding_page(spec, signals))
- )
-
- # Levels 6, 7, and 8 share no data dependencies with each other
- # — Level 6 needs only the graph + repo metadata; Level 7
- # documents standalone infra files; Level 8 consumes a frozen
- # ``OnboardingSignals`` snapshot. Run them in a single merged
- # batch instead of three sequential barriers.
- final_pages = await run_level(level6_coros + level7_coros + level8_coros, 8)
- # Tag promoted onboarding slots (repo_overview / architecture_diagram)
- # so the UI groups them into the Onboarding folder. No content change.
- self._tag_promoted_pages(final_pages)
- all_pages.extend(final_pages)
-
- # Post-generation: resolve backtick-quoted refs in every page's
- # markdown to other pages' ``page_id``s and stash the result in
- # ``metadata["wiki_links"]``. The reverse index lands in
- # ``metadata["backlinks"]``. Pure regex + dict lookup — no LLM
- # call, safe to run on every generation.
- try:
- from .interlinking import attach_wiki_links_and_backlinks
-
- attach_wiki_links_and_backlinks(all_pages, parsed_files)
- except Exception as exc:
- log.debug("interlinking.failed", error=str(exc))
-
- # Finalize job
- if job_system is not None and job_id is not None:
- job_system.complete_job(job_id)
-
- log.info(
- "Generation complete",
- total_pages=len(all_pages),
- provider=self._provider.provider_name,
- model=self._provider.model_name,
- )
- return all_pages
-
- # ------------------------------------------------------------------
- # Internal helpers
- # ------------------------------------------------------------------
-
- async def _generate_file_page_from_ctx(
- self,
- parsed: ParsedFile,
- ctx: FilePageContext,
- ) -> GeneratedPage:
- """Generate a file_page from a pre-assembled context (avoids double-assembly)."""
- # RAG context: query vector store for related pages (B1).
- # Gated by two short-circuits so we don't burn an embedder
- # round-trip on every page when the result wouldn't help:
- # 1. ``enable_rag_context`` config flag (off → fully skip).
- # 2. ``rag_min_store_size`` — early pages run against an empty
- # or near-empty store and the search returns nothing useful.
- if self._vector_store is not None and getattr(
- self._config, "enable_rag_context", True
- ):
- min_store_size = max(0, int(getattr(self._config, "rag_min_store_size", 10) or 0))
- store_ok = True
- if min_store_size > 0:
- try:
- current_ids = await self._vector_store.list_page_ids()
- store_ok = len(current_ids) >= min_store_size
- except Exception:
- # If the store can't be sized cheaply, fall through to
- # the search — it'll either succeed or hit the
- # existing exception path.
- store_ok = True
- if store_ok:
- query_terms = parsed.exports or [
- s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"
- ]
- if query_terms:
- try:
- results = await self._vector_store.search(
- ", ".join(query_terms[:5]), limit=3
- )
- self_id = f"file_page:{parsed.file_info.path}"
- ctx.rag_context = [
- f"[{r.page_id}]\n{r.snippet}"
- for r in results
- if r.page_id != self_id
- ]
- except Exception as e:
- log.debug(
- "rag.search_failed", path=parsed.file_info.path, error=str(e)
- )
- user_prompt = self._render("file_page.j2", ctx=ctx)
- response = await self._call_provider(
- "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
- )
- page = self._build_generated_page(
- "file_page",
- parsed.file_info.path,
- f"File: {parsed.file_info.path}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["file_page"],
- )
- # Cross-check LLM output against actual symbols
- hal_warnings = _validate_symbol_references(response.content, parsed)
- if hal_warnings:
- log.warning(
- "hallucination_check",
- path=parsed.file_info.path,
- count=len(hal_warnings),
- refs=hal_warnings[:5],
- )
- page.metadata["hallucination_warnings"] = hal_warnings
- return page
-
- async def _call_provider(
- self,
- page_type: str,
- user_prompt: str,
- request_id: str,
- target_path: str | None = None,
- ) -> GeneratedResponse:
- """Call the provider with caching, optionally prefixing a language instruction."""
- # Persistent cross-run cache: if the page exists from a prior run, was
- # produced by the same model, and the prompt's source_hash matches,
- # reuse the stored content without an LLM call.
- if self._config.cache_enabled and target_path is not None:
- page_id = compute_page_id(page_type, target_path)
- prior = self._prior_pages.get(page_id)
- if prior is not None and prior.model_name == self._provider.model_name:
- current_hash = compute_source_hash(user_prompt)
- if prior.source_hash == current_hash:
- self._reuse_count += 1
- log.debug(
- "page_cache.persistent_hit",
- page_type=page_type,
- target_path=target_path,
- )
- return GeneratedResponse(
- content=prior.content,
- input_tokens=0,
- output_tokens=0,
- cached_tokens=0,
- usage={"reused_from_prior_run": True},
- )
-
- key = self._compute_cache_key(page_type, user_prompt)
- if self._config.cache_enabled and key in self._cache:
- log.debug("Cache hit", page_type=page_type, key=key[:8])
- return self._cache[key]
-
- system_prompt = self._build_system_prompt(page_type)
-
- # The same system prompt is reused for every page of a given type, so
- # mark it as cacheable. Providers without server-side prompt caching
- # ignore the hint safely.
- cache_hints: tuple[CacheHint, ...] = (
- (CacheHint(segment="system"),) if self._config.cache_enabled else ()
- )
-
- response = await self._provider.generate(
- system_prompt,
- user_prompt,
- max_tokens=self._config.max_tokens,
- temperature=self._config.temperature,
- request_id=request_id,
- reasoning=self._config.reasoning,
- cache_hints=cache_hints,
- )
-
- if self._config.cache_enabled:
- self._cache[key] = response
-
- return response
-
- def _build_system_prompt(self, page_type: str) -> str:
- base_system = SYSTEM_PROMPTS[page_type]
- # Sanitize the configured language code: lower, strip, drop anything that isn't
- # alphanumeric or underscore. Prevents user-supplied config from injecting
- # newlines or extra instructions into the system prompt.
- raw = (self._language or "en").lower().strip()
- lang_code = "".join(ch for ch in raw if ch.isalnum() or ch == "_")
- if lang_code not in _LANGUAGE_NAMES:
- if lang_code != "en":
- log.warning("unknown_language_code", code=lang_code, fallback="en")
- lang_code = "en"
- if lang_code == "en":
- return base_system
- lang_name = _LANGUAGE_NAMES[lang_code]
- instruction = (
- f"Generate all documentation content in {lang_name}. "
- "Keep all code, file paths, and symbol names in their original form. "
- "Do not translate them.\n\n"
- )
- return instruction + base_system
-
- def _compute_cache_key(self, page_type: str, user_prompt: str) -> str:
- """Return SHA256(model + language + page_type + user_prompt) as cache key."""
- raw = f"{self._provider.model_name}:{self._language}:{page_type}:{user_prompt}"
- return hashlib.sha256(raw.encode()).hexdigest()
-
- def _build_generated_page(
- self,
- page_type: str,
- target_path: str,
- title: str,
- response: GeneratedResponse,
- source_hash: str,
- level: int,
- ) -> GeneratedPage:
- """Wrap a GeneratedResponse in a GeneratedPage."""
- now = _now_iso()
- return GeneratedPage(
- page_id=compute_page_id(page_type, target_path),
- page_type=page_type,
- title=title,
- content=response.content,
- summary=_extract_summary(response.content),
- source_hash=source_hash,
- model_name=self._provider.model_name,
- provider_name=self._provider.provider_name,
- input_tokens=response.input_tokens,
- output_tokens=response.output_tokens,
- cached_tokens=response.cached_tokens,
- generation_level=level,
- target_path=target_path,
- created_at=now,
- updated_at=now,
- )
-
- def _render(self, template_name: str, **kwargs: Any) -> str:
- """Render a Jinja2 template with the given kwargs."""
- template = self._jinja_env.get_template(template_name)
- return template.render(**kwargs)
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _extract_summary(content: str, max_chars: int = 320) -> str:
- """Extract a 1–3 sentence purpose blurb from rendered wiki markdown.
-
- Strategy: walk lines top-to-bottom, skip blanks/headings/list-markers/HTML
- comments, and take the first prose paragraph. Truncate at sentence boundary
- near max_chars. Fully deterministic — no extra LLM call.
- """
- if not content:
- return ""
- para_lines: list[str] = []
- for raw in content.splitlines():
- line = raw.strip()
- if not line:
- if para_lines:
- break
- continue
- if line.startswith(("#", ">", "```", "---", "<!--", "|", "- ", "* ", "1.")):
- if para_lines:
- break
- continue
- para_lines.append(line)
- if not para_lines:
- return ""
- text = " ".join(para_lines)
- if len(text) <= max_chars:
- return text
- # Truncate at the last sentence boundary before max_chars
- cut = text[:max_chars]
- last_period = max(cut.rfind(". "), cut.rfind("? "), cut.rfind("! "))
- if last_period > max_chars // 2:
- return cut[: last_period + 1]
- return cut.rstrip() + "…"
-
-
-def _is_infra_file(parsed: ParsedFile) -> bool:
- """Return True if the file is an infrastructure file."""
- lang = parsed.file_info.language
- if lang in _INFRA_LANGUAGES:
- return True
- name = Path(parsed.file_info.path).name
- return name in _INFRA_FILENAMES
-
-
-def _is_significant_file(
- parsed: ParsedFile,
- pagerank: dict[str, float],
- betweenness: dict[str, float],
- config: Any, # GenerationConfig
- pr_threshold: float,
-) -> bool:
- """Return True if this code file deserves its own file_page.
-
- A file is significant if it is connected/important in the dependency graph
- (entry point, top PageRank percentile, or bridge file) AND has enough
- content to document.
-
- The symbol requirement is waived for files with no original definitions
- (state modules, __init__ re-exporters, config files) that are still heavily
- imported — these are architecturally important even without function bodies.
- Package __init__.py files with any symbols are always included since they
- are the public interface of their module.
- """
- path = parsed.file_info.path
- pr = pagerank.get(path, 0.0)
- bet = betweenness.get(path, 0.0)
- is_entry = parsed.file_info.is_entry_point
-
- # Package __init__.py files are module interfaces — always include them
- # if they have any symbols (re-exports, __getattr__, etc.)
- if path.endswith("__init__.py") and len(parsed.symbols) > 0:
- return True
-
- # Test files are always significant when present. They have near-zero
- # PageRank because nothing imports them back, but they answer "what
- # tests exercise X" / "where is Y verified" questions that the doc layer
- # is the right place to surface. Users who want to exclude tests
- # entirely can do so via skip_tests in the orchestrator upstream.
- if parsed.file_info.is_test and len(parsed.symbols) > 0:
- return True
-
- # Must appear significant in the graph
- if not (is_entry or pr >= pr_threshold or bet > 0.0):
- return False
-
- # Trivial-file gate: small files with almost no symbols (data classes,
- # marker classes, single-message wrappers like Messages/*.cs) produce
- # low-value pages. Entry points and graph hubs (high PageRank) bypass.
- if (
- getattr(config, "skip_trivial_files", True)
- and not is_entry
- and pr < pr_threshold * 2
- and len(parsed.symbols) <= 2
- and parsed.file_info.size_bytes < 1500
- ):
- return False
-
- # Waive the symbol-count requirement for graph-connected files that have
- # no original definitions of their own (e.g. state/config modules that
- # are imported by many files but mostly re-export or assemble values).
- if len(parsed.symbols) < config.file_page_min_symbols:
- return is_entry or pr >= pr_threshold
-
- return True
-
-
-def _select_clone_representatives(
- code_files: list[ParsedFile],
- pagerank: dict[str, float],
- *,
- min_cluster_size: int = 3,
-) -> set[str]:
- """Return paths of files to *drop* because they are near-clones.
-
- Groups files by (parent_directory, signature shape), where the shape is the
- sorted tuple of ``(symbol_kind, symbol_name)`` pairs from the parser. When
- a cluster has at least ``min_cluster_size`` members, the highest-PageRank
- member is kept and the rest are dropped. Entry points are never dropped.
-
- Language-agnostic: works for any language whose symbols carry a kind+name,
- which the parser guarantees.
- """
- from collections import defaultdict
-
- clusters: dict[tuple[str, tuple[tuple[str, str], ...]], list[ParsedFile]] = defaultdict(list)
- for p in code_files:
- if p.file_info.is_entry_point or not p.symbols:
- continue
- parent = str(Path(p.file_info.path).parent.as_posix())
- shape = tuple(sorted((str(s.kind), s.name) for s in p.symbols))
- clusters[(parent, shape)].append(p)
-
- drop: set[str] = set()
- for members in clusters.values():
- if len(members) < min_cluster_size:
- continue
- members.sort(key=lambda p: pagerank.get(p.file_info.path, 0.0), reverse=True)
- for loser in members[1:]:
- drop.add(loser.file_info.path)
- return drop
-
-
-# ---------------------------------------------------------------------------
-# LLM output validation
-# ---------------------------------------------------------------------------
-
-# Common words that appear in backticks but are not code symbols.
-_BACKTICK_SKIP = frozenset(
- {
- # Python builtins & keywords
- "True",
- "False",
- "None",
- "self",
- "cls",
- "super",
- "str",
- "int",
- "float",
- "bool",
- "list",
- "dict",
- "set",
- "tuple",
- "bytes",
- "object",
- "type",
- "Any",
- "Optional",
- "Union",
- "async",
- "await",
- "return",
- "yield",
- "import",
- "from",
- "class",
- "def",
- "if",
- "else",
- "for",
- "while",
- "try",
- "except",
- "raise",
- "with",
- "pass",
- "break",
- "continue",
- "lambda",
- "in",
- "not",
- "and",
- "or",
- "is",
- "del",
- "assert",
- "finally",
- "elif",
- "as",
- "global",
- "nonlocal",
- # JS/TS keywords
- "null",
- "undefined",
- "this",
- "const",
- "let",
- "var",
- "function",
- "export",
- "default",
- "extends",
- "implements",
- "interface",
- "enum",
- "new",
- "typeof",
- "instanceof",
- "void",
- "never",
- "string",
- "number",
- "boolean",
- "symbol",
- "bigint",
- "unknown",
- "readonly",
- "abstract",
- "static",
- "private",
- "protected",
- "public",
- "require",
- "module",
- "exports",
- "Promise",
- "Map",
- "Set",
- "Array",
- "Object",
- "Error",
- "Date",
- "RegExp",
- "JSON",
- "Math",
- "console",
- # Common tool/ecosystem names
- "pip",
- "npm",
- "npx",
- "yarn",
- "pnpm",
- "go",
- "rust",
- "python",
- "node",
- "cargo",
- "uv",
- "git",
- "docker",
- "make",
- # Common framework/lib names the LLM mentions in prose
- "FastAPI",
- "React",
- "Next",
- "Express",
- "Django",
- "Flask",
- "SQLAlchemy",
- "Pydantic",
- "Click",
- "Typer",
- "pytest",
- "asyncio",
- "pathlib",
- "dataclass",
- "dataclasses",
- }
-)
-
-# Regex: single-backtick references that look like identifiers.
-_BACKTICK_REF_RE = re.compile(r"(?<!`)` *([A-Za-z_]\w*(?:\.\w+)*) *`(?!`)")
-
-# Patterns that indicate the backtick content is a path, command, or
-# value rather than a symbol reference — these should never be flagged.
-_PATH_OR_CMD_RE = re.compile(
- r"[/\\]" # contains path separator
- r"|\.(?:py|ts|js|json|yaml|yml|toml|md|sh|sql|css|html)$" # file extension
- r"|^[a-z][\w-]*$" # all-lowercase with hyphens = CLI command/flag
-)
-
-
-def _validate_symbol_references(
- content: str,
- parsed: ParsedFile,
-) -> list[str]:
- """Cross-check backtick-quoted names in LLM output against actual symbols.
-
- Returns a list of warning strings for references that don't match any
- known symbol, export, or import in the ParsedFile. Designed to have low
- false-positive rates — only flags references that look like symbol names
- but can't be found anywhere in the file's AST, imports, or source text.
- """
- refs = set(_BACKTICK_REF_RE.findall(content))
- if not refs:
- return []
-
- # Build the known-names set from AST data
- known: set[str] = set()
- for s in parsed.symbols:
- known.add(s.name)
- known.add(s.qualified_name)
- # Decorator names are valid references (e.g. @app.command("init"))
- for dec in s.decorators:
- # Extract the decorator function name: "@app.command" → "command"
- dec_name = dec.lstrip("@").split("(")[0]
- known.add(dec_name)
- known.add(dec_name.split(".")[-1])
- known.update(parsed.exports)
- for imp in parsed.imports:
- if imp.module_path:
- # Add both the final component and intermediate segments
- parts = imp.module_path.split(".")
- known.update(parts)
- known.update(imp.imported_names)
- # Named bindings from import resolution
- for binding in getattr(imp, "bindings", []):
- known.add(binding.local_name)
- if binding.exported_name:
- known.add(binding.exported_name)
-
- # Also add all string literals from the source that look like identifiers
- # (catches Click command names, decorator arguments, dict keys, etc.)
- source_text = ""
- if hasattr(parsed, "file_info") and hasattr(parsed.file_info, "path"):
- # The source is in the context, but we only have the parsed file here.
- # Use docstring and symbol names as a cheap approximation.
- if parsed.docstring:
- known.update(w for w in parsed.docstring.split() if w.isidentifier())
-
- warnings: list[str] = []
- for ref in refs:
- if ref in _BACKTICK_SKIP:
- continue
- # Skip short refs (1-2 chars are usually variables like `x`, `i`, `db`)
- if len(ref) <= 2:
- continue
- # Skip anything that looks like a path, file, or CLI command
- if _PATH_OR_CMD_RE.search(ref):
- continue
- # Skip all-uppercase (likely constants from other files: `MAX_RETRIES`)
- if ref.isupper():
- continue
- # Check against known names
- base = ref.split(".")[-1]
- if ref in known or base in known:
- continue
- # Skip if the ref is a substring of any known symbol (covers partial
- # references like `parse` when `parse_file` exists)
- if any(ref in k for k in known if len(k) > len(ref)):
- continue
- warnings.append(ref)
- return warnings
diff --git a/packages/core/src/repowise/core/generation/page_generator/README.md b/packages/core/src/repowise/core/generation/page_generator/README.md
new file mode 100644
index 000000000..ee28050a1
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/README.md
@@ -0,0 +1,64 @@
+# `page_generator`
+
+Converts assembled context dataclasses into `GeneratedPage` objects by
+rendering Jinja prompts, calling an LLM provider, and (for the long tail of
+large repos) rendering deterministic template-only pages.
+
+## Purpose
+
+`PageGenerator.generate_all()` is the top of the documentation-generation
+stage. It selects which pages to produce (via `generation.selection`), then
+generates them in ordered, concurrency-bounded levels:
+
+| Level | Page type | Notes |
+| --- | --- | --- |
+| 0/1 | `api_contract`, `symbol_spotlight` | merged batch |
+| 2 | `file_page` | topo-ordered; tier-1 LLM or tier-2 template |
+| 3 | `scc_page` | circular-dependency cycles |
+| 4 | `module_page` | community / directory groups |
+| 6/7/8 | `repo_overview` + `architecture_diagram`, `infra_page`, `onboarding` | merged batch |
+
+### Tiered doc generation
+
+`GenerationConfig.tier1_top_n` caps how many file pages get full LLM
+generation. The top *N* selected files by PageRank are **tier-1** (the
+existing LLM path); the rest are **tier-2** — rendered from
+`templates/file_page_tier2.j2` with no LLM call and embedded for search.
+`tier1_top_n = None` (default) puts every selected page in tier-1, so
+behaviour is unchanged unless the knob is set.
+
+## Public API
+
+```python
+from repowise.core.generation.page_generator import PageGenerator, SYSTEM_PROMPTS, PriorPage
+```
+
+- `PageGenerator(provider, assembler, config, ...)` — `await .generate_all(...)`
+ returns `list[GeneratedPage]`. Per-type `generate_file_page` / `generate_*`
+ methods are also public.
+- `SYSTEM_PROMPTS` — constant system prompt per page type (prefix-cacheable).
+- `PriorPage` — cross-run reuse snapshot keyed by `page_id`.
+
+## Internal layout
+
+- `core.py` — `PageGenerator` (provider call, two-tier caching, file-page LLM
+ path + deterministic tier-2 renderer).
+- `pertype.py` — `PerTypeGenerationMixin` with the per-page-type methods.
+- `orchestrate.py` — `_GenerationRun` (per-call state, level runner) and
+ `run_generate_all` (the entry point `generate_all` delegates to).
+- `levels.py` — pure per-level coroutine builders reading the run state.
+- `tiering.py` — `partition_file_tiers` (tier-1/tier-2 split by PageRank).
+- `prompts.py`, `validation.py`, `helpers.py` — constants and pure helpers.
+
+## Extension points
+
+- New page type: add a `generate_*` method to `pertype.py`, a level builder to
+ `levels.py`, and wire it into `_GenerationRun.execute()`.
+- New tier policy: extend `partition_file_tiers` / add a renderer template.
+
+## Tests
+
+- `tests/unit/generation/test_page_generator.py`
+- `tests/unit/generation/test_page_generator_tiering.py`
+- `tests/integration/test_generation_pipeline.py`
+"""
diff --git a/packages/core/src/repowise/core/generation/page_generator/__init__.py b/packages/core/src/repowise/core/generation/page_generator/__init__.py
new file mode 100644
index 000000000..7f90e6b31
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/__init__.py
@@ -0,0 +1,41 @@
+"""Page generator package — converts context dataclasses into GeneratedPage objects.
+
+Public surface (import path preserved from the former single-module form):
+
+ from repowise.core.generation.page_generator import PageGenerator, SYSTEM_PROMPTS, PriorPage
+
+Internal layout:
+ core.py — the PageGenerator class (provider call, caching, file pages)
+ pertype.py — per-page-type generate_* methods (mixin)
+ orchestrate.py — level-by-level generate_all orchestration
+ levels.py — per-level coroutine builders
+ tiering.py — tier-1/tier-2 partition (large-repo doc generation)
+ prompts.py — SYSTEM_PROMPTS constants + language names
+ validation.py — hallucination detection for LLM output
+ helpers.py — pure helpers (summaries, significance, clone dedupe)
+"""
+
+from __future__ import annotations
+
+from .core import PageGenerator, PriorPage
+from .helpers import (
+ _extract_summary,
+ _is_infra_file,
+ _is_significant_file,
+ _now_iso,
+ _select_clone_representatives,
+)
+from .prompts import SYSTEM_PROMPTS
+from .validation import _validate_symbol_references
+
+__all__ = [
+ "SYSTEM_PROMPTS",
+ "PageGenerator",
+ "PriorPage",
+ "_extract_summary",
+ "_is_infra_file",
+ "_is_significant_file",
+ "_now_iso",
+ "_select_clone_representatives",
+ "_validate_symbol_references",
+]
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
new file mode 100644
index 000000000..2d57e8db3
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -0,0 +1,389 @@
+"""PageGenerator — renders prompts, calls the provider, wraps responses.
+
+PageGenerator is the main orchestration layer. It:
+ 1. Calls ContextAssembler to build template context from ingestion data.
+ 2. Renders the Jinja2 user-prompt template.
+ 3. Calls the provider with the rendered prompt + system prompt constant.
+ 4. Wraps the response in a GeneratedPage.
+ 5. Manages concurrency (asyncio.Semaphore) and prompt caching (SHA256).
+
+The level-by-level orchestration of ``generate_all`` lives in
+``orchestrate.py``; the per-type ``generate_*`` methods live in ``pertype.py``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import jinja2
+import structlog
+
+from repowise.core.ingestion.models import ParsedFile, RepoStructure
+from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse
+
+from ..context_assembler import ContextAssembler, FilePageContext
+from ..models import (
+ GENERATION_LEVELS,
+ GeneratedPage,
+ GenerationConfig,
+ compute_page_id,
+ compute_source_hash,
+)
+from .helpers import _extract_summary, _now_iso
+from .pertype import PerTypeGenerationMixin
+from .prompts import _LANGUAGE_NAMES, SYSTEM_PROMPTS
+from .validation import _validate_symbol_references
+
+if TYPE_CHECKING:
+ from pathlib import Path as _Path # noqa: F401
+
+log = structlog.get_logger(__name__)
+
+
+@dataclass(frozen=True)
+class PriorPage:
+ """Snapshot of a previously-generated page used for cross-run reuse.
+
+ Lives in :class:`PageGenerator` keyed by ``page_id``. When the freshly
+ rendered prompt produces a matching ``source_hash`` under the same
+ ``model_name``, the LLM call is skipped and ``content`` is reused.
+ """
+
+ source_hash: str
+ model_name: str
+ content: str
+ input_tokens: int = 0
+ output_tokens: int = 0
+ cached_tokens: int = 0
+
+
+class PageGenerator(PerTypeGenerationMixin):
+ """Generate wiki pages by rendering prompts and calling an LLM provider.
+
+ Args:
+ provider: Any BaseProvider implementation.
+ assembler: ContextAssembler instance.
+ config: GenerationConfig controlling budget, concurrency, caching.
+ jinja_env: Optional Jinja2 Environment (defaults to FileSystemLoader
+ pointing at the templates/ directory next to this package).
+ """
+
+ def __init__(
+ self,
+ provider: BaseProvider,
+ assembler: ContextAssembler,
+ config: GenerationConfig,
+ jinja_env: jinja2.Environment | None = None,
+ vector_store: Any | None = None,
+ language: str = "en",
+ prior_pages: dict[str, PriorPage] | None = None,
+ ) -> None:
+ self._provider = provider
+ self._assembler = assembler
+ self._config = config
+ self._vector_store = vector_store
+ self._language = language
+ self._cache: dict[str, GeneratedResponse] = {}
+ # Map of page_id → PriorPage from previous generation runs. When the
+ # rendered prompt's source_hash matches the prior page's hash AND the
+ # model is the same, the LLM call is skipped and the prior content is
+ # reused. Wired by the orchestrator from the persisted wiki_pages
+ # table.
+ self._prior_pages: dict[str, PriorPage] = prior_pages or {}
+ self._reuse_count: int = 0
+
+ if jinja_env is None:
+ templates_dir = Path(__file__).parent.parent / "templates"
+ loader = jinja2.FileSystemLoader(str(templates_dir))
+ jinja_env = jinja2.Environment(
+ loader=loader,
+ undefined=jinja2.StrictUndefined,
+ autoescape=False,
+ )
+ self._jinja_env = jinja_env
+
+ # ------------------------------------------------------------------
+ # generate_all — orchestration (delegates to orchestrate.py)
+ # ------------------------------------------------------------------
+
+ async def generate_all(
+ self,
+ parsed_files: list[ParsedFile],
+ source_map: dict[str, bytes],
+ graph_builder: Any, # GraphBuilder
+ repo_structure: RepoStructure,
+ repo_name: str,
+ job_system: Any | None = None, # JobSystem | None
+ on_page_done: Callable[[str], None] | None = None,
+ on_total_known: Callable[[int], None] | None = None,
+ on_subphase: Callable[[str, int | None], None] | None = None,
+ git_meta_map: dict[str, dict] | None = None,
+ resume: bool = False,
+ repo_path: Path | str | None = None,
+ dead_code_report: Any | None = None,
+ decision_report: Any | None = None,
+ external_systems: list[dict] | None = None,
+ ) -> list[GeneratedPage]:
+ """Generate all wiki pages for a repository.
+
+ Runs generation in ordered levels. Each level's pages are generated
+ concurrently (up to config.max_concurrency). Failures within a level
+ are logged but do not abort the remaining levels.
+ """
+ from .orchestrate import run_generate_all
+
+ return await run_generate_all(
+ self,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ graph_builder=graph_builder,
+ repo_structure=repo_structure,
+ repo_name=repo_name,
+ job_system=job_system,
+ on_page_done=on_page_done,
+ on_total_known=on_total_known,
+ on_subphase=on_subphase,
+ git_meta_map=git_meta_map,
+ resume=resume,
+ repo_path=repo_path,
+ dead_code_report=dead_code_report,
+ decision_report=decision_report,
+ external_systems=external_systems,
+ )
+
+ # ------------------------------------------------------------------
+ # File-page generation (LLM + deterministic tier-2)
+ # ------------------------------------------------------------------
+
+ async def _generate_file_page_from_ctx(
+ self,
+ parsed: ParsedFile,
+ ctx: FilePageContext,
+ ) -> GeneratedPage:
+ """Generate a file_page from a pre-assembled context (avoids double-assembly)."""
+ # RAG context: query vector store for related pages (B1).
+ # Gated by two short-circuits so we don't burn an embedder
+ # round-trip on every page when the result wouldn't help:
+ # 1. ``enable_rag_context`` config flag (off → fully skip).
+ # 2. ``rag_min_store_size`` — early pages run against an empty
+ # or near-empty store and the search returns nothing useful.
+ if self._vector_store is not None and getattr(
+ self._config, "enable_rag_context", True
+ ):
+ min_store_size = max(0, int(getattr(self._config, "rag_min_store_size", 10) or 0))
+ store_ok = True
+ if min_store_size > 0:
+ try:
+ current_ids = await self._vector_store.list_page_ids()
+ store_ok = len(current_ids) >= min_store_size
+ except Exception:
+ # If the store can't be sized cheaply, fall through to
+ # the search — it'll either succeed or hit the
+ # existing exception path.
+ store_ok = True
+ if store_ok:
+ query_terms = parsed.exports or [
+ s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"
+ ]
+ if query_terms:
+ try:
+ results = await self._vector_store.search(
+ ", ".join(query_terms[:5]), limit=3
+ )
+ self_id = f"file_page:{parsed.file_info.path}"
+ ctx.rag_context = [
+ f"[{r.page_id}]\n{r.snippet}"
+ for r in results
+ if r.page_id != self_id
+ ]
+ except Exception as e:
+ log.debug(
+ "rag.search_failed", path=parsed.file_info.path, error=str(e)
+ )
+ user_prompt = self._render("file_page.j2", ctx=ctx)
+ response = await self._call_provider(
+ "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
+ page = self._build_generated_page(
+ "file_page",
+ parsed.file_info.path,
+ f"File: {parsed.file_info.path}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["file_page"],
+ )
+ # Cross-check LLM output against actual symbols
+ hal_warnings = _validate_symbol_references(response.content, parsed)
+ if hal_warnings:
+ log.warning(
+ "hallucination_check",
+ path=parsed.file_info.path,
+ count=len(hal_warnings),
+ refs=hal_warnings[:5],
+ )
+ page.metadata["hallucination_warnings"] = hal_warnings
+ return page
+
+ async def _generate_file_page_tier2(
+ self,
+ parsed: ParsedFile,
+ ctx: FilePageContext,
+ ) -> GeneratedPage:
+ """Render a deterministic (no-LLM) tier-2 file page.
+
+ Used for the long tail of selected files on large repos. The page is
+ built straight from the assembled context via a Jinja template, marked
+ as template-generated, and carries zero token cost. It is embedded for
+ search by the level runner like any other page. No provider call and
+ no hallucination check (the content is factual by construction).
+ """
+ content = self._render("file_page_tier2.j2", ctx=ctx)
+ now = _now_iso()
+ page = GeneratedPage(
+ page_id=compute_page_id("file_page", parsed.file_info.path),
+ page_type="file_page",
+ title=f"File: {parsed.file_info.path}",
+ content=content,
+ summary=_extract_summary(content),
+ source_hash=compute_source_hash(content),
+ model_name=self._provider.model_name,
+ provider_name="template",
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ generation_level=GENERATION_LEVELS["file_page"],
+ target_path=parsed.file_info.path,
+ created_at=now,
+ updated_at=now,
+ )
+ page.metadata["doc_tier"] = 2
+ return page
+
+ # ------------------------------------------------------------------
+ # Provider call + page assembly
+ # ------------------------------------------------------------------
+
+ async def _call_provider(
+ self,
+ page_type: str,
+ user_prompt: str,
+ request_id: str,
+ target_path: str | None = None,
+ ) -> GeneratedResponse:
+ """Call the provider with caching, optionally prefixing a language instruction."""
+ # Persistent cross-run cache: if the page exists from a prior run, was
+ # produced by the same model, and the prompt's source_hash matches,
+ # reuse the stored content without an LLM call.
+ if self._config.cache_enabled and target_path is not None:
+ page_id = compute_page_id(page_type, target_path)
+ prior = self._prior_pages.get(page_id)
+ if prior is not None and prior.model_name == self._provider.model_name:
+ current_hash = compute_source_hash(user_prompt)
+ if prior.source_hash == current_hash:
+ self._reuse_count += 1
+ log.debug(
+ "page_cache.persistent_hit",
+ page_type=page_type,
+ target_path=target_path,
+ )
+ return GeneratedResponse(
+ content=prior.content,
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ usage={"reused_from_prior_run": True},
+ )
+
+ key = self._compute_cache_key(page_type, user_prompt)
+ if self._config.cache_enabled and key in self._cache:
+ log.debug("Cache hit", page_type=page_type, key=key[:8])
+ return self._cache[key]
+
+ system_prompt = self._build_system_prompt(page_type)
+
+ # The same system prompt is reused for every page of a given type, so
+ # mark it as cacheable. Providers without server-side prompt caching
+ # ignore the hint safely.
+ cache_hints: tuple[CacheHint, ...] = (
+ (CacheHint(segment="system"),) if self._config.cache_enabled else ()
+ )
+
+ response = await self._provider.generate(
+ system_prompt,
+ user_prompt,
+ max_tokens=self._config.max_tokens,
+ temperature=self._config.temperature,
+ request_id=request_id,
+ reasoning=self._config.reasoning,
+ cache_hints=cache_hints,
+ )
+
+ if self._config.cache_enabled:
+ self._cache[key] = response
+
+ return response
+
+ def _build_system_prompt(self, page_type: str) -> str:
+ base_system = SYSTEM_PROMPTS[page_type]
+ # Sanitize the configured language code: lower, strip, drop anything that isn't
+ # alphanumeric or underscore. Prevents user-supplied config from injecting
+ # newlines or extra instructions into the system prompt.
+ raw = (self._language or "en").lower().strip()
+ lang_code = "".join(ch for ch in raw if ch.isalnum() or ch == "_")
+ if lang_code not in _LANGUAGE_NAMES:
+ if lang_code != "en":
+ log.warning("unknown_language_code", code=lang_code, fallback="en")
+ lang_code = "en"
+ if lang_code == "en":
+ return base_system
+ lang_name = _LANGUAGE_NAMES[lang_code]
+ instruction = (
+ f"Generate all documentation content in {lang_name}. "
+ "Keep all code, file paths, and symbol names in their original form. "
+ "Do not translate them.\n\n"
+ )
+ return instruction + base_system
+
+ def _compute_cache_key(self, page_type: str, user_prompt: str) -> str:
+ """Return SHA256(model + language + page_type + user_prompt) as cache key."""
+ raw = f"{self._provider.model_name}:{self._language}:{page_type}:{user_prompt}"
+ return hashlib.sha256(raw.encode()).hexdigest()
+
+ def _build_generated_page(
+ self,
+ page_type: str,
+ target_path: str,
+ title: str,
+ response: GeneratedResponse,
+ source_hash: str,
+ level: int,
+ ) -> GeneratedPage:
+ """Wrap a GeneratedResponse in a GeneratedPage."""
+ now = _now_iso()
+ return GeneratedPage(
+ page_id=compute_page_id(page_type, target_path),
+ page_type=page_type,
+ title=title,
+ content=response.content,
+ summary=_extract_summary(response.content),
+ source_hash=source_hash,
+ model_name=self._provider.model_name,
+ provider_name=self._provider.provider_name,
+ input_tokens=response.input_tokens,
+ output_tokens=response.output_tokens,
+ cached_tokens=response.cached_tokens,
+ generation_level=level,
+ target_path=target_path,
+ created_at=now,
+ updated_at=now,
+ )
+
+ def _render(self, template_name: str, **kwargs: Any) -> str:
+ """Render a Jinja2 template with the given kwargs."""
+ template = self._jinja_env.get_template(template_name)
+ return template.render(**kwargs)
diff --git a/packages/core/src/repowise/core/generation/page_generator/helpers.py b/packages/core/src/repowise/core/generation/page_generator/helpers.py
new file mode 100644
index 000000000..1a0b65f74
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/helpers.py
@@ -0,0 +1,219 @@
+"""Pure helper functions for the page generator.
+
+These have no dependency on :class:`PageGenerator` state and are unit-tested
+in isolation. ``_extract_summary`` / ``_is_significant_file`` /
+``_select_clone_representatives`` keep their original names because external
+docs and call sites reference them.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from repowise.core.ingestion.models import ParsedFile
+
+_INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()
+_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})
+_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()
+
+
+def _now_iso() -> str:
+ """Return current UTC time as ISO-8601 string."""
+ return datetime.now(UTC).isoformat()
+
+
+def _extract_summary(content: str, max_chars: int = 320) -> str:
+ """Extract a 1-3 sentence purpose blurb from rendered wiki markdown.
+
+ Strategy: walk lines top-to-bottom, skip blanks/headings/list-markers/HTML
+ comments, and take the first prose paragraph. Truncate at sentence boundary
+ near max_chars. Fully deterministic — no extra LLM call.
+ """
+ if not content:
+ return ""
+ para_lines: list[str] = []
+ for raw in content.splitlines():
+ line = raw.strip()
+ if not line:
+ if para_lines:
+ break
+ continue
+ if line.startswith(("#", ">", "```", "---", "<!--", "|", "- ", "* ", "1.")):
+ if para_lines:
+ break
+ continue
+ para_lines.append(line)
+ if not para_lines:
+ return ""
+ text = " ".join(para_lines)
+ if len(text) <= max_chars:
+ return text
+ # Truncate at the last sentence boundary before max_chars
+ cut = text[:max_chars]
+ last_period = max(cut.rfind(". "), cut.rfind("? "), cut.rfind("! "))
+ if last_period > max_chars // 2:
+ return cut[: last_period + 1]
+ return cut.rstrip() + "…"
+
+
+def overview_summary(content: str) -> str:
+ """Return a dependency-context blurb favouring the ## Overview section.
+
+ Distinct from :func:`_extract_summary`: this is the summary fed into
+ downstream pages' dependency context and into the vector-store embed
+ payload, where the ## Overview lead sentence is the most useful anchor.
+ """
+ if "## Overview" in content:
+ start = content.index("## Overview") + len("## Overview")
+ end = content.find("\n##", start)
+ return content[start : end if end > 0 else start + 1600].strip()[:400]
+ return content[:400]
+
+
+def _is_infra_file(parsed: ParsedFile) -> bool:
+ """Return True if the file is an infrastructure file."""
+ lang = parsed.file_info.language
+ if lang in _INFRA_LANGUAGES:
+ return True
+ name = Path(parsed.file_info.path).name
+ return name in _INFRA_FILENAMES
+
+
+def _is_significant_file(
+ parsed: ParsedFile,
+ pagerank: dict[str, float],
+ betweenness: dict[str, float],
+ config: Any, # GenerationConfig
+ pr_threshold: float,
+) -> bool:
+ """Return True if this code file deserves its own file_page.
+
+ A file is significant if it is connected/important in the dependency graph
+ (entry point, top PageRank percentile, or bridge file) AND has enough
+ content to document.
+
+ The symbol requirement is waived for files with no original definitions
+ (state modules, __init__ re-exporters, config files) that are still heavily
+ imported — these are architecturally important even without function bodies.
+ Package __init__.py files with any symbols are always included since they
+ are the public interface of their module.
+ """
+ path = parsed.file_info.path
+ pr = pagerank.get(path, 0.0)
+ bet = betweenness.get(path, 0.0)
+ is_entry = parsed.file_info.is_entry_point
+
+ # Package __init__.py files are module interfaces — always include them
+ # if they have any symbols (re-exports, __getattr__, etc.)
+ if path.endswith("__init__.py") and len(parsed.symbols) > 0:
+ return True
+
+ # Test files are always significant when present. They have near-zero
+ # PageRank because nothing imports them back, but they answer "what
+ # tests exercise X" / "where is Y verified" questions that the doc layer
+ # is the right place to surface. Users who want to exclude tests
+ # entirely can do so via skip_tests in the orchestrator upstream.
+ if parsed.file_info.is_test and len(parsed.symbols) > 0:
+ return True
+
+ # Must appear significant in the graph
+ if not (is_entry or pr >= pr_threshold or bet > 0.0):
+ return False
+
+ # Trivial-file gate: small files with almost no symbols (data classes,
+ # marker classes, single-message wrappers like Messages/*.cs) produce
+ # low-value pages. Entry points and graph hubs (high PageRank) bypass.
+ if (
+ getattr(config, "skip_trivial_files", True)
+ and not is_entry
+ and pr < pr_threshold * 2
+ and len(parsed.symbols) <= 2
+ and parsed.file_info.size_bytes < 1500
+ ):
+ return False
+
+ # Waive the symbol-count requirement for graph-connected files that have
+ # no original definitions of their own (e.g. state/config modules that
+ # are imported by many files but mostly re-export or assemble values).
+ if len(parsed.symbols) < config.file_page_min_symbols:
+ return is_entry or pr >= pr_threshold
+
+ return True
+
+
+def _select_clone_representatives(
+ code_files: list[ParsedFile],
+ pagerank: dict[str, float],
+ *,
+ min_cluster_size: int = 3,
+) -> set[str]:
+ """Return paths of files to *drop* because they are near-clones.
+
+ Groups files by (parent_directory, signature shape), where the shape is the
+ sorted tuple of ``(symbol_kind, symbol_name)`` pairs from the parser. When
+ a cluster has at least ``min_cluster_size`` members, the highest-PageRank
+ member is kept and the rest are dropped. Entry points are never dropped.
+
+ Language-agnostic: works for any language whose symbols carry a kind+name,
+ which the parser guarantees.
+ """
+ clusters: dict[tuple[str, tuple[tuple[str, str], ...]], list[ParsedFile]] = defaultdict(list)
+ for p in code_files:
+ if p.file_info.is_entry_point or not p.symbols:
+ continue
+ parent = str(Path(p.file_info.path).parent.as_posix())
+ shape = tuple(sorted((str(s.kind), s.name) for s in p.symbols))
+ clusters[(parent, shape)].append(p)
+
+ drop: set[str] = set()
+ for members in clusters.values():
+ if len(members) < min_cluster_size:
+ continue
+ members.sort(key=lambda p: pagerank.get(p.file_info.path, 0.0), reverse=True)
+ for loser in members[1:]:
+ drop.add(loser.file_info.path)
+ return drop
+
+
+def build_dead_code_map(dead_code_report: Any | None) -> dict[str, list[dict]]:
+ """Index dead-code findings by file path for per-page lookup."""
+ dead_code_by_file: dict[str, list[dict]] = {}
+ if dead_code_report is not None and getattr(dead_code_report, "findings", None):
+ for f in dead_code_report.findings:
+ dead_code_by_file.setdefault(f.file_path, []).append(
+ {
+ "symbol_name": f.symbol_name,
+ "symbol_kind": f.symbol_kind,
+ "kind": str(f.kind),
+ "reason": f.reason,
+ "confidence": f.confidence,
+ "safe_to_delete": f.safe_to_delete,
+ }
+ )
+ return dead_code_by_file
+
+
+def build_decision_maps(
+ decision_report: Any | None,
+) -> tuple[dict[str, list[dict]], list[dict]]:
+ """Index decision records by file path and as a flat list."""
+ decisions_by_file: dict[str, list[dict]] = {}
+ decisions_all: list[dict] = []
+ if decision_report is not None and getattr(decision_report, "decisions", None):
+ for d in decision_report.decisions:
+ payload = {
+ "title": d.title,
+ "decision": d.decision,
+ "rationale": d.rationale,
+ "source": d.source,
+ "confidence": d.confidence,
+ "evidence_file": d.evidence_file,
+ }
+ decisions_all.append(payload)
+ for fp in d.affected_files or []:
+ decisions_by_file.setdefault(fp, []).append(payload)
+ return decisions_by_file, decisions_all
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
new file mode 100644
index 000000000..271157488
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -0,0 +1,306 @@
+"""Per-level coroutine builders for the generation orchestrator.
+
+Each function takes the live :class:`_GenerationRun` and returns a list of
+``(page_id, coroutine)`` tuples for one generation level. They read graph
+metrics, selection allow-sets, and the shared context cache off the run
+object. Behaviour mirrors the original inline ``generate_all`` exactly.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+from .. import onboarding as _onboarding
+from ..context_assembler import FilePageContext
+from ..models import compute_page_id
+from .helpers import _is_infra_file
+
+if TYPE_CHECKING:
+ from .orchestrate import _GenerationRun
+
+log = structlog.get_logger(__name__)
+
+
+def build_level01_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 0 (api_contract) + level 1 (symbol_spotlight), merged."""
+ gen = run.gen
+ # ---- Level 0: api_contract (allow-set filtered) ----
+ api_files = [
+ p
+ for p in run.parsed_files
+ if p.file_info.is_api_contract and p.file_info.path in run.sel_api_paths
+ ]
+ level0 = [
+ (
+ compute_page_id("api_contract", p.file_info.path),
+ gen.generate_api_contract(p, run.source_map.get(p.file_info.path, b"")),
+ )
+ for p in api_files
+ if compute_page_id("api_contract", p.file_info.path) not in run.completed_ids
+ ]
+
+ # ---- Level 1: symbol_spotlight (allow-set filtered) ----
+ parsed_by_path: dict[str, Any] = {p.file_info.path: p for p in run.parsed_files}
+ top_symbols: list[tuple[Any, Any]] = []
+ for file_path, sym_name in run.selection.symbol_spotlights:
+ pf = parsed_by_path.get(file_path)
+ if pf is None:
+ continue
+ sym = next((s for s in pf.symbols if s.name == sym_name), None)
+ if sym is not None:
+ top_symbols.append((sym, pf))
+
+ level1 = [
+ (
+ compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}"),
+ gen.generate_symbol_spotlight(
+ sym, pf, run.pagerank, run.graph, source_map=run.source_map
+ ),
+ )
+ for sym, pf in top_symbols
+ if compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}")
+ not in run.completed_ids
+ ]
+ return level0 + level1
+
+
+def _topo_order_code_files(run: _GenerationRun) -> None:
+ """Reorder ``run.code_files`` so dependencies are generated before dependents."""
+ code_file_paths = [p.file_info.path for p in run.code_files]
+ graph = run.graph
+ try:
+ import networkx as nx # type: ignore[import]
+
+ code_file_set = set(code_file_paths)
+ dag = nx.DiGraph()
+ dag.add_nodes_from(code_file_paths)
+ for path_ in code_file_paths:
+ if path_ in graph:
+ for succ in graph.successors(path_):
+ if succ in code_file_set:
+ dag.add_edge(path_, succ) # path_ depends on succ
+
+ if nx.is_directed_acyclic_graph(dag):
+ # topological_sort yields u before v for each edge u→v (dependents
+ # before dependencies). We want leaves first, so reverse.
+ topo_order = list(reversed(list(nx.topological_sort(dag))))
+ else:
+ condensation = nx.condensation(dag)
+ topo_order_scc = list(reversed(list(nx.topological_sort(condensation))))
+ scc_members: dict[int, list[str]] = {
+ n: list(condensation.nodes[n]["members"]) for n in condensation.nodes
+ }
+ topo_order = [node for scc_id in topo_order_scc for node in scc_members[scc_id]]
+
+ priority_index = {p: i for i, p in enumerate(code_file_paths)}
+ topo_order = [p for p in topo_order if p in priority_index]
+ path_to_parsed = {p.file_info.path: p for p in run.code_files}
+ run.code_files = [path_to_parsed[p] for p in topo_order if p in path_to_parsed]
+ except Exception:
+ pass # Keep existing priority order on any failure
+
+
+async def _prefetch_dependency_summaries(run: _GenerationRun) -> None:
+ """Batch-prefetch dependency summaries from the vector store in one call."""
+ if run.vector_store is None:
+ return
+ needed_deps: set[str] = set()
+ for p in run.code_files:
+ path_ = p.file_info.path
+ if path_ not in run.graph:
+ continue
+ for dep in run.graph.successors(path_):
+ if dep.startswith("external:"):
+ continue
+ if dep in run.completed_page_summaries:
+ continue
+ needed_deps.add(dep)
+ if not needed_deps:
+ return
+ try:
+ batch = await run.vector_store.get_page_summaries_by_paths(list(needed_deps))
+ for dep_path, payload in batch.items():
+ summary = payload.get("summary") if payload else None
+ if summary:
+ run.completed_page_summaries[dep_path] = summary
+ except Exception as exc:
+ log.debug("rag.batch_dep_prefetch_failed", error=str(exc))
+
+
+async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 2 (file_page): topo-ordered context assembly + tier routing.
+
+ Context is assembled for ALL code files (module pages need it). Pages are
+ emitted only for files in the selection allow-set. Tier-1 paths get the
+ full LLM path; tier-2 paths get the deterministic template renderer.
+ """
+ gen = run.gen
+ _topo_order_code_files(run)
+ await _prefetch_dependency_summaries(run)
+
+ coros: list[tuple[str, Any]] = []
+ for p in run.code_files:
+ ctx: FilePageContext = gen._assembler.assemble_file_page(
+ p,
+ run.graph,
+ run.pagerank,
+ run.betweenness,
+ run.community,
+ run.source_map.get(p.file_info.path, b""),
+ git_meta=run.git_meta_map.get(p.file_info.path) if run.git_meta_map else None,
+ page_summaries=run.completed_page_summaries,
+ dead_code_findings=run.dead_code_by_file.get(p.file_info.path),
+ decision_records=run.decisions_by_file.get(p.file_info.path),
+ )
+ run.file_page_contexts[p.file_info.path] = ctx
+ path = p.file_info.path
+ pid = compute_page_id("file_page", path)
+ if path in run.sel_file_paths and pid not in run.completed_ids:
+ if path in run.tier1_paths:
+ coros.append((pid, gen._generate_file_page_from_ctx(p, ctx)))
+ else:
+ coros.append((pid, gen._generate_file_page_tier2(p, ctx)))
+ return coros
+
+
+def build_level3_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 3 (scc_page), allow-set filtered."""
+ gen = run.gen
+ coros: list[tuple[str, Any]] = []
+ for scc_id, scc_files in run.sel_scc_groups:
+ fc_list = [
+ run.file_page_contexts[f] for f in scc_files if f in run.file_page_contexts
+ ]
+ pid = compute_page_id("scc_page", scc_id)
+ if pid not in run.completed_ids:
+ coros.append((pid, gen.generate_scc_page(scc_id, scc_files, fc_list)))
+ return coros
+
+
+def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 4 (module_page), allow-set filtered."""
+ gen = run.gen
+ coros: list[tuple[str, Any]] = []
+ for mg in run.sel_module_groups:
+ fcs = [
+ run.file_page_contexts[fp]
+ for fp in mg.file_paths
+ if fp in run.file_page_contexts
+ ]
+ if not fcs:
+ continue
+ page_id = compute_page_id("module_page", mg.key)
+ if page_id in run.completed_ids:
+ continue
+ coros.append(
+ (
+ page_id,
+ gen.generate_module_page(
+ mg.display,
+ mg.language,
+ fcs,
+ run.graph,
+ git_meta_map=run.git_meta_map,
+ page_summaries=run.completed_page_summaries,
+ decision_records=run.decisions_all,
+ dead_code_findings=[
+ d for fc in fcs for d in run.dead_code_by_file.get(fc.file_path, [])
+ ],
+ external_systems=run.external_systems,
+ community_label=mg.label,
+ community_cohesion=mg.cohesion,
+ target_path=mg.key,
+ ),
+ )
+ )
+ return coros
+
+
+def build_level6_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 6 (repo_overview + architecture_diagram)."""
+ gen = run.gen
+ coros: list[tuple[str, Any]] = []
+ if compute_page_id("repo_overview", run.repo_name) not in run.completed_ids:
+ coros.append(
+ (
+ compute_page_id("repo_overview", run.repo_name),
+ gen.generate_repo_overview(
+ run.repo_structure,
+ run.pagerank,
+ run.sccs,
+ run.community,
+ git_meta_map=run.git_meta_map,
+ graph_builder=run.graph_builder,
+ repo_name=run.repo_name,
+ external_systems=run.external_systems,
+ decision_records=run.decisions_all[:10],
+ ),
+ )
+ )
+ if compute_page_id("architecture_diagram", run.repo_name) not in run.completed_ids:
+ coros.append(
+ (
+ compute_page_id("architecture_diagram", run.repo_name),
+ gen.generate_architecture_diagram(
+ run.graph, run.pagerank, run.community, run.sccs, run.repo_name
+ ),
+ )
+ )
+ return coros
+
+
+def build_level7_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 7 (infra_page), allow-set filtered."""
+ gen = run.gen
+ infra_files = [
+ p
+ for p in run.parsed_files
+ if _is_infra_file(p) and p.file_info.path in run.sel_infra_paths
+ ]
+ return [
+ (
+ compute_page_id("infra_page", p.file_info.path),
+ gen.generate_infra_page(p, run.source_map.get(p.file_info.path, b"")),
+ )
+ for p in infra_files
+ if compute_page_id("infra_page", p.file_info.path) not in run.completed_ids
+ ]
+
+
+def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+ """Level 8 (curated onboarding collection)."""
+ gen = run.gen
+ coros: list[tuple[str, Any]] = []
+ if not getattr(run.config, "enable_onboarding", True):
+ return coros
+ specs = _onboarding.iter_specs()
+ if not specs:
+ return coros
+ if run.on_subphase is not None:
+ with contextlib.suppress(Exception):
+ run.on_subphase("onboarding", len(specs))
+ signals = _onboarding.OnboardingSignals(
+ repo_name=run.repo_name,
+ repo_structure=run.repo_structure,
+ parsed_files=tuple(run.parsed_files),
+ source_map=run.source_map,
+ graph_builder=run.graph_builder,
+ pagerank=run.pagerank,
+ betweenness=run.betweenness,
+ community=run.community,
+ sccs=tuple(run.sccs),
+ git_meta_map=run.git_meta_map,
+ dead_code_by_file=run.dead_code_by_file,
+ decisions_all=tuple(run.decisions_all),
+ external_systems=tuple(run.external_systems),
+ completed_page_summaries=dict(run.completed_page_summaries),
+ )
+ for spec in specs:
+ page_id = compute_page_id("onboarding", _onboarding.target_path(spec.slot))
+ if page_id in run.completed_ids:
+ continue
+ coros.append((page_id, gen.generate_onboarding_page(spec, signals)))
+ return coros
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
new file mode 100644
index 000000000..e46da8c16
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -0,0 +1,373 @@
+"""Level-by-level orchestration for ``PageGenerator.generate_all``.
+
+``run_generate_all`` builds a :class:`_GenerationRun` that holds the shared
+per-run state (graph metrics, selection allow-sets, job bookkeeping, the
+concurrency semaphore) and drives the ordered generation levels. The
+per-level coroutine builders live in ``levels.py`` and read this state object.
+
+Behaviour is identical to the previous single-method implementation; this is
+purely a structural split to satisfy the project's 400-line ceiling.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+from ..context_assembler import FilePageContext
+from ..models import GeneratedPage
+from . import levels as _levels
+from .helpers import (
+ _CODE_LANGUAGES,
+ _is_infra_file,
+ _select_clone_representatives,
+ build_dead_code_map,
+ build_decision_maps,
+ overview_summary,
+)
+from .tiering import partition_file_tiers
+
+if TYPE_CHECKING:
+ from .core import PageGenerator
+
+log = structlog.get_logger(__name__)
+
+
+class _GenerationRun:
+ """Mutable per-call state for one ``generate_all`` invocation."""
+
+ def __init__(
+ self,
+ gen: PageGenerator,
+ *,
+ parsed_files: list[Any],
+ source_map: dict[str, bytes],
+ graph_builder: Any,
+ repo_structure: Any,
+ repo_name: str,
+ job_system: Any | None,
+ on_page_done: Callable[[str], None] | None,
+ on_total_known: Callable[[int], None] | None,
+ on_subphase: Callable[[str, int | None], None] | None,
+ git_meta_map: dict[str, dict] | None,
+ resume: bool,
+ repo_path: Path | str | None,
+ dead_code_report: Any | None,
+ decision_report: Any | None,
+ external_systems: list[dict] | None,
+ ) -> None:
+ self.gen = gen
+ self.config = gen._config
+ self.vector_store = gen._vector_store
+ self.parsed_files = parsed_files
+ self.source_map = source_map
+ self.graph_builder = graph_builder
+ self.repo_structure = repo_structure
+ self.repo_name = repo_name
+ self.job_system = job_system
+ self.on_page_done = on_page_done
+ self.on_total_known = on_total_known
+ self.on_subphase = on_subphase
+ self.git_meta_map = git_meta_map
+ self.resume = resume
+ self.repo_path = repo_path
+ self.external_systems = external_systems or []
+
+ # ---- Graph metrics ----
+ self.graph = graph_builder.graph()
+ self.pagerank = graph_builder.pagerank()
+ self.betweenness = graph_builder.betweenness_centrality()
+ self.community = graph_builder.community_detection()
+ self.sccs = graph_builder.strongly_connected_components()
+
+ # ---- Per-file signal maps ----
+ self.dead_code_by_file = build_dead_code_map(dead_code_report)
+ self.decisions_by_file, self.decisions_all = build_decision_maps(decision_report)
+
+ # ---- Run bookkeeping ----
+ self.semaphore = asyncio.Semaphore(self.config.max_concurrency)
+ self.completed_page_summaries: dict[str, str] = {}
+ self.completed_ids: set[str] = set()
+ self.job_id: str | None = None
+ self.file_page_contexts: dict[str, FilePageContext] = {}
+
+ # Selection allow-sets (populated by _compute_selection).
+ self.selection: Any = None
+ self.code_files: list[Any] = []
+ self.sel_file_paths: set[str] = set()
+ self.sel_api_paths: set[str] = set()
+ self.sel_infra_paths: set[str] = set()
+ self.sel_module_groups: list[Any] = []
+ self.sel_scc_groups: list[Any] = []
+ self.tier1_paths: set[str] = set()
+ self.tier2_paths: set[str] = set()
+
+ # ------------------------------------------------------------------
+ # Setup
+ # ------------------------------------------------------------------
+
+ def _setup_job(self) -> None:
+ """Create the resume job and, on resume, seed completed page ids."""
+ if self.job_system is None:
+ return
+ repo_path_str = (
+ str(Path(self.repo_path).resolve())
+ if self.repo_path
+ else str(getattr(self.repo_structure, "root_path", "."))
+ )
+ # On resume, query the vector store directly — it is the ground truth.
+ if self.resume and self.vector_store is not None:
+ # Note: caller drives this synchronously enough; resume seeding is
+ # awaited in execute() to keep __init__ side-effect free.
+ pass
+ self.job_id = self.job_system.create_job(
+ repo_path_str,
+ self.config,
+ self.gen._provider.provider_name,
+ self.gen._provider.model_name,
+ )
+
+ async def _seed_resume(self) -> None:
+ if (
+ self.job_system is not None
+ and self.resume
+ and self.vector_store is not None
+ ):
+ self.completed_ids = await self.vector_store.list_page_ids()
+ if self.completed_ids:
+ log.info(
+ "Resuming generation from vector store",
+ already_completed=len(self.completed_ids),
+ )
+
+ def _compute_selection(self) -> None:
+ """Run the selection subsystem and derive the level allow-sets."""
+ code_files = [
+ p
+ for p in self.parsed_files
+ if not p.file_info.is_api_contract
+ and not _is_infra_file(p)
+ and p.file_info.language in _CODE_LANGUAGES
+ ]
+
+ # Near-clone dedupe runs before scoring so clone losers never consume
+ # scoring budget. Entry points are never dropped.
+ parsed_files_for_selection = self.parsed_files
+ if getattr(self.config, "dedupe_near_clones", True):
+ drop_paths = _select_clone_representatives(code_files, self.pagerank)
+ if drop_paths:
+ log.info("page_selection.clone_dedupe", dropped=len(drop_paths))
+ code_files = [p for p in code_files if p.file_info.path not in drop_paths]
+ parsed_files_for_selection = [
+ p for p in self.parsed_files if p.file_info.path not in drop_paths
+ ]
+
+ try:
+ community_info_map = self.graph_builder.community_info() or {}
+ except Exception:
+ community_info_map = {}
+
+ from ..selection import SelectionInputs, select_pages
+
+ selection = select_pages(
+ SelectionInputs(
+ parsed_files=parsed_files_for_selection,
+ pagerank=self.pagerank,
+ betweenness=self.betweenness,
+ community=self.community,
+ community_info=community_info_map,
+ sccs=list(self.sccs),
+ git_meta_map=self.git_meta_map,
+ config=self.config,
+ )
+ )
+
+ self.selection = selection
+ self.sel_file_paths = set(selection.file_page_paths)
+ self.sel_api_paths = set(selection.api_contract_paths)
+ self.sel_infra_paths = set(selection.infra_paths)
+ self.sel_module_groups = list(selection.module_groups)
+ self.sel_scc_groups = list(selection.scc_groups)
+
+ # Tiered doc generation: split the selected file pages into a full-LLM
+ # tier-1 and a deterministic template-only tier-2. When tier1_top_n is
+ # None this puts every selected page in tier-1 (no behaviour change).
+ self.tier1_paths, self.tier2_paths = partition_file_tiers(
+ self.sel_file_paths,
+ self.pagerank,
+ getattr(self.config, "tier1_top_n", None),
+ )
+
+ # Sort code_files for stable level-2 ordering: selected files first
+ # (so dep summaries land in the store earliest), then by PageRank desc.
+ self.code_files = sorted(
+ code_files,
+ key=lambda p: (
+ p.file_info.path not in self.sel_file_paths,
+ not p.file_info.is_entry_point,
+ -self.pagerank.get(p.file_info.path, 0.0),
+ ),
+ )
+
+ def _announce_total(self) -> None:
+ counts = self.selection.counts()
+ estimated_total = (
+ counts["api_contract"]
+ + counts["symbol_spotlight"]
+ + counts["file_page"]
+ + counts["scc_page"]
+ + counts["module_page"]
+ + int(self.selection.emit_repo_overview)
+ + int(self.selection.emit_arch_diagram)
+ + counts["infra_page"]
+ )
+ remaining_total = max(0, estimated_total - len(self.completed_ids))
+ if self.on_total_known is not None:
+ self.on_total_known(remaining_total)
+ if self.job_system is not None and self.job_id is not None:
+ self.job_system.start_job(self.job_id, estimated_total)
+
+ # ------------------------------------------------------------------
+ # Level runner
+ # ------------------------------------------------------------------
+
+ async def run_level(
+ self, named_coros: list[tuple[str, Any]], level: int
+ ) -> list[GeneratedPage]:
+ """Run one level's coroutines under the shared semaphore + embed batch."""
+ if self.job_system is not None and self.job_id is not None:
+ self.job_system.update_level(self.job_id, level)
+
+ # Pages finished during this level, collected for a single batched
+ # embed at the end. Embedding the whole wave in one call amortises the
+ # embedder round-trip and the level drains before the next level's RAG
+ # search runs, so there is no freshness regression.
+ embed_items: list[tuple[str, str, dict]] = []
+
+ async def guarded_named(page_id: str, coro: Any) -> Any:
+ try:
+ async with self.semaphore:
+ result = await coro
+
+ if isinstance(result, GeneratedPage):
+ # Summary capture is cheap (string ops) — keep inline so
+ # the next page's context assembly sees it immediately.
+ self.completed_page_summaries[result.target_path] = overview_summary(
+ result.content
+ )
+ # Progress tick fires the moment the page is ready.
+ if self.on_page_done is not None:
+ self.on_page_done(result.page_type)
+ if self.vector_store is not None:
+ embed_items.append(_embed_item(result))
+ return result
+ except Exception as exc:
+ if self.job_system is not None and self.job_id is not None:
+ self.job_system.fail_page(self.job_id, page_id, str(exc))
+ log.error(
+ "page_generation_failed",
+ page_id=page_id,
+ level=level,
+ error=str(exc),
+ )
+ return exc # return as value so gather works
+
+ tasks = [guarded_named(pid, c) for pid, c in named_coros]
+ results = await asyncio.gather(*tasks)
+ # Embed the whole level in one batch before declaring it done — the
+ # next level's RAG search depends on these landing in the store.
+ # Embedding is a RAG enhancement, not load-bearing, so failures are
+ # swallowed at debug level.
+ if embed_items and self.vector_store is not None:
+ try:
+ await self.vector_store.embed_batch(embed_items)
+ except Exception as e:
+ log.debug("rag.embed_batch_failed", count=len(embed_items), error=str(e))
+ pages = [r for r in results if isinstance(r, GeneratedPage)]
+ if self.job_system is not None and self.job_id is not None:
+ for r in pages:
+ self.job_system.complete_page(self.job_id, r.page_id)
+ return pages
+
+ # ------------------------------------------------------------------
+ # Orchestration
+ # ------------------------------------------------------------------
+
+ async def execute(self) -> list[GeneratedPage]:
+ self._setup_job()
+ await self._seed_resume()
+ self._compute_selection()
+ self._announce_total()
+
+ all_pages: list[GeneratedPage] = []
+
+ # Levels 0 (api_contract) + 1 (symbol_spotlight) share no data
+ # dependencies, so they run in one merged batch.
+ level01 = _levels.build_level01_coros(self)
+ all_pages.extend(await self.run_level(level01, 1))
+
+ # Level 2 (file_page) needs context assembly + topo ordering.
+ level2 = await _levels.build_level2_coros(self)
+ all_pages.extend(await self.run_level(level2, 2))
+
+ # Level 3 (scc_page).
+ all_pages.extend(await self.run_level(_levels.build_level3_coros(self), 3))
+
+ # Level 4 (module_page).
+ all_pages.extend(await self.run_level(_levels.build_level4_coros(self), 4))
+
+ # Levels 6 (repo_overview + architecture_diagram), 7 (infra_page),
+ # and 8 (onboarding) share no data dependencies — run merged.
+ final = (
+ _levels.build_level6_coros(self)
+ + _levels.build_level7_coros(self)
+ + _levels.build_level8_coros(self)
+ )
+ final_pages = await self.run_level(final, 8)
+ # Tag promoted onboarding slots (repo_overview / architecture_diagram).
+ self.gen._tag_promoted_pages(final_pages)
+ all_pages.extend(final_pages)
+
+ # Post-generation: resolve backtick refs into wiki links + backlinks.
+ try:
+ from ..interlinking import attach_wiki_links_and_backlinks
+
+ attach_wiki_links_and_backlinks(all_pages, self.parsed_files)
+ except Exception as exc:
+ log.debug("interlinking.failed", error=str(exc))
+
+ if self.job_system is not None and self.job_id is not None:
+ self.job_system.complete_job(self.job_id)
+
+ log.info(
+ "Generation complete",
+ total_pages=len(all_pages),
+ provider=self.gen._provider.provider_name,
+ model=self.gen._provider.model_name,
+ )
+ return all_pages
+
+
+def _embed_item(page: GeneratedPage) -> tuple[str, str, dict]:
+ """Build the ``(page_id, text, metadata)`` tuple for embedding."""
+ summary = overview_summary(page.content)
+ return (
+ page.page_id,
+ page.content,
+ {
+ "page_type": page.page_type,
+ "target_path": page.target_path,
+ "content": page.content[:600],
+ "summary": summary,
+ },
+ )
+
+
+async def run_generate_all(gen: PageGenerator, **kwargs: Any) -> list[GeneratedPage]:
+ """Entry point used by ``PageGenerator.generate_all``."""
+ run = _GenerationRun(gen, **kwargs)
+ return await run.execute()
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
new file mode 100644
index 000000000..71441cfdb
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -0,0 +1,325 @@
+"""Per-page-type generation methods, mixed into :class:`PageGenerator`.
+
+Each method assembles a context, renders its Jinja template into a user
+prompt, calls the provider, and wraps the response in a ``GeneratedPage``.
+They are grouped here (rather than inline on the generator) purely to keep
+each module under the project's 400-line ceiling.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+import structlog
+
+from repowise.core.ingestion.models import ParsedFile, RepoStructure
+
+from .. import onboarding as _onboarding
+from ..context_assembler import FilePageContext
+from ..models import GENERATION_LEVELS, GeneratedPage, compute_source_hash
+
+log = structlog.get_logger(__name__)
+
+
+class PerTypeGenerationMixin:
+ """Per-type ``generate_*`` methods. Requires the host to provide
+ ``_assembler``, ``_render``, ``_call_provider`` and
+ ``_build_generated_page`` (all supplied by :class:`PageGenerator`).
+ """
+
+ async def generate_file_page(
+ self,
+ parsed: ParsedFile,
+ graph: Any,
+ pagerank: dict[str, float],
+ betweenness: dict[str, float],
+ community: dict[str, int],
+ source_bytes: bytes,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_file_page(
+ parsed, graph, pagerank, betweenness, community, source_bytes
+ )
+ user_prompt = self._render("file_page.j2", ctx=ctx)
+ response = await self._call_provider(
+ "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
+ return self._build_generated_page(
+ "file_page",
+ parsed.file_info.path,
+ f"File: {parsed.file_info.path}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["file_page"],
+ )
+
+ async def generate_symbol_spotlight(
+ self,
+ symbol: Any,
+ parsed: ParsedFile,
+ pagerank: dict[str, float],
+ graph: Any,
+ source_map: dict[str, bytes] | None = None,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_symbol_spotlight(
+ symbol,
+ parsed,
+ pagerank,
+ graph,
+ source_bytes=(source_map or {}).get(parsed.file_info.path, b""),
+ )
+ user_prompt = self._render("symbol_spotlight.j2", ctx=ctx)
+ target = f"{parsed.file_info.path}::{symbol.name}"
+ response = await self._call_provider(
+ "symbol_spotlight", user_prompt, str(uuid.uuid4()), target_path=target
+ )
+ return self._build_generated_page(
+ "symbol_spotlight",
+ f"{parsed.file_info.path}::{symbol.name}",
+ f"Symbol: {symbol.qualified_name}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["symbol_spotlight"],
+ )
+
+ async def generate_module_page(
+ self,
+ module_path: str,
+ language: str,
+ file_contexts: list[FilePageContext],
+ graph: Any,
+ git_meta_map: dict[str, dict] | None = None,
+ page_summaries: dict[str, str] | None = None,
+ decision_records: list[dict] | None = None,
+ dead_code_findings: list[dict] | None = None,
+ external_systems: list[dict] | None = None,
+ community_label: str | None = None,
+ community_cohesion: float | None = None,
+ target_path: str | None = None,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_module_page(
+ module_path,
+ language,
+ file_contexts,
+ graph,
+ page_summaries=page_summaries,
+ git_meta_map=git_meta_map,
+ decision_records=decision_records,
+ dead_code_findings=dead_code_findings,
+ external_systems=external_systems,
+ community_label=community_label,
+ community_cohesion=community_cohesion,
+ )
+ module_git_summary = None
+ if git_meta_map:
+ from collections import Counter
+
+ file_paths = [fc.file_path for fc in file_contexts]
+ metas = [git_meta_map[f] for f in file_paths if f in git_meta_map]
+ if metas:
+ owner_counts = Counter(
+ m.get("primary_owner_name") for m in metas if m.get("primary_owner_name")
+ )
+ most_active = max(metas, key=lambda m: m.get("commit_count_90d", 0))
+ module_git_summary = {
+ "top_owners": [
+ {"name": n, "file_count": c} for n, c in owner_counts.most_common(3)
+ ],
+ "most_active_file": most_active.get("file_path", ""),
+ "most_active_commits_90d": most_active.get("commit_count_90d", 0),
+ }
+ user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)
+ page_target = target_path or module_path
+ response = await self._call_provider(
+ "module_page", user_prompt, str(uuid.uuid4()), target_path=page_target
+ )
+ return self._build_generated_page(
+ "module_page",
+ page_target,
+ f"Module: {module_path}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["module_page"],
+ )
+
+ async def generate_scc_page(
+ self,
+ scc_id: str,
+ scc_files: list[str],
+ file_contexts: list[FilePageContext],
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_scc_page(scc_id, scc_files, file_contexts)
+ user_prompt = self._render("scc_page.j2", ctx=ctx)
+ response = await self._call_provider(
+ "scc_page", user_prompt, str(uuid.uuid4()), target_path=scc_id
+ )
+ return self._build_generated_page(
+ "scc_page",
+ scc_id,
+ f"Circular Dependency: {scc_id}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["scc_page"],
+ )
+
+ async def generate_repo_overview(
+ self,
+ repo_structure: RepoStructure,
+ pagerank: dict[str, float],
+ sccs: list[Any],
+ community: dict[str, int],
+ git_meta_map: dict[str, dict] | None = None,
+ graph_builder: Any | None = None,
+ repo_name: str | None = None,
+ external_systems: list[dict] | None = None,
+ decision_records: list[dict] | None = None,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_repo_overview(
+ repo_structure,
+ pagerank,
+ sccs,
+ community,
+ graph_builder=graph_builder,
+ external_systems=external_systems,
+ decision_records=decision_records,
+ )
+ repo_git_summary = None
+ if git_meta_map:
+ metas = list(git_meta_map.values())
+ top_churn = sorted(metas, key=lambda m: m.get("commit_count_90d", 0), reverse=True)[:3]
+ oldest = min(
+ (m for m in metas if m.get("first_commit_at")),
+ key=lambda m: m["first_commit_at"],
+ default=None,
+ )
+ repo_git_summary = {
+ "hotspot_count": sum(1 for m in metas if m.get("is_hotspot")),
+ "stable_count": sum(1 for m in metas if m.get("is_stable")),
+ "top_churn_files": [m.get("file_path", "") for m in top_churn],
+ "oldest_file": oldest.get("file_path", "") if oldest else "",
+ "oldest_file_age_days": oldest.get("age_days", 0) if oldest else 0,
+ }
+ if not repo_name:
+ repo_name = getattr(repo_structure, "name", None) or "repo"
+ user_prompt = self._render("repo_overview.j2", ctx=ctx, repo_git_summary=repo_git_summary)
+ response = await self._call_provider(
+ "repo_overview", user_prompt, str(uuid.uuid4()), target_path=repo_name
+ )
+ return self._build_generated_page(
+ "repo_overview",
+ repo_name,
+ f"Repository Overview: {repo_name}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["repo_overview"],
+ )
+
+ async def generate_architecture_diagram(
+ self,
+ graph: Any,
+ pagerank: dict[str, float],
+ community: dict[str, int],
+ sccs: list[Any],
+ repo_name: str,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_architecture_diagram(
+ graph, pagerank, community, sccs, repo_name
+ )
+ user_prompt = self._render("architecture_diagram.j2", ctx=ctx)
+ response = await self._call_provider(
+ "architecture_diagram", user_prompt, str(uuid.uuid4()), target_path=repo_name
+ )
+ return self._build_generated_page(
+ "architecture_diagram",
+ repo_name,
+ f"Architecture Diagram: {repo_name}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["architecture_diagram"],
+ )
+
+ async def generate_api_contract(
+ self,
+ parsed: ParsedFile,
+ source_bytes: bytes,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_api_contract(parsed, source_bytes)
+ user_prompt = self._render("api_contract.j2", ctx=ctx)
+ response = await self._call_provider(
+ "api_contract", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
+ return self._build_generated_page(
+ "api_contract",
+ parsed.file_info.path,
+ f"API Contract: {parsed.file_info.path}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["api_contract"],
+ )
+
+ async def generate_onboarding_page(
+ self,
+ spec: _onboarding.SubkindSpec,
+ signals: _onboarding.OnboardingSignals,
+ ) -> GeneratedPage | None:
+ """Generate one onboarding page from a registered subkind spec.
+
+ Returns ``None`` when the subkind's gate fails (``build_context``
+ returned ``None``) — the slot is silently skipped for this repo.
+ """
+ ctx = spec.build_context(signals)
+ if ctx is None:
+ log.debug("onboarding.gate_skipped", slot=spec.slot)
+ return None
+
+ template_name = f"onboarding/{spec.template}"
+ user_prompt = self._render(template_name, ctx=ctx, slot=spec.slot)
+ target = _onboarding.target_path(spec.slot)
+ response = await self._call_provider(
+ "onboarding", user_prompt, str(uuid.uuid4()), target_path=target
+ )
+ page = self._build_generated_page(
+ "onboarding",
+ target,
+ spec.title,
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["onboarding"],
+ )
+ # Subkind discriminator lives in metadata; page_type alone is shared
+ # across all six generated onboarding slots.
+ page.metadata["subkind"] = spec.slot
+ page.metadata["onboarding_slot"] = spec.slot
+ return page
+
+ @staticmethod
+ def _tag_promoted_pages(pages: list[GeneratedPage]) -> None:
+ """Tag repo_overview / architecture_diagram pages with their slot.
+
+ Mutates each matching page's ``metadata["onboarding_slot"]`` so the
+ UI groups them into the Onboarding folder without changing their
+ underlying ``page_type``. Idempotent and tolerant of missing pages.
+ """
+ for page in pages:
+ slot = _onboarding.PROMOTED_SLOTS.get(page.page_type)
+ if slot is not None:
+ page.metadata["onboarding_slot"] = slot
+
+ async def generate_infra_page(
+ self,
+ parsed: ParsedFile,
+ source_bytes: bytes,
+ ) -> GeneratedPage:
+ ctx = self._assembler.assemble_infra_page(parsed, source_bytes)
+ user_prompt = self._render("infra_page.j2", ctx=ctx)
+ response = await self._call_provider(
+ "infra_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
+ return self._build_generated_page(
+ "infra_page",
+ parsed.file_info.path,
+ f"Infrastructure: {parsed.file_info.path}",
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["infra_page"],
+ )
diff --git a/packages/core/src/repowise/core/generation/page_generator/prompts.py b/packages/core/src/repowise/core/generation/page_generator/prompts.py
new file mode 100644
index 000000000..7e8e9dd5f
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/prompts.py
@@ -0,0 +1,119 @@
+"""System prompts and language metadata for the page generator.
+
+System prompts are module-level constants — the same string per page type on
+every call. This enables Anthropic server-side prefix caching.
+"""
+
+from __future__ import annotations
+
+_LANGUAGE_NAMES = {
+ "en": "English",
+ "ru": "Russian",
+ "es": "Spanish",
+ "fr": "French",
+ "de": "German",
+ "zh": "Chinese",
+ "ja": "Japanese",
+ "ko": "Korean",
+ "it": "Italian",
+ "pt": "Portuguese",
+ "nl": "Dutch",
+ "pl": "Polish",
+ "tr": "Turkish",
+ "ar": "Arabic",
+ "hi": "Hindi",
+}
+
+# ---------------------------------------------------------------------------
+# System prompts — one per page type (constant strings for prefix caching)
+# ---------------------------------------------------------------------------
+
+SYSTEM_PROMPTS: dict[str, str] = {
+ "file_page": (
+ "You are repowise, an expert technical documentation generator. "
+ "Your task is to produce comprehensive, accurate wiki pages from source code. "
+ "Output markdown only. Do not include preamble or apologies. "
+ "\n"
+ "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "
+ "state this file's job in one line. Use concrete architectural vocabulary "
+ "(orchestrator, entry point, parser, adapter, dispatcher, factory, resolver, "
+ "persistence layer, indexer, analyzer, etc.) and name what it produces or "
+ "consumes. This sentence is what semantic search and grep both anchor on, "
+ "so role keywords must appear at the very front — not buried in paragraph 3. "
+ "Bad: 'This file contains the X class which is used by Y.' "
+ "Good: 'X is the orchestrator for the indexing pipeline — it sequences "
+ "traversal, parsing, graph analysis, git enrichment, and persistence.' "
+ "If the user prompt lists Architectural signals (entry point, high PageRank, "
+ "bridge), weave them into that first sentence rather than restating them. "
+ "Required sections: ## Overview, ## Public API, ## Dependencies, ## Usage Notes."
+ ),
+ "symbol_spotlight": (
+ "You are repowise, an expert technical documentation generator. "
+ "Write a detailed spotlight page for a single code symbol. "
+ "Output markdown only. "
+ "Required sections: ## Purpose, ## Signature, ## Parameters, ## Returns, ## Example Usage."
+ ),
+ "module_page": (
+ "You are repowise, an expert technical documentation generator. "
+ "Write a module-level overview page summarising all files in the module. "
+ "Output markdown only. "
+ "\n"
+ "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "
+ "state the module's job in the larger system. Use architectural vocabulary "
+ "(ingestion subsystem, generation pipeline, persistence layer, transport "
+ "adapter, etc.) and name the inputs it consumes and the outputs it produces. "
+ "Bad: 'The X module contains 15 files responsible for...' "
+ "Good: 'The ingestion module is the entry stage of repowise's indexing "
+ "pipeline — it traverses a repository, parses files into ASTs, extracts "
+ "symbols, and yields ParsedFile objects for downstream analysis.' "
+ "Required sections: ## Overview, ## Public API Summary, ## Architecture Notes."
+ ),
+ "scc_page": (
+ "You are repowise, an expert technical documentation generator. "
+ "Document this circular dependency cycle and provide actionable refactoring advice. "
+ "Output markdown only. "
+ "Required sections: ## Cycle Description, ## Files Involved, ## Why This Exists, "
+ "## Refactoring Suggestions."
+ ),
+ "repo_overview": (
+ "You are repowise, an expert technical documentation generator. "
+ "Write a high-level repository overview suitable for onboarding new developers. "
+ "Output markdown only. "
+ "\n"
+ "CRITICAL — lead-with-purpose rule: the FIRST sentence under ## Project "
+ "Summary must answer 'what does this repository do, end-to-end?' in one "
+ "concrete sentence. Name the inputs, the pipeline, and the outputs in "
+ "architectural vocabulary. "
+ "Bad: 'This repository implements a documentation tool with 15 packages.' "
+ "Good: 'Repowise is a codebase documentation engine: it indexes a repository "
+ "by traversing files, parsing code into ASTs, analyzing dependencies, and "
+ "generating LLM-synthesised wiki pages served via MCP and a web UI.' "
+ "Required sections: ## Project Summary, ## Technology Stack, ## Entry Points, ## Architecture."
+ ),
+ "architecture_diagram": (
+ "You are repowise, an expert technical documentation generator. "
+ "Generate an architecture overview with a Mermaid diagram. "
+ "You MUST include a fenced mermaid block with graph TD showing key dependencies. "
+ "Output markdown only."
+ ),
+ "api_contract": (
+ "You are repowise, an expert technical documentation generator. "
+ "Document this API contract file for developers integrating with the service. "
+ "Output markdown only. "
+ "Required sections: ## Overview, ## Endpoints, ## Schemas, ## Authentication, ## Examples."
+ ),
+ "infra_page": (
+ "You are repowise, an expert technical documentation generator. "
+ "Document this infrastructure file for DevOps and platform engineers. "
+ "Output markdown only. "
+ "Required sections: ## Purpose, ## Key Targets/Stages, ## Configuration, ## Operational Notes."
+ ),
+ "onboarding": (
+ "You are repowise, an expert technical documentation generator producing "
+ "a single page in a curated Onboarding collection that a new contributor "
+ "or LLM agent reads first. "
+ "Write concise, navigable prose grounded in the structured signals supplied. "
+ "Do not invent file paths, symbol names, or rationale that is not in the context. "
+ "Output markdown only — follow the exact section structure the user prompt prescribes."
+ ),
+}
diff --git a/packages/core/src/repowise/core/generation/page_generator/tiering.py b/packages/core/src/repowise/core/generation/page_generator/tiering.py
new file mode 100644
index 000000000..0480d9c1a
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/tiering.py
@@ -0,0 +1,46 @@
+"""Tiered doc-generation partition logic.
+
+Splits the set of selected file pages into two tiers for large repos:
+
+* **Tier-1** — the top ``tier1_top_n`` files by PageRank get full LLM
+ generation (the existing, unchanged path).
+* **Tier-2** — the long tail is rendered from a deterministic Jinja
+ template and embedded for search, with no LLM call.
+
+When ``tier1_top_n`` is ``None`` (the default) every selected file page is
+tier-1, so behaviour is identical to the pre-tiering generator.
+"""
+
+from __future__ import annotations
+
+
+def partition_file_tiers(
+ selected_paths: set[str],
+ pagerank: dict[str, float],
+ tier1_top_n: int | None,
+) -> tuple[set[str], set[str]]:
+ """Split selected file-page paths into (tier1, tier2) sets.
+
+ Args:
+ selected_paths: Allow-set of file paths chosen for a ``file_page``.
+ pagerank: File-level PageRank scores (higher = more central).
+ tier1_top_n: Cap on the number of full-LLM (tier-1) pages. ``None``
+ or a value >= len(selected_paths) puts every page in
+ tier-1, exactly reproducing the prior behaviour.
+
+ Returns:
+ ``(tier1_paths, tier2_paths)`` — a partition of ``selected_paths``.
+ """
+ if tier1_top_n is None or tier1_top_n >= len(selected_paths):
+ return set(selected_paths), set()
+ if tier1_top_n <= 0:
+ return set(), set(selected_paths)
+
+ # Rank by PageRank desc, then path for a deterministic tie-break.
+ ranked = sorted(
+ selected_paths,
+ key=lambda p: (-pagerank.get(p, 0.0), p),
+ )
+ tier1 = set(ranked[:tier1_top_n])
+ tier2 = set(ranked[tier1_top_n:])
+ return tier1, tier2
diff --git a/packages/core/src/repowise/core/generation/page_generator/validation.py b/packages/core/src/repowise/core/generation/page_generator/validation.py
new file mode 100644
index 000000000..ffa27b461
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_generator/validation.py
@@ -0,0 +1,233 @@
+"""LLM-output validation — hallucination detection for generated pages.
+
+Cross-checks backtick-quoted names in LLM output against the symbols,
+exports, and imports of the source ``ParsedFile`` so that references the
+model invented (but that do not exist in the AST) can be surfaced.
+"""
+
+from __future__ import annotations
+
+import re
+
+from repowise.core.ingestion.models import ParsedFile
+
+# Common words that appear in backticks but are not code symbols.
+_BACKTICK_SKIP = frozenset(
+ {
+ # Python builtins & keywords
+ "True",
+ "False",
+ "None",
+ "self",
+ "cls",
+ "super",
+ "str",
+ "int",
+ "float",
+ "bool",
+ "list",
+ "dict",
+ "set",
+ "tuple",
+ "bytes",
+ "object",
+ "type",
+ "Any",
+ "Optional",
+ "Union",
+ "async",
+ "await",
+ "return",
+ "yield",
+ "import",
+ "from",
+ "class",
+ "def",
+ "if",
+ "else",
+ "for",
+ "while",
+ "try",
+ "except",
+ "raise",
+ "with",
+ "pass",
+ "break",
+ "continue",
+ "lambda",
+ "in",
+ "not",
+ "and",
+ "or",
+ "is",
+ "del",
+ "assert",
+ "finally",
+ "elif",
+ "as",
+ "global",
+ "nonlocal",
+ # JS/TS keywords
+ "null",
+ "undefined",
+ "this",
+ "const",
+ "let",
+ "var",
+ "function",
+ "export",
+ "default",
+ "extends",
+ "implements",
+ "interface",
+ "enum",
+ "new",
+ "typeof",
+ "instanceof",
+ "void",
+ "never",
+ "string",
+ "number",
+ "boolean",
+ "symbol",
+ "bigint",
+ "unknown",
+ "readonly",
+ "abstract",
+ "static",
+ "private",
+ "protected",
+ "public",
+ "require",
+ "module",
+ "exports",
+ "Promise",
+ "Map",
+ "Set",
+ "Array",
+ "Object",
+ "Error",
+ "Date",
+ "RegExp",
+ "JSON",
+ "Math",
+ "console",
+ # Common tool/ecosystem names
+ "pip",
+ "npm",
+ "npx",
+ "yarn",
+ "pnpm",
+ "go",
+ "rust",
+ "python",
+ "node",
+ "cargo",
+ "uv",
+ "git",
+ "docker",
+ "make",
+ # Common framework/lib names the LLM mentions in prose
+ "FastAPI",
+ "React",
+ "Next",
+ "Express",
+ "Django",
+ "Flask",
+ "SQLAlchemy",
+ "Pydantic",
+ "Click",
+ "Typer",
+ "pytest",
+ "asyncio",
+ "pathlib",
+ "dataclass",
+ "dataclasses",
+ }
+)
+
+# Regex: single-backtick references that look like identifiers.
+_BACKTICK_REF_RE = re.compile(r"(?<!`)` *([A-Za-z_]\w*(?:\.\w+)*) *`(?!`)")
+
+# Patterns that indicate the backtick content is a path, command, or
+# value rather than a symbol reference — these should never be flagged.
+_PATH_OR_CMD_RE = re.compile(
+ r"[/\\]" # contains path separator
+ r"|\.(?:py|ts|js|json|yaml|yml|toml|md|sh|sql|css|html)$" # file extension
+ r"|^[a-z][\w-]*$" # all-lowercase with hyphens = CLI command/flag
+)
+
+
+def _validate_symbol_references(
+ content: str,
+ parsed: ParsedFile,
+) -> list[str]:
+ """Cross-check backtick-quoted names in LLM output against actual symbols.
+
+ Returns a list of warning strings for references that don't match any
+ known symbol, export, or import in the ParsedFile. Designed to have low
+ false-positive rates — only flags references that look like symbol names
+ but can't be found anywhere in the file's AST, imports, or source text.
+ """
+ refs = set(_BACKTICK_REF_RE.findall(content))
+ if not refs:
+ return []
+
+ # Build the known-names set from AST data
+ known: set[str] = set()
+ for s in parsed.symbols:
+ known.add(s.name)
+ known.add(s.qualified_name)
+ # Decorator names are valid references (e.g. @app.command("init"))
+ for dec in s.decorators:
+ # Extract the decorator function name: "@app.command" → "command"
+ dec_name = dec.lstrip("@").split("(")[0]
+ known.add(dec_name)
+ known.add(dec_name.split(".")[-1])
+ known.update(parsed.exports)
+ for imp in parsed.imports:
+ if imp.module_path:
+ # Add both the final component and intermediate segments
+ parts = imp.module_path.split(".")
+ known.update(parts)
+ known.update(imp.imported_names)
+ # Named bindings from import resolution
+ for binding in getattr(imp, "bindings", []):
+ known.add(binding.local_name)
+ if binding.exported_name:
+ known.add(binding.exported_name)
+
+ # Also add all string literals from the source that look like identifiers
+ # (catches Click command names, decorator arguments, dict keys, etc.)
+ # The source is in the context, but we only have the parsed file here.
+ # Use docstring and symbol names as a cheap approximation.
+ if (
+ hasattr(parsed, "file_info")
+ and hasattr(parsed.file_info, "path")
+ and parsed.docstring
+ ):
+ known.update(w for w in parsed.docstring.split() if w.isidentifier())
+
+ warnings: list[str] = []
+ for ref in refs:
+ if ref in _BACKTICK_SKIP:
+ continue
+ # Skip short refs (1-2 chars are usually variables like `x`, `i`, `db`)
+ if len(ref) <= 2:
+ continue
+ # Skip anything that looks like a path, file, or CLI command
+ if _PATH_OR_CMD_RE.search(ref):
+ continue
+ # Skip all-uppercase (likely constants from other files: `MAX_RETRIES`)
+ if ref.isupper():
+ continue
+ # Check against known names
+ base = ref.split(".")[-1]
+ if ref in known or base in known:
+ continue
+ # Skip if the ref is a substring of any known symbol (covers partial
+ # references like `parse` when `parse_file` exists)
+ if any(ref in k for k in known if len(k) > len(ref)):
+ continue
+ warnings.append(ref)
+ return warnings
diff --git a/packages/core/src/repowise/core/generation/templates/file_page_tier2.j2 b/packages/core/src/repowise/core/generation/templates/file_page_tier2.j2
new file mode 100644
index 000000000..9d92d9487
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/file_page_tier2.j2
@@ -0,0 +1,45 @@
+{#- Deterministic (no-LLM) tier-2 file page.
+
+ Rendered straight from the assembled FilePageContext for the long tail
+ of selected files on large repos. No model call — the content is a
+ structured, factual stub built entirely from parsed AST + graph signals,
+ and is embedded for search like any other page.
+-#}
+# {{ ctx.file_path }}
+
+## Overview
+
+{% if ctx.docstring %}{{ ctx.docstring | trim }}
+{% else %}`{{ ctx.file_path }}` is a {{ ctx.language }} {% if ctx.is_entry_point %}entry-point {% endif %}{% if ctx.is_test %}test {% endif %}source file in this repository.{% endif %}
+
+{% set public_syms = ctx.symbols | selectattr("visibility", "equalto", "public") | list %}
+{% if public_syms %}It exposes {{ public_syms | length }} public symbol{{ "s" if public_syms | length != 1 else "" }}{% if ctx.dependencies %} and depends on {{ ctx.dependencies | length }} internal module{{ "s" if ctx.dependencies | length != 1 else "" }}{% endif %}.{% endif %}
+
+## Public API
+
+{% if public_syms %}
+| Symbol | Kind | Signature |
+| --- | --- | --- |
+{% for s in public_syms -%}
+| `{{ s.name }}` | {{ s.kind }} | {{ (s.signature or "") | replace("|", "\\|") | replace("\n", " ") | truncate(120, true, "…") }} |
+{% endfor %}
+{% else %}
+_No public symbols were extracted from this file._
+{% endif %}
+
+## Dependencies
+
+{% if ctx.dependencies %}
+{% for dep in ctx.dependencies -%}
+- `{{ dep }}`
+{% endfor %}
+{% else %}
+_No internal dependencies resolved._
+{% endif %}
+
+## Usage Notes
+
+{% if ctx.dependents %}Imported by {{ ctx.dependents | length }} file{{ "s" if ctx.dependents | length != 1 else "" }} in this repository.{% else %}No internal callers were resolved for this file.{% endif %}
+{% if ctx.community_label %} Part of the **{{ ctx.community_label }}** module cluster.{% endif %}
+
+<!-- Generated deterministically (tier-2, no LLM). -->
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer.py
deleted file mode 100644
index 54b85532c..000000000
--- a/packages/core/src/repowise/core/ingestion/git_indexer.py
+++ /dev/null
@@ -1,1070 +0,0 @@
-"""Git history indexer for the repowise ingestion pipeline.
-
-Mines git history into the git_metadata table. Uses gitpython for git
-operations. Parallelizes per-file git log calls with asyncio.Semaphore(20).
-
-Non-blocking: if git is unavailable or repo has no history, log a warning
-and return an empty summary. All downstream features degrade gracefully.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import enum
-import json
-import math
-import re
-import time
-from collections import Counter, defaultdict
-from collections.abc import Callable
-from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-from typing import Any
-
-import structlog
-
-from .languages.registry import REGISTRY as _LANG_REGISTRY
-
-logger = structlog.get_logger(__name__)
-
-
-class GitIndexTier(enum.StrEnum):
- """Public knob for git-indexing depth on large repositories.
-
- ``FULL`` (the default) preserves today's behavior — last *commit_depth*
- commits with co-change accumulation and extended blame. ``ESSENTIAL``
- runs only the cheap baseline (last commit, no co-change graph) and is
- intended for the fast orchestrator path; the implementation that
- actually switches on this tier is delivered in a follow-up phase
- addressing large-repo scale. The enum is published in this revision so
- plugin authors and downstream tooling can target it now.
- """
-
- ESSENTIAL = "essential"
- FULL = "full"
-
-# Silence GitPython's _CatFileContentStream.__del__ ValueError spam.
-# When git cat-file streams are GC'd after the subprocess pipe is closed,
-# their __del__ tries to drain remaining bytes and hits a closed file.
-# This is harmless but floods stderr with tracebacks.
-try:
- from git.cmd import _CatFileContentStream
-
- _orig_del = _CatFileContentStream.__del__
-
- def _quiet_del(self: Any) -> None:
- with contextlib.suppress(ValueError, OSError):
- _orig_del(self)
-
- _CatFileContentStream.__del__ = _quiet_del # type: ignore[assignment]
-except Exception:
- pass # git not installed — nothing to patch
-
-# Commit message prefixes that are ALWAYS skipped (no signal value).
-_HARD_SKIP_PREFIXES = ("Merge ",)
-
-# Conventional-commit prefixes normally skipped — but kept if the message
-# contains a decision-signal keyword (e.g. "build: migrate from webpack to vite").
-_SOFT_SKIP_PREFIXES = ("Bump ", "chore:", "ci:", "style:", "build:", "release:")
-
-# Lightweight subset of decision-signal keywords (mirrors decision_extractor.py).
-# Used to rescue soft-skipped commits that carry architectural intent.
-_DECISION_SIGNAL_WORDS: frozenset[str] = frozenset(
- {
- "migrate",
- "migration",
- "switch to",
- "replace",
- "refactor",
- "adopt",
- "introduce",
- "deprecate",
- "remove",
- "upgrade",
- "rewrite",
- "extract",
- "convert",
- "transition",
- }
-)
-
-_SKIP_AUTHORS = ("dependabot", "renovate", "github-actions")
-_MIN_MESSAGE_LEN = 12
-
-# Default per-file commit history depth.
-_DEFAULT_COMMIT_LIMIT: int = 500
-
-# Per-file persisted contributor / commit fan-out. Previously hard-coded
-# to 5 / 10 inline, which silently hid co-owners and meaningful history on
-# multi-team modules. 50 is generous enough that any realistic UI surface
-# can render the full list while keeping the JSON blob bounded.
-_MAX_TOP_AUTHORS: int = 50
-_MAX_SIGNIFICANT_COMMITS: int = 50
-
-# Co-change pair extraction widens the window because individual files
-# may only co-change a handful of times in 500 commits — well below the
-# ``min_count`` threshold. On low-churn repos the 500-commit window
-# produced 0 co-change pairs every run; 2000 commits captures enough
-# history for the decay-weighted score to clear the bar without
-# meaningfully blowing up wall-clock time (single `git log` call).
-_DEFAULT_CO_CHANGE_COMMIT_LIMIT: int = 2000
-
-# Minimum decay-weighted co-occurrence weight for a pair to be recorded.
-# Was 3 historically; on repos with sparse change history (libraries,
-# stable services) that produced empty co-change tables. Two recent
-# co-changes is enough signal to surface in the UI, and the dashboard
-# already sorts partners by weight so the ranking is unaffected.
-_DEFAULT_CO_CHANGE_MIN_COUNT: int = 2
-
-# Commits that touch a very large number of files (mass renames,
-# copyright header sweeps, code-mod runs) produce O(N^2) pairs and
-# contribute no useful co-change signal. Skip pair generation for any
-# commit above this threshold. The decay-weighted score already
-# de-prioritises mass-edit commits, but the pairs are materialised
-# first — for a worst-case 500 files/commit × 2000 commits run that's
-# 250M pairs ≈ 16 GB RAM. The cap is a memory safeguard, not a
-# correctness change: typical commits sit well under 20 files.
-_MAX_FILES_PER_COMMIT_FOR_COCHANGE: int = 200
-
-# Commit message classification regexes (Phase 2.2).
-_COMMIT_CATEGORIES: dict[str, re.Pattern[str]] = {
- "feature": re.compile(
- r"\b(add|implement|introduce|create|new|feat)\b",
- re.IGNORECASE,
- ),
- "refactor": re.compile(
- r"\b(refactor|restructure|cleanup|clean.up|rename|reorganize|extract|simplify|move)\b",
- re.IGNORECASE,
- ),
- "fix": re.compile(
- r"\b(fix|bug|patch|hotfix|revert|regression|broken|crash|error)\b",
- re.IGNORECASE,
- ),
- "dependency": re.compile(
- r"\b(upgrade|bump|update.dep|migrate.to|switch.to|dependency|dependencies)\b",
- re.IGNORECASE,
- ),
-}
-
-# Co-change temporal decay: half-life ~125 days (lambda for exp(-t/tau)).
-_CO_CHANGE_DECAY_TAU: float = 180.0
-
-# Hotspot temporal decay: half-life for exponentially weighted churn score.
-HOTSPOT_HALFLIFE_DAYS: float = 180.0
-
-# Regex to extract PR/MR numbers from commit messages.
-# Matches: "#123", "Merge pull request #456", "(#789)", "!42" (GitLab MR)
-_PR_NUMBER_RE = re.compile(r"(?:pull request |)\#(\d+)|\(#(\d+)\)|!(\d+)")
-
-# Allowlist of extensions for which per-file git indexing (blame, commit
-# history, hotspot/stable classification) is worth running. Anything NOT in
-# this set is skipped — data, config, markup, dotfiles, and binaries add no
-# documentation value, and git blame on large JSON/YAML files is very slow.
-# Co-change detection still runs across ALL tracked files regardless.
-# Derived from the centralised LanguageRegistry — all code language extensions.
-_CODE_EXTENSIONS: frozenset[str] = _LANG_REGISTRY.all_code_extensions()
-
-# Files larger than this skip git blame. blame is O(lines) and blocks the
-# executor thread — for large files the commit-based ownership estimate is
-# used as a fallback instead.
-_MAX_BLAME_SIZE_BYTES: int = 100 * 1024 # 100 KB
-
-# Maximum seconds to wait for a single file's git indexing. If exceeded the
-# file is recorded with whatever data was collected before the timeout and the
-# semaphore slot is released so other files can proceed.
-_FILE_INDEX_TIMEOUT_SECS: float = 45.0
-
-
-@dataclass
-class _CommitRec:
- """Lightweight commit record parsed from ``git log --numstat``."""
-
- sha: str
- author_name: str
- author_email: str
- ts: int # unix epoch
- is_merge: bool
- subject: str
- added: int = 0
- deleted: int = 0
-
-
-_RENAME_RE = re.compile(r"\{(.+?) => (.+?)\}")
-
-
-def _extract_rename_paths(stat_path: str, known_paths: set[str]) -> tuple[str | None, str | None]:
- """Extract old/new paths from a git numstat rename line and add to *known_paths*.
-
- Git ``--numstat`` with ``--follow`` emits rename lines like::
-
- 10\t5\t{old => new}/shared_suffix
- 10\t5\told_dir/{old_name => new_name}.py
-
- This helper parses both forms, adds both expanded paths to *known_paths*,
- and returns ``(old_path, new_path)`` so the caller can attribute churn to
- the correct file. Returns ``(None, None)`` if the pattern is not found.
- """
- m = _RENAME_RE.search(stat_path)
- if m:
- prefix = stat_path[: m.start()]
- suffix = stat_path[m.end() :]
- old_path = prefix + m.group(1) + suffix
- new_path = prefix + m.group(2) + suffix
- known_paths.add(old_path)
- known_paths.add(new_path)
- return old_path, new_path
- return None, None
-
-
-def _should_skip_index(file_path: str) -> bool:
- """Return True for files where per-file git indexing should be skipped.
-
- Uses an allowlist: only files with known source-code extensions are indexed.
- Everything else (data, config, markup, dotfiles, binaries) is skipped.
- """
- return Path(file_path).suffix.lower() not in _CODE_EXTENSIONS
-
-
-@dataclass
-class GitIndexSummary:
- files_indexed: int
- hotspots: int
- stable_files: int
- duration_seconds: float = 0.0
-
-
-class GitIndexer:
- """Mines git history into the git_metadata table.
-
- Uses gitpython (already a dependency) for git operations.
- Parallelizes per-file git log calls with asyncio.Semaphore(20).
-
- Non-blocking: if git is unavailable or repo has no history, log a warning
- and return an empty summary. All downstream features degrade gracefully.
- """
-
- def __init__(
- self,
- repo_path: str | Path,
- *,
- commit_limit: int | None = None,
- follow_renames: bool = False,
- ) -> None:
- self.repo_path = Path(repo_path)
- self.commit_limit = commit_limit or _DEFAULT_COMMIT_LIMIT
- self.follow_renames = follow_renames
-
- async def index_repo(
- self,
- repo_id: str,
- on_start: Callable[[int], None] | None = None,
- on_file_done: Callable[[], None] | None = None,
- on_commit_done: Callable[[], None] | None = None,
- on_co_change_start: Callable[[int], None] | None = None,
- on_co_change_done: Callable[[], None] | None = None,
- ) -> tuple[GitIndexSummary, list[dict]]:
- """Full index of all tracked files. Returns summary + list of metadata dicts
- ready for bulk upsert.
-
- Optional progress callbacks (all thread-safe, called from the event loop
- or executor threads):
- on_start(total) — fired once with the total number of tracked files
- on_file_done() — fired after each file is indexed
- on_co_change_start(total) — fired once with actual commit count for
- co-change analysis
- on_commit_done() — fired after each commit is processed during
- co-change analysis
- on_co_change_done() — fired the moment co-change accumulation
- completes (BEFORE per-file git indexing
- finishes). Lets callers stop a co_change
- phase timer without including the wallclock
- cost of the still-running per-file walk.
- """
- start = time.monotonic()
- repo = self._get_repo()
- if repo is None:
- return GitIndexSummary(0, 0, 0, 0.0), []
-
- tracked_files = self._get_tracked_files(repo)
- if not tracked_files:
- return GitIndexSummary(0, 0, 0, 0.0), []
-
- # Only run expensive per-file indexing (git log + blame) on code files.
- # Data/config/markup files are skipped here but still passed to
- # _compute_co_changes so co-change relationships remain complete.
- indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]
-
- if on_start is not None:
- on_start(len(indexable_files))
-
- # Use a dedicated executor so timed-out threads don't block
- # asyncio.run() cleanup (the default executor waits for ALL threads).
- from concurrent.futures import ThreadPoolExecutor
-
- executor = ThreadPoolExecutor(max_workers=20, thread_name_prefix="git-idx")
- semaphore = asyncio.Semaphore(20)
- loop = asyncio.get_event_loop()
-
- # Build a repo-wide commit index in ONE git log subprocess when
- # rename-tracking is off (the default). Each per-file worker then
- # reads its commits from this shared dict instead of spawning its
- # own ``git log -- <file>`` — turning O(files) process spawns
- # into O(1) on large repos.
- commit_index: dict[str, list[_CommitRec]] = {}
- if not self.follow_renames:
- from .git_commit_index import load_commit_index
-
- commit_index = load_commit_index(
- repo, self.commit_limit, set(indexable_files)
- )
-
- def _index_one_sync(file_path: str) -> dict:
- """Use a per-thread Repo to avoid shared-handle issues on Windows."""
- try:
- import git as gitpython
-
- thread_repo = gitpython.Repo(
- self.repo_path,
- search_parent_directories=True,
- )
- try:
- precomputed = (
- commit_index.get(file_path) if commit_index else None
- )
- return self._index_file(
- file_path, thread_repo, precomputed_commits=precomputed
- )
- finally:
- thread_repo.close()
- except Exception:
- return {"file_path": file_path}
-
- async def index_one(file_path: str) -> dict:
- async with semaphore:
- try:
- result = await asyncio.wait_for(
- loop.run_in_executor(executor, _index_one_sync, file_path),
- timeout=_FILE_INDEX_TIMEOUT_SECS,
- )
- except TimeoutError:
- logger.debug(
- "Git indexing timed out for file — using partial data",
- path=file_path,
- timeout=_FILE_INDEX_TIMEOUT_SECS,
- )
- result = {"file_path": file_path}
- except Exception as exc:
- logger.debug(
- "Git indexing failed for file",
- path=file_path,
- error=str(exc),
- )
- result = {"file_path": file_path}
- if on_file_done is not None:
- on_file_done()
- return result
-
- # Run per-file indexing and co-change analysis in parallel — they are
- # independent (co-change only needs tracked_files, not per-file results).
- file_tasks = [index_one(fp) for fp in indexable_files]
-
- async def _co_change_task() -> dict[str, list[dict]]:
- # Co-change uses a wider window than per-file history (set
- # by ``self.commit_limit``) so low-churn repos surface
- # any pairs at all. Defaults live in module constants so
- # tests and re-index paths can override without touching
- # this call site.
- result = await loop.run_in_executor(
- executor,
- self._compute_co_changes,
- repo,
- set(tracked_files),
- max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),
- _DEFAULT_CO_CHANGE_MIN_COUNT,
- on_commit_done,
- on_co_change_start,
- )
- # Fire the done callback the instant accumulation finishes so
- # any co_change phase timer stops here, NOT after the parallel
- # per-file git walk also completes (it can run for many more
- # minutes on large repos).
- if on_co_change_done is not None:
- try:
- on_co_change_done()
- except Exception:
- pass
- return result
-
- metadata_list, co_changes = await asyncio.gather(
- asyncio.gather(*file_tasks, return_exceptions=True),
- _co_change_task(),
- )
-
- # Abandon any timed-out threads immediately instead of letting
- # asyncio.run() block for minutes during default-executor cleanup.
- executor.shutdown(wait=False, cancel_futures=True)
-
- # Filter out failures
- results: list[dict] = []
- for r in metadata_list:
- if isinstance(r, Exception):
- logger.warning("Failed to index file", error=str(r))
- else:
- results.append(r)
-
- # Merge co-change partners into per-file metadata
- for meta in results:
- fp = meta["file_path"]
- if fp in co_changes:
- meta["co_change_partners_json"] = json.dumps(co_changes[fp])
-
- # Compute percentiles
- self._compute_percentiles(results)
-
- duration = time.monotonic() - start
- hotspots = sum(1 for m in results if m.get("is_hotspot", False))
- stable = sum(1 for m in results if m.get("is_stable", False))
-
- summary = GitIndexSummary(
- files_indexed=len(results),
- hotspots=hotspots,
- stable_files=stable,
- duration_seconds=duration,
- )
- # Explicitly close the Repo to shut down the persistent git cat-file
- # process. Without this, garbage-collected _CatFileContentStream objects
- # try to read from an already-closed pipe and spam ValueError tracebacks.
- repo.close()
-
- logger.info(
- "Git indexing complete",
- files=summary.files_indexed,
- hotspots=summary.hotspots,
- stable=summary.stable_files,
- duration=f"{summary.duration_seconds:.1f}s",
- )
- return summary, results
-
- async def index_changed_files(self, changed_file_paths: list[str]) -> list[dict]:
- """Incremental update: re-index only changed files.
- Also re-index any file whose co_change_partners include a changed file.
- """
- repo = self._get_repo()
- if repo is None:
- return []
-
- loop = asyncio.get_event_loop()
- semaphore = asyncio.Semaphore(20)
-
- def _index_one_sync(file_path: str) -> dict:
- """Use a per-thread Repo to avoid shared-handle issues on Windows."""
- try:
- import git as gitpython
-
- thread_repo = gitpython.Repo(
- self.repo_path,
- search_parent_directories=True,
- )
- try:
- return self._index_file(file_path, thread_repo)
- finally:
- thread_repo.close()
- except Exception:
- return {"file_path": file_path}
-
- async def index_one(file_path: str) -> dict:
- async with semaphore:
- try:
- return await asyncio.wait_for(
- loop.run_in_executor(None, _index_one_sync, file_path),
- timeout=_FILE_INDEX_TIMEOUT_SECS,
- )
- except (TimeoutError, Exception) as exc:
- logger.debug(
- "Git indexing failed for changed file",
- path=file_path,
- error=str(exc),
- )
- return {"file_path": file_path}
-
- tasks = [index_one(fp) for fp in changed_file_paths]
- results_raw = await asyncio.gather(*tasks, return_exceptions=True)
-
- results: list[dict] = []
- for r in results_raw:
- if isinstance(r, Exception):
- logger.warning("Failed to index changed file", error=str(r))
- else:
- results.append(r)
-
- repo.close()
- return results
-
- # ------------------------------------------------------------------
- # Internal methods
- # ------------------------------------------------------------------
-
- def _get_repo(self) -> Any | None:
- try:
- import git as gitpython
-
- return gitpython.Repo(self.repo_path, search_parent_directories=True)
- except Exception as exc:
- logger.warning(
- "Git unavailable or not a repository",
- path=str(self.repo_path),
- error=str(exc),
- )
- return None
-
- def _get_tracked_files(self, repo: Any) -> list[str]:
- try:
- output = repo.git.ls_files()
- return [f for f in output.splitlines() if f.strip()]
- except Exception as exc:
- logger.warning("Failed to list tracked files", error=str(exc))
- return []
-
- def _index_file(
- self,
- file_path: str,
- repo: Any,
- precomputed_commits: list[_CommitRec] | None = None,
- ) -> dict:
- """Index a single file's git history. Runs in executor.
-
- Uses a single ``git log --numstat`` call to collect commit metadata,
- line-churn stats, and merge-commit flag in one subprocess instead of
- the previous three separate calls (_get_commits, _get_line_stats,
- _get_merge_commit_count).
-
- When *precomputed_commits* is provided (the default when called
- from the batched full-repo path), the per-file ``git log``
- subprocess is skipped entirely — eliminating the dominant
- process-spawn cost on large repos.
- """
- now = datetime.now(UTC)
- ninety_days_ago = now - timedelta(days=90)
- thirty_days_ago = now - timedelta(days=30)
- ninety_days_ago_ts = ninety_days_ago.timestamp()
- thirty_days_ago_ts = thirty_days_ago.timestamp()
-
- meta: dict[str, Any] = {
- "file_path": file_path,
- "commit_count_total": 0,
- "commit_count_90d": 0,
- "commit_count_30d": 0,
- "commit_count_capped": False,
- "first_commit_at": None,
- "last_commit_at": None,
- "primary_owner_name": None,
- "primary_owner_email": None,
- "primary_owner_commit_pct": None,
- "top_authors_json": "[]",
- "significant_commits_json": "[]",
- "co_change_partners_json": "[]",
- "commit_categories_json": "{}",
- "is_hotspot": False,
- "is_stable": False,
- "churn_percentile": 0.0,
- "age_days": 0,
- # Phase 2 fields
- "lines_added_90d": 0,
- "lines_deleted_90d": 0,
- "avg_commit_size": 0.0,
- "recent_owner_name": None,
- "recent_owner_commit_pct": None,
- "bus_factor": 0,
- "contributor_count": 0,
- # Phase 3 fields
- "original_path": None,
- "merge_commit_count_90d": 0,
- # Temporal hotspot score (exponentially decayed churn)
- "temporal_hotspot_score": 0.0,
- }
-
- orig_path: str | None = None
- if precomputed_commits is not None:
- commits: list[_CommitRec] = precomputed_commits
- else:
- try:
- # Single git log call: header line + optional numstat lines per commit.
- # Format: NUL-delimited record per commit so we can split reliably.
- # Fields: sha, author name, author email, unix timestamp, parent SHAs, subject
- log_args: list[str] = []
- if self.follow_renames:
- log_args.append("--follow")
- log_args += [
- f"-{self.commit_limit}",
- "--numstat",
- "--format=%x00%H%x1f%an%x1f%ae%x1f%ct%x1f%P%x1f%s",
- "--",
- file_path,
- ]
- raw = repo.git.log(*log_args)
- except Exception:
- return meta
-
- if not raw.strip():
- return meta
-
- # Parse records — each starts with a \x00 marker line followed by
- # zero-or-more numstat lines (added\tdeleted\tpath).
- # When --follow is active, older commits may reference previous file
- # names so we track all names seen via rename markers ("{old => new}").
- known_paths: set[str] = {file_path}
- # When --follow is active, seed known_paths with the original path
- # so that numstat lines referencing the old name (without a rename
- # marker in the log window) are still counted.
- if self.follow_renames:
- orig_path = self._detect_original_path(file_path, repo)
- if orig_path:
- known_paths.add(orig_path)
- commits = []
- current: _CommitRec | None = None
-
- for line in raw.splitlines():
- if line.startswith("\x00"):
- parts = line.lstrip("\x00").split("\x1f")
- if len(parts) >= 6:
- sha, an, ae, ct, parents, subj = (
- parts[0],
- parts[1],
- parts[2],
- parts[3],
- parts[4],
- parts[5],
- )
- try:
- ts = int(ct)
- except ValueError:
- ts = 0
- current = _CommitRec(
- sha=sha,
- author_name=an or "unknown",
- author_email=ae,
- ts=ts,
- is_merge=len(parents.split()) > 1,
- subject=subj,
- )
- commits.append(current)
- elif current is not None and line.strip():
- # numstat line: added\tdeleted\tpath — only count the target file.
- # With --follow, git may emit rename lines like
- # "10\t5\t{old_dir => new_dir}/file.py" or just the old path.
- numstat_parts = line.split("\t")
- if len(numstat_parts) >= 3:
- stat_path = numstat_parts[2]
- # Track renamed paths so we can match them in older commits.
- # For rename lines the raw stat_path (with {old => new}) won't
- # match known_paths directly — check the expanded new path instead.
- match_path = stat_path
- if "=>" in stat_path:
- _old, _new = _extract_rename_paths(stat_path, known_paths)
- match_path = _new or stat_path
- if match_path in known_paths or match_path == file_path:
- try:
- current.added += int(numstat_parts[0]) if numstat_parts[0] != "-" else 0
- current.deleted += (
- int(numstat_parts[1]) if numstat_parts[1] != "-" else 0
- )
- except ValueError:
- pass
-
- if not commits:
- return meta
-
- meta["commit_count_total"] = len(commits)
- meta["commit_count_capped"] = len(commits) >= self.commit_limit
-
- try:
- timestamps = [c.ts for c in commits if c.ts > 0]
- if timestamps:
- first_ts = min(timestamps)
- last_ts = max(timestamps)
- meta["first_commit_at"] = datetime.fromtimestamp(first_ts, tz=UTC)
- meta["last_commit_at"] = datetime.fromtimestamp(last_ts, tz=UTC)
- meta["age_days"] = (now - datetime.fromtimestamp(first_ts, tz=UTC)).days
-
- author_counts: Counter[str] = Counter()
- author_emails: dict[str, str] = {}
- recent_author_counts: Counter[str] = Counter()
-
- for c in commits:
- is_recent_90 = c.ts >= ninety_days_ago_ts
- is_recent_30 = c.ts >= thirty_days_ago_ts
- if is_recent_90:
- meta["commit_count_90d"] += 1
- recent_author_counts[c.author_name] += 1
- meta["lines_added_90d"] += c.added
- meta["lines_deleted_90d"] += c.deleted
- if c.is_merge:
- meta["merge_commit_count_90d"] += 1
- if is_recent_30:
- meta["commit_count_30d"] += 1
- author_counts[c.author_name] += 1
- if c.author_name not in author_emails and c.author_email:
- author_emails[c.author_name] = c.author_email
-
- c90 = meta["commit_count_90d"]
- total_churn = meta["lines_added_90d"] + meta["lines_deleted_90d"]
- meta["avg_commit_size"] = total_churn / c90 if c90 > 0 else 0.0
-
- # Temporal hotspot score: exponentially decayed per-commit churn.
- # Each commit contributes weight * clamped_lines where weight decays
- # with a half-life of HOTSPOT_HALFLIFE_DAYS days from now.
- _ln2 = math.log(2)
- temporal_score = 0.0
- for c in commits:
- age_days = max((now.timestamp() - c.ts) / 86400.0, 0.0)
- weight = math.exp(-_ln2 * age_days / HOTSPOT_HALFLIFE_DAYS)
- lines = min((c.added + c.deleted) / 100.0, 3.0)
- temporal_score += weight * lines
- meta["temporal_hotspot_score"] = temporal_score
-
- # Contributor count & bus factor
- meta["contributor_count"] = len(author_counts)
- total_commits = sum(author_counts.values())
- if total_commits > 0:
- threshold = total_commits * 0.8
- running = 0
- bus = 0
- for _name, cnt in author_counts.most_common():
- running += cnt
- bus += 1
- if running >= threshold:
- break
- meta["bus_factor"] = bus
-
- # Top authors — keep a generous slice (50) so the API can show
- # the full contributor surface for a file; the old top-5 cap
- # silently hid co-owners on multi-team modules.
- top_authors = []
- for name, count in author_counts.most_common(_MAX_TOP_AUTHORS):
- top_authors.append(
- {
- "name": name,
- "email": author_emails.get(name, ""),
- "commit_count": count,
- }
- )
- meta["top_authors_json"] = json.dumps(top_authors)
-
- if top_authors:
- primary = top_authors[0]
- meta["primary_owner_name"] = primary["name"]
- meta["primary_owner_email"] = primary["email"]
- meta["primary_owner_commit_pct"] = (
- primary["commit_count"] / total_commits if total_commits > 0 else 0.0
- )
-
- # Recent owner (90d)
- if recent_author_counts:
- recent_top = recent_author_counts.most_common(1)[0]
- meta["recent_owner_name"] = recent_top[0]
- recent_total = sum(recent_author_counts.values())
- meta["recent_owner_commit_pct"] = (
- recent_top[1] / recent_total if recent_total > 0 else 0.0
- )
-
- # Blame ownership (overrides author-based if available).
- # Skipped for large files — git blame is O(lines) and can block the
- # executor thread for many seconds on files > 100 KB.
- try:
- file_size = (self.repo_path / file_path).stat().st_size
- if file_size <= _MAX_BLAME_SIZE_BYTES:
- blame_name, blame_email, blame_pct = self._get_blame_ownership(file_path, repo)
- if blame_name:
- meta["primary_owner_name"] = blame_name
- meta["primary_owner_email"] = blame_email
- meta["primary_owner_commit_pct"] = blame_pct
- except Exception:
- pass # blame is best-effort
-
- # Significant commits + classification + PR extraction
- sig_commits: list[dict[str, Any]] = []
- sig_full = False
- category_counts: Counter[str] = Counter()
- for c in commits:
- msg = c.subject[:200]
- if not sig_full and self._is_significant_commit(msg, c.author_name):
- entry: dict[str, Any] = {
- "sha": c.sha[:8],
- "date": datetime.fromtimestamp(c.ts, tz=UTC).isoformat() if c.ts else "",
- "message": msg,
- "author": c.author_name,
- }
- pr_match = _PR_NUMBER_RE.search(msg)
- if pr_match:
- pr_num = pr_match.group(1) or pr_match.group(2) or pr_match.group(3)
- entry["pr_number"] = int(pr_num)
- sig_commits.append(entry)
- if len(sig_commits) >= _MAX_SIGNIFICANT_COMMITS:
- sig_full = True
- # Classify ALL commits for accurate category ratios
- for cat, pattern in _COMMIT_CATEGORIES.items():
- if pattern.search(msg):
- category_counts[cat] += 1
- break
-
- meta["significant_commits_json"] = json.dumps(sig_commits)
- meta["commit_categories_json"] = json.dumps(dict(category_counts))
-
- # Original path detection (rename tracking) — reuse the result
- # from the known_paths seeding above to avoid a duplicate subprocess.
- if self.follow_renames and orig_path:
- meta["original_path"] = orig_path
-
- # Stable classification
- if meta["commit_count_total"] > 10 and meta["commit_count_90d"] == 0:
- meta["is_stable"] = True
-
- except Exception:
- logger.debug("git_indexer_partial_failure", file_path=file_path, exc_info=True)
-
- return meta
-
- def _detect_original_path(self, file_path: str, repo: Any) -> str | None:
- """If --follow reveals the file was renamed, return its earliest prior path."""
- try:
- raw = repo.git.log(
- "--follow",
- f"-{self.commit_limit}",
- "--format=",
- "--name-only",
- "--",
- file_path,
- )
- except Exception:
- return None
-
- # Paths appear newest-first; the last distinct path is the original.
- prev_path: str | None = None
- for line in raw.splitlines():
- p = line.strip()
- if p and p != file_path:
- prev_path = p # keep overwriting — last one is oldest
- return prev_path
-
- def _get_blame_ownership(
- self, file_path: str, repo: Any
- ) -> tuple[str | None, str | None, float | None]:
- """Compute primary owner from git blame (who wrote the most lines)."""
- try:
- blame = repo.blame("HEAD", file_path)
- except Exception:
- return None, None, None
-
- line_counts: Counter[str] = Counter()
- emails: dict[str, str] = {}
- total_lines = 0
-
- for commit, lines in blame:
- name = commit.author.name or "unknown"
- count = len(lines)
- line_counts[name] += count
- total_lines += count
- if name not in emails and commit.author.email:
- emails[name] = commit.author.email
-
- if not line_counts or total_lines == 0:
- return None, None, None
-
- top_name = line_counts.most_common(1)[0][0]
- pct = line_counts[top_name] / total_lines
- return top_name, emails.get(top_name), pct
-
- def _is_significant_commit(self, message: str, author: str) -> bool:
- """Return True if the commit is considered significant.
-
- Filtering rules:
- 1. Always skip messages shorter than _MIN_MESSAGE_LEN characters.
- 2. Always skip merge commits and bot authors (no useful signal).
- 3. Conventional-commit prefixes (chore:, ci:, style:, build:,
- release:, Bump) are normally skipped — UNLESS the message also
- contains a decision-signal keyword (e.g. "build: migrate from
- webpack to vite"). This rescues architecturally meaningful
- commits that happen to use a low-signal prefix.
- """
- msg = message.strip()
- if len(msg) < _MIN_MESSAGE_LEN:
- return False
- # Always skip merge commits
- for prefix in _HARD_SKIP_PREFIXES:
- if msg.startswith(prefix):
- return False
- # Always skip bot authors
- author_lower = author.lower()
- for skip in _SKIP_AUTHORS:
- if skip in author_lower:
- return False
- # Soft-skip conventional prefixes unless decision signal present
- for prefix in _SOFT_SKIP_PREFIXES:
- if msg.startswith(prefix):
- msg_lower = msg.lower()
- return any(word in msg_lower for word in _DECISION_SIGNAL_WORDS)
- return True
-
- def _compute_co_changes(
- self,
- repo: Any,
- all_files: set[str],
- commit_limit: int = _DEFAULT_CO_CHANGE_COMMIT_LIMIT,
- min_count: int = _DEFAULT_CO_CHANGE_MIN_COUNT,
- on_commit_done: Callable[[], None] | None = None,
- on_co_change_start: Callable[[int], None] | None = None,
- ) -> dict[str, list[dict]]:
- """Walk recent commits and record co-occurrence pairs for tracked files.
-
- Uses a single ``git log --name-only`` call instead of spawning one
- ``git diff`` subprocess per commit — O(1) processes vs O(commit_limit).
-
- Applies exponential temporal decay so recent co-changes weigh more
- than ancient ones. The ``%ct`` format captures commit timestamps.
-
- on_co_change_start(total) is called once with the actual number of
- commits found. on_commit_done() is called after each commit block is
- processed. Both are invoked from a thread-pool thread; callers must
- ensure thread safety (Rich Progress is thread-safe).
- """
- pair_scores: defaultdict[tuple[str, str], float] = defaultdict(float)
- pair_last_date: dict[tuple[str, str], int] = {} # pair → latest Unix ts
- now_ts = time.time()
-
- try:
- # %x00 = commit separator, %ct = committer timestamp (Unix epoch).
- raw = repo.git.log(
- f"-{commit_limit}",
- "--name-only",
- "--no-merges",
- "--format=%x00%ct",
- )
- except Exception:
- return {}
-
- # Count actual commits so the caller can set an accurate progress total.
- actual_commits = raw.count("\x00")
- if on_co_change_start is not None:
- on_co_change_start(actual_commits)
-
- current: set[str] = set()
- current_ts: int = 0
-
- def _flush_commit() -> None:
- if len(current) < 2:
- return
- if len(current) > _MAX_FILES_PER_COMMIT_FOR_COCHANGE:
- # Mass-edit commit — skip pair generation entirely (see
- # constant docstring). Logged at debug for traceability.
- logger.debug(
- "co_change_skip_oversized_commit",
- files_in_commit=len(current),
- threshold=_MAX_FILES_PER_COMMIT_FOR_COCHANGE,
- )
- return
- age_days = max((now_ts - current_ts) / 86400.0, 0.0)
- weight = math.exp(-age_days / _CO_CHANGE_DECAY_TAU)
- sorted_files = sorted(current)
- for i in range(len(sorted_files)):
- for j in range(i + 1, len(sorted_files)):
- pair = (sorted_files[i], sorted_files[j])
- pair_scores[pair] += weight
- if pair not in pair_last_date or current_ts > pair_last_date[pair]:
- pair_last_date[pair] = current_ts
-
- for line in raw.splitlines():
- if line == "\x00" or line.startswith("\x00"):
- # Commit boundary — flush previous, parse timestamp.
- _flush_commit()
- current = set()
- # Extract Unix timestamp after the \x00 marker
- ts_part = line.lstrip("\x00").strip()
- try:
- current_ts = int(ts_part)
- except (ValueError, TypeError):
- current_ts = 0
- if on_commit_done is not None:
- on_commit_done()
- else:
- path = line.strip()
- if path and path in all_files:
- current.add(path)
-
- _flush_commit() # final commit
-
- # Build result: for each file, list partners above threshold.
- # min_count is compared against raw weight (decay-adjusted).
- # A score of 3.0 roughly equals 3 recent co-changes or many older ones.
- result: dict[str, list[dict]] = defaultdict(list)
- for (a, b), score in pair_scores.items():
- if score >= min_count:
- last_ts = pair_last_date.get((a, b), 0)
- last_date = (
- datetime.fromtimestamp(last_ts, tz=UTC).strftime("%Y-%m-%d")
- if last_ts > 0
- else None
- )
- entry_a = {
- "file_path": b,
- "co_change_count": round(score, 2),
- "last_co_change": last_date,
- }
- entry_b = {
- "file_path": a,
- "co_change_count": round(score, 2),
- "last_co_change": last_date,
- }
- result[a].append(entry_a)
- result[b].append(entry_b)
-
- # Sort partners by score descending
- for fp in result:
- result[fp].sort(key=lambda x: x["co_change_count"], reverse=True)
-
- # Lightweight observability: every audit so far has had to
- # answer "why are co-change tables empty?" by guessing. Logging
- # the filter funnel lets future debugging look at one log line.
- logger.debug(
- "co_change_computed",
- commits=actual_commits,
- tracked_files=len(all_files),
- pairs_considered=len(pair_scores),
- pairs_above_threshold=sum(1 for s in pair_scores.values() if s >= min_count),
- files_with_partners=len(result),
- min_count=min_count,
- commit_limit=commit_limit,
- )
-
- return dict(result)
-
- @staticmethod
- def _compute_percentiles(metadata_list: list[dict]) -> None:
- """Compute churn_percentile and is_hotspot. Mutates in place.
-
- Primary sort key is temporal_hotspot_score (exponentially decayed churn);
- commit_count_90d is used as a tiebreak, matching the SQL PERCENT_RANK path.
- """
- if not metadata_list:
- return
-
- # Sort by temporal_hotspot_score (primary) then commit_count_90d (tiebreak)
- sorted_by_churn = sorted(
- range(len(metadata_list)),
- key=lambda i: (
- metadata_list[i].get("temporal_hotspot_score") or 0.0,
- metadata_list[i].get("commit_count_90d", 0),
- ),
- )
-
- total = len(metadata_list)
- for rank, idx in enumerate(sorted_by_churn):
- metadata_list[idx]["churn_percentile"] = rank / total if total > 0 else 0.0
-
- # Hotspot: top 25% churn (i.e., churn_percentile >= 0.75)
- for meta in metadata_list:
- commit_90d = meta.get("commit_count_90d", 0)
- churn_pct = meta.get("churn_percentile", 0.0)
- if churn_pct >= 0.75 and commit_90d > 0:
- meta["is_hotspot"] = True
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/README.md b/packages/core/src/repowise/core/ingestion/git_indexer/README.md
new file mode 100644
index 000000000..a2db8eb97
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/README.md
@@ -0,0 +1,59 @@
+# `git_indexer`
+
+Mines git history into per-file metadata (ownership, churn, hotspots,
+co-change partners) for the ingestion pipeline.
+
+## Purpose
+
+Turn a repository's commit history into the signals the wiki and health
+layers consume: commit counts and recency windows, contributor/bus-factor,
+significant commits, a temporal hotspot score, and co-change relationships.
+
+## Public API
+
+Imported via `repowise.core.ingestion.git_indexer`:
+
+- `GitIndexer` — the orchestrator (`index_repo`, `index_changed_files`).
+- `GitIndexTier` — `FULL` (default) or `ESSENTIAL` indexing depth.
+- `GitIndexSummary` — counts + duration returned by `index_repo`.
+- `backfill_full_tier(indexer, repo_id, *, job_store=None)` — promote an
+ ESSENTIAL index to FULL as a resumable phase.
+- Module-level building blocks (unit-tested directly): `index_file`,
+ `compute_co_changes`, `compute_percentiles`, `get_blame_ownership`,
+ `is_significant_commit`, `detect_original_path`.
+
+## Tiers
+
+| Tier | Per-file history | `git blame` ownership | Co-change walk |
+|------|------------------|-----------------------|----------------|
+| `FULL` (default) | yes | yes | yes |
+| `ESSENTIAL` | yes | no (commit-author fallback) | no |
+
+`ESSENTIAL` is for the fast orchestrator path (`--mode fast`) on very large
+repos: it skips the two O(repo) signals so a first index lands quickly, then
+`backfill_full_tier` fills in blame + co-change later.
+
+## Internal layout
+
+| Module | Contents |
+|--------|----------|
+| `tiers.py` | `GitIndexTier` enum + `includes_blame` / `includes_co_change` |
+| `_constants.py` | commit-depth defaults, decay half-lives, skip heuristics, GitPython noise patch |
+| `records.py` | `_CommitRec`, `GitIndexSummary`, rename/skip path helpers |
+| `file_history.py` | `index_file` — per-file parse + base metrics (blame gated by tier) |
+| `enrich.py` | blame ownership, commit significance, rename detection, percentiles |
+| `co_change.py` | `compute_co_changes` — repo-wide decay-weighted pair walk |
+| `indexer.py` | `GitIndexer` class wiring the above; back-compat instance shims |
+| `backfill.py` | `backfill_full_tier` resumable ESSENTIAL→FULL promotion |
+
+## Extension points
+
+A downstream indexer can call the module functions directly with its own
+executor, or subclass nothing and just pass a different `GitIndexTier`.
+`backfill_full_tier` accepts any `JobStore` for checkpoint/resume.
+
+## Tests
+
+`tests/unit/test_git_indexer.py` (per-function behaviour),
+`tests/unit/ingestion/test_git_indexer_tiers.py` (tier gating + backfill),
+`tests/integration/test_git_intelligence_integration.py` (end-to-end).
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py b/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py
new file mode 100644
index 000000000..b6f53e27a
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py
@@ -0,0 +1,58 @@
+"""Git history indexer for the repowise ingestion pipeline.
+
+Mines git history into the git_metadata table. Uses gitpython for git
+operations and parallelises per-file git log calls.
+
+Non-blocking: if git is unavailable or the repo has no history, a warning is
+logged and an empty summary returned. All downstream features degrade
+gracefully.
+
+The indexer is tiered (:class:`GitIndexTier`): ``FULL`` (default) preserves the
+historical behaviour exactly, while ``ESSENTIAL`` skips the two expensive
+signals — per-file ``git blame`` and the repo-wide co-change walk — for fast
+large-repo indexing. :func:`~.backfill.backfill_full_tier` promotes an
+ESSENTIAL index to FULL afterwards.
+
+This package was split out of a single 1k-line module; the public import path
+(``repowise.core.ingestion.git_indexer``) is unchanged.
+"""
+
+from __future__ import annotations
+
+from ._constants import HOTSPOT_HALFLIFE_DAYS
+from .backfill import BACKFILL_PHASE, backfill_full_tier
+from .co_change import compute_co_changes
+from .enrich import (
+ compute_percentiles,
+ detect_original_path,
+ get_blame_ownership,
+ is_significant_commit,
+)
+from .file_history import index_file
+from .indexer import GitIndexer
+from .records import (
+ GitIndexSummary,
+ _CommitRec,
+ _extract_rename_paths,
+ _should_skip_index,
+)
+from .tiers import GitIndexTier
+
+__all__ = [
+ "BACKFILL_PHASE",
+ "HOTSPOT_HALFLIFE_DAYS",
+ "GitIndexSummary",
+ "GitIndexTier",
+ "GitIndexer",
+ # Re-exported for ``git_commit_index`` which builds the shared commit index.
+ "_CommitRec",
+ "_extract_rename_paths",
+ "_should_skip_index",
+ "backfill_full_tier",
+ "compute_co_changes",
+ "compute_percentiles",
+ "detect_original_path",
+ "get_blame_ownership",
+ "index_file",
+ "is_significant_commit",
+]
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py b/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py
new file mode 100644
index 000000000..81334eeae
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py
@@ -0,0 +1,144 @@
+"""Module constants, regexes, and the GitPython noise patch for git indexing.
+
+Kept in one place so the per-tier modules share a single source of truth for
+the commit-depth defaults, decay half-lives, and skip heuristics.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import re
+from typing import Any
+
+from ..languages.registry import REGISTRY as _LANG_REGISTRY
+
+# Silence GitPython's _CatFileContentStream.__del__ ValueError spam.
+# When git cat-file streams are GC'd after the subprocess pipe is closed,
+# their __del__ tries to drain remaining bytes and hits a closed file.
+# This is harmless but floods stderr with tracebacks.
+try:
+ from git.cmd import _CatFileContentStream
+
+ _orig_del = _CatFileContentStream.__del__
+
+ def _quiet_del(self: Any) -> None:
+ with contextlib.suppress(ValueError, OSError):
+ _orig_del(self)
+
+ _CatFileContentStream.__del__ = _quiet_del # type: ignore[assignment]
+except Exception:
+ pass # git not installed — nothing to patch
+
+# Commit message prefixes that are ALWAYS skipped (no signal value).
+_HARD_SKIP_PREFIXES = ("Merge ",)
+
+# Conventional-commit prefixes normally skipped — but kept if the message
+# contains a decision-signal keyword (e.g. "build: migrate from webpack to vite").
+_SOFT_SKIP_PREFIXES = ("Bump ", "chore:", "ci:", "style:", "build:", "release:")
+
+# Lightweight subset of decision-signal keywords (mirrors decision_extractor.py).
+# Used to rescue soft-skipped commits that carry architectural intent.
+_DECISION_SIGNAL_WORDS: frozenset[str] = frozenset(
+ {
+ "migrate",
+ "migration",
+ "switch to",
+ "replace",
+ "refactor",
+ "adopt",
+ "introduce",
+ "deprecate",
+ "remove",
+ "upgrade",
+ "rewrite",
+ "extract",
+ "convert",
+ "transition",
+ }
+)
+
+_SKIP_AUTHORS = ("dependabot", "renovate", "github-actions")
+_MIN_MESSAGE_LEN = 12
+
+# Default per-file commit history depth.
+_DEFAULT_COMMIT_LIMIT: int = 500
+
+# Per-file persisted contributor / commit fan-out. Previously hard-coded
+# to 5 / 10 inline, which silently hid co-owners and meaningful history on
+# multi-team modules. 50 is generous enough that any realistic UI surface
+# can render the full list while keeping the JSON blob bounded.
+_MAX_TOP_AUTHORS: int = 50
+_MAX_SIGNIFICANT_COMMITS: int = 50
+
+# Co-change pair extraction widens the window because individual files
+# may only co-change a handful of times in 500 commits — well below the
+# ``min_count`` threshold. On low-churn repos the 500-commit window
+# produced 0 co-change pairs every run; 2000 commits captures enough
+# history for the decay-weighted score to clear the bar without
+# meaningfully blowing up wall-clock time (single `git log` call).
+_DEFAULT_CO_CHANGE_COMMIT_LIMIT: int = 2000
+
+# Minimum decay-weighted co-occurrence weight for a pair to be recorded.
+# Was 3 historically; on repos with sparse change history (libraries,
+# stable services) that produced empty co-change tables. Two recent
+# co-changes is enough signal to surface in the UI, and the dashboard
+# already sorts partners by weight so the ranking is unaffected.
+_DEFAULT_CO_CHANGE_MIN_COUNT: int = 2
+
+# Commits that touch a very large number of files (mass renames,
+# copyright header sweeps, code-mod runs) produce O(N^2) pairs and
+# contribute no useful co-change signal. Skip pair generation for any
+# commit above this threshold. The decay-weighted score already
+# de-prioritises mass-edit commits, but the pairs are materialised
+# first — for a worst-case 500 files/commit x 2000 commits run that's
+# 250M pairs ~= 16 GB RAM. The cap is a memory safeguard, not a
+# correctness change: typical commits sit well under 20 files.
+_MAX_FILES_PER_COMMIT_FOR_COCHANGE: int = 200
+
+# Commit message classification regexes (Phase 2.2).
+_COMMIT_CATEGORIES: dict[str, re.Pattern[str]] = {
+ "feature": re.compile(
+ r"\b(add|implement|introduce|create|new|feat)\b",
+ re.IGNORECASE,
+ ),
+ "refactor": re.compile(
+ r"\b(refactor|restructure|cleanup|clean.up|rename|reorganize|extract|simplify|move)\b",
+ re.IGNORECASE,
+ ),
+ "fix": re.compile(
+ r"\b(fix|bug|patch|hotfix|revert|regression|broken|crash|error)\b",
+ re.IGNORECASE,
+ ),
+ "dependency": re.compile(
+ r"\b(upgrade|bump|update.dep|migrate.to|switch.to|dependency|dependencies)\b",
+ re.IGNORECASE,
+ ),
+}
+
+# Co-change temporal decay: half-life ~125 days (lambda for exp(-t/tau)).
+_CO_CHANGE_DECAY_TAU: float = 180.0
+
+# Hotspot temporal decay: half-life for exponentially weighted churn score.
+HOTSPOT_HALFLIFE_DAYS: float = 180.0
+
+# Regex to extract PR/MR numbers from commit messages.
+# Matches: "#123", "Merge pull request #456", "(#789)", "!42" (GitLab MR)
+_PR_NUMBER_RE = re.compile(r"(?:pull request |)\#(\d+)|\(#(\d+)\)|!(\d+)")
+
+# Allowlist of extensions for which per-file git indexing (blame, commit
+# history, hotspot/stable classification) is worth running. Anything NOT in
+# this set is skipped — data, config, markup, dotfiles, and binaries add no
+# documentation value, and git blame on large JSON/YAML files is very slow.
+# Co-change detection still runs across ALL tracked files regardless.
+# Derived from the centralised LanguageRegistry — all code language extensions.
+_CODE_EXTENSIONS: frozenset[str] = _LANG_REGISTRY.all_code_extensions()
+
+# Files larger than this skip git blame. blame is O(lines) and blocks the
+# executor thread — for large files the commit-based ownership estimate is
+# used as a fallback instead.
+_MAX_BLAME_SIZE_BYTES: int = 100 * 1024 # 100 KB
+
+# Maximum seconds to wait for a single file's git indexing. If exceeded the
+# file is recorded with whatever data was collected before the timeout and the
+# semaphore slot is released so other files can proceed.
+_FILE_INDEX_TIMEOUT_SECS: float = 45.0
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/backfill.py b/packages/core/src/repowise/core/ingestion/git_indexer/backfill.py
new file mode 100644
index 000000000..fb23d4b9a
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/backfill.py
@@ -0,0 +1,83 @@
+"""Background backfill worker — promote an ESSENTIAL index to FULL.
+
+When a large repo is indexed with :class:`~.tiers.GitIndexTier.ESSENTIAL`, the
+expensive signals (per-file blame, repo-wide co-change) are skipped to get the
+user a usable index fast. ``backfill_full_tier`` runs those signals afterwards
+as a separate, resumable phase.
+
+It is checkpoint-aware: when a :class:`JobStore` is supplied it records a
+``git.backfill`` job, advances the cursor as it completes, and marks the job
+COMPLETED / FAILED so a crashed backfill can be detected and re-run. The
+worker is deliberately a thin scaffold — it re-runs FULL indexing for the repo
+— so the resume contract and wiring are exercised now; finer-grained
+per-file resume is a follow-up.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import structlog
+
+from .records import GitIndexSummary
+from .tiers import GitIndexTier
+
+if TYPE_CHECKING:
+ from repowise.core.persistence._interfaces.job_store import JobStore
+
+ from .indexer import GitIndexer
+
+logger = structlog.get_logger(__name__)
+
+__all__ = ["BACKFILL_PHASE", "backfill_full_tier"]
+
+# Phase name used for the JobStore record so resume detection can find it.
+BACKFILL_PHASE = "git.backfill"
+
+
+async def backfill_full_tier(
+ indexer: GitIndexer,
+ repo_id: str,
+ *,
+ job_store: JobStore | None = None,
+) -> tuple[GitIndexSummary, list[dict]]:
+ """Run the FULL-tier git signals for *repo_id* and return the result.
+
+ *indexer* may have been constructed at any tier; this forces FULL for the
+ duration of the backfill and restores the original tier afterwards. When
+ *job_store* is provided the run is bracketed by a resumable job record.
+ """
+ from repowise.core.persistence._interfaces.job_store import JobState
+
+ job_id: str | None = None
+ if job_store is not None:
+ job = await job_store.create_job(
+ repository_id=repo_id,
+ phase=BACKFILL_PHASE,
+ metadata={"tier": GitIndexTier.FULL.value},
+ )
+ job_id = job.id
+ await job_store.update_state(job_id, JobState.RUNNING)
+
+ original_tier = indexer.tier
+ indexer.tier = GitIndexTier.FULL
+ try:
+ summary, results = await indexer.index_repo(repo_id)
+ except Exception as exc:
+ if job_store is not None and job_id is not None:
+ await job_store.update_state(job_id, JobState.FAILED, error=str(exc))
+ logger.warning("git_backfill_failed", repo_id=repo_id, error=str(exc))
+ raise
+ finally:
+ indexer.tier = original_tier
+
+ if job_store is not None and job_id is not None:
+ await job_store.update_state(
+ job_id, JobState.COMPLETED, cursor=str(summary.files_indexed)
+ )
+ logger.info(
+ "git_backfill_complete",
+ repo_id=repo_id,
+ files=summary.files_indexed,
+ )
+ return summary, results
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/co_change.py b/packages/core/src/repowise/core/ingestion/git_indexer/co_change.py
new file mode 100644
index 000000000..311192ec9
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/co_change.py
@@ -0,0 +1,154 @@
+"""Repo-wide co-change accumulation (FULL-tier signal).
+
+A single ``git log --name-only`` walk records decay-weighted co-occurrence
+pairs across all tracked files. This is the second of the two expensive
+signals the ESSENTIAL tier defers; the FULL tier (and the backfill worker)
+run it.
+"""
+
+from __future__ import annotations
+
+import math
+import time
+from collections import defaultdict
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+
+import structlog
+
+from ._constants import (
+ _CO_CHANGE_DECAY_TAU,
+ _DEFAULT_CO_CHANGE_COMMIT_LIMIT,
+ _DEFAULT_CO_CHANGE_MIN_COUNT,
+ _MAX_FILES_PER_COMMIT_FOR_COCHANGE,
+)
+
+logger = structlog.get_logger(__name__)
+
+__all__ = ["compute_co_changes"]
+
+
+def compute_co_changes(
+ repo: Any,
+ all_files: set[str],
+ commit_limit: int = _DEFAULT_CO_CHANGE_COMMIT_LIMIT,
+ min_count: int = _DEFAULT_CO_CHANGE_MIN_COUNT,
+ on_commit_done: Callable[[], None] | None = None,
+ on_co_change_start: Callable[[int], None] | None = None,
+) -> dict[str, list[dict]]:
+ """Walk recent commits and record co-occurrence pairs for tracked files.
+
+ Uses a single ``git log --name-only`` call instead of spawning one
+ ``git diff`` subprocess per commit — O(1) processes vs O(commit_limit).
+
+ Applies exponential temporal decay so recent co-changes weigh more than
+ ancient ones. ``on_co_change_start(total)`` is called once with the actual
+ number of commits found; ``on_commit_done()`` after each commit block.
+ Both run from a thread-pool thread; callers must ensure thread safety.
+ """
+ pair_scores: defaultdict[tuple[str, str], float] = defaultdict(float)
+ pair_last_date: dict[tuple[str, str], int] = {} # pair → latest Unix ts
+ now_ts = time.time()
+
+ try:
+ # %x00 = commit separator, %ct = committer timestamp (Unix epoch).
+ raw = repo.git.log(
+ f"-{commit_limit}",
+ "--name-only",
+ "--no-merges",
+ "--format=%x00%ct",
+ )
+ except Exception:
+ return {}
+
+ actual_commits = raw.count("\x00")
+ if on_co_change_start is not None:
+ on_co_change_start(actual_commits)
+
+ current: set[str] = set()
+ current_ts: int = 0
+
+ def _flush_commit() -> None:
+ nonlocal current_ts
+ if len(current) < 2:
+ return
+ if len(current) > _MAX_FILES_PER_COMMIT_FOR_COCHANGE:
+ # Mass-edit commit — skip pair generation entirely (see constant
+ # docstring). Logged at debug for traceability.
+ logger.debug(
+ "co_change_skip_oversized_commit",
+ files_in_commit=len(current),
+ threshold=_MAX_FILES_PER_COMMIT_FOR_COCHANGE,
+ )
+ return
+ age_days = max((now_ts - current_ts) / 86400.0, 0.0)
+ weight = math.exp(-age_days / _CO_CHANGE_DECAY_TAU)
+ sorted_files = sorted(current)
+ for i in range(len(sorted_files)):
+ for j in range(i + 1, len(sorted_files)):
+ pair = (sorted_files[i], sorted_files[j])
+ pair_scores[pair] += weight
+ if pair not in pair_last_date or current_ts > pair_last_date[pair]:
+ pair_last_date[pair] = current_ts
+
+ for line in raw.splitlines():
+ if line == "\x00" or line.startswith("\x00"):
+ # Commit boundary — flush previous, parse timestamp.
+ _flush_commit()
+ current = set()
+ ts_part = line.lstrip("\x00").strip()
+ try:
+ current_ts = int(ts_part)
+ except (ValueError, TypeError):
+ current_ts = 0
+ if on_commit_done is not None:
+ on_commit_done()
+ else:
+ path = line.strip()
+ if path and path in all_files:
+ current.add(path)
+
+ _flush_commit() # final commit
+
+ # Build result: for each file, list partners above threshold.
+ result: dict[str, list[dict]] = defaultdict(list)
+ for (a, b), score in pair_scores.items():
+ if score >= min_count:
+ last_ts = pair_last_date.get((a, b), 0)
+ last_date = (
+ datetime.fromtimestamp(last_ts, tz=UTC).strftime("%Y-%m-%d")
+ if last_ts > 0
+ else None
+ )
+ result[a].append(
+ {
+ "file_path": b,
+ "co_change_count": round(score, 2),
+ "last_co_change": last_date,
+ }
+ )
+ result[b].append(
+ {
+ "file_path": a,
+ "co_change_count": round(score, 2),
+ "last_co_change": last_date,
+ }
+ )
+
+ # Sort partners by score descending
+ for fp in result:
+ result[fp].sort(key=lambda x: x["co_change_count"], reverse=True)
+
+ logger.debug(
+ "co_change_computed",
+ commits=actual_commits,
+ tracked_files=len(all_files),
+ pairs_considered=len(pair_scores),
+ pairs_above_threshold=sum(1 for s in pair_scores.values() if s >= min_count),
+ files_with_partners=len(result),
+ min_count=min_count,
+ commit_limit=commit_limit,
+ )
+
+ return dict(result)
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py b/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py
new file mode 100644
index 000000000..1dbf8a320
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py
@@ -0,0 +1,142 @@
+"""FULL-tier enrichment helpers: blame ownership, commit significance,
+rename detection, and churn percentiles.
+
+These are pure functions (no GitIndexer instance state) so each can be unit
+tested in isolation. The cheap baseline (commit counts, authors, temporal
+score) lives in :mod:`file_history`; the expensive blame call and the
+significance/percentile logic that only the FULL tier needs live here.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from ._constants import (
+ _DECISION_SIGNAL_WORDS,
+ _HARD_SKIP_PREFIXES,
+ _MIN_MESSAGE_LEN,
+ _SKIP_AUTHORS,
+ _SOFT_SKIP_PREFIXES,
+)
+
+__all__ = [
+ "compute_percentiles",
+ "detect_original_path",
+ "get_blame_ownership",
+ "is_significant_commit",
+]
+
+
+def detect_original_path(repo: Any, file_path: str, commit_limit: int) -> str | None:
+ """If --follow reveals the file was renamed, return its earliest prior path."""
+ try:
+ raw = repo.git.log(
+ "--follow",
+ f"-{commit_limit}",
+ "--format=",
+ "--name-only",
+ "--",
+ file_path,
+ )
+ except Exception:
+ return None
+
+ # Paths appear newest-first; the last distinct path is the original.
+ prev_path: str | None = None
+ for line in raw.splitlines():
+ p = line.strip()
+ if p and p != file_path:
+ prev_path = p # keep overwriting — last one is oldest
+ return prev_path
+
+
+def get_blame_ownership(
+ repo: Any, file_path: str
+) -> tuple[str | None, str | None, float | None]:
+ """Compute primary owner from git blame (who wrote the most lines)."""
+ try:
+ blame = repo.blame("HEAD", file_path)
+ except Exception:
+ return None, None, None
+
+ line_counts: Counter[str] = Counter()
+ emails: dict[str, str] = {}
+ total_lines = 0
+
+ for commit, lines in blame:
+ name = commit.author.name or "unknown"
+ count = len(lines)
+ line_counts[name] += count
+ total_lines += count
+ if name not in emails and commit.author.email:
+ emails[name] = commit.author.email
+
+ if not line_counts or total_lines == 0:
+ return None, None, None
+
+ top_name = line_counts.most_common(1)[0][0]
+ pct = line_counts[top_name] / total_lines
+ return top_name, emails.get(top_name), pct
+
+
+def is_significant_commit(message: str, author: str) -> bool:
+ """Return True if the commit is considered significant.
+
+ Filtering rules:
+ 1. Always skip messages shorter than _MIN_MESSAGE_LEN characters.
+ 2. Always skip merge commits and bot authors (no useful signal).
+ 3. Conventional-commit prefixes (chore:, ci:, style:, build:,
+ release:, Bump) are normally skipped — UNLESS the message also
+ contains a decision-signal keyword (e.g. "build: migrate from
+ webpack to vite"). This rescues architecturally meaningful
+ commits that happen to use a low-signal prefix.
+ """
+ msg = message.strip()
+ if len(msg) < _MIN_MESSAGE_LEN:
+ return False
+ # Always skip merge commits
+ for prefix in _HARD_SKIP_PREFIXES:
+ if msg.startswith(prefix):
+ return False
+ # Always skip bot authors
+ author_lower = author.lower()
+ for skip in _SKIP_AUTHORS:
+ if skip in author_lower:
+ return False
+ # Soft-skip conventional prefixes unless decision signal present
+ for prefix in _SOFT_SKIP_PREFIXES:
+ if msg.startswith(prefix):
+ msg_lower = msg.lower()
+ return any(word in msg_lower for word in _DECISION_SIGNAL_WORDS)
+ return True
+
+
+def compute_percentiles(metadata_list: list[dict]) -> None:
+ """Compute churn_percentile and is_hotspot. Mutates in place.
+
+ Primary sort key is temporal_hotspot_score (exponentially decayed churn);
+ commit_count_90d is used as a tiebreak, matching the SQL PERCENT_RANK path.
+ """
+ if not metadata_list:
+ return
+
+ # Sort by temporal_hotspot_score (primary) then commit_count_90d (tiebreak)
+ sorted_by_churn = sorted(
+ range(len(metadata_list)),
+ key=lambda i: (
+ metadata_list[i].get("temporal_hotspot_score") or 0.0,
+ metadata_list[i].get("commit_count_90d", 0),
+ ),
+ )
+
+ total = len(metadata_list)
+ for rank, idx in enumerate(sorted_by_churn):
+ metadata_list[idx]["churn_percentile"] = rank / total if total > 0 else 0.0
+
+ # Hotspot: top 25% churn (i.e., churn_percentile >= 0.75)
+ for meta in metadata_list:
+ commit_90d = meta.get("commit_count_90d", 0)
+ churn_pct = meta.get("churn_percentile", 0.0)
+ if churn_pct >= 0.75 and commit_90d > 0:
+ meta["is_hotspot"] = True
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py b/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py
new file mode 100644
index 000000000..b88ab48ab
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py
@@ -0,0 +1,340 @@
+"""Per-file git history indexing — the ESSENTIAL baseline plus optional
+FULL-tier blame enrichment.
+
+``index_file`` parses a single file's commit history (from a precomputed
+commit index when available, else a per-file ``git log``) and derives commit
+counts, authorship, line churn, the temporal hotspot score, and significant
+commits. The expensive ``git blame`` ownership pass is gated by
+*include_blame* so the ESSENTIAL tier can skip it.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections import Counter
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from ._constants import (
+ _COMMIT_CATEGORIES,
+ _MAX_BLAME_SIZE_BYTES,
+ _MAX_SIGNIFICANT_COMMITS,
+ _MAX_TOP_AUTHORS,
+ _PR_NUMBER_RE,
+ HOTSPOT_HALFLIFE_DAYS,
+)
+from .enrich import detect_original_path, get_blame_ownership, is_significant_commit
+from .records import _CommitRec, _extract_rename_paths
+
+logger = structlog.get_logger(__name__)
+
+__all__ = ["index_file", "new_meta"]
+
+
+def new_meta(file_path: str) -> dict[str, Any]:
+ """Return the default metadata dict for *file_path* (all fields zeroed)."""
+ return {
+ "file_path": file_path,
+ "commit_count_total": 0,
+ "commit_count_90d": 0,
+ "commit_count_30d": 0,
+ "commit_count_capped": False,
+ "first_commit_at": None,
+ "last_commit_at": None,
+ "primary_owner_name": None,
+ "primary_owner_email": None,
+ "primary_owner_commit_pct": None,
+ "top_authors_json": "[]",
+ "significant_commits_json": "[]",
+ "co_change_partners_json": "[]",
+ "commit_categories_json": "{}",
+ "is_hotspot": False,
+ "is_stable": False,
+ "churn_percentile": 0.0,
+ "age_days": 0,
+ # Phase 2 fields
+ "lines_added_90d": 0,
+ "lines_deleted_90d": 0,
+ "avg_commit_size": 0.0,
+ "recent_owner_name": None,
+ "recent_owner_commit_pct": None,
+ "bus_factor": 0,
+ "contributor_count": 0,
+ # Phase 3 fields
+ "original_path": None,
+ "merge_commit_count_90d": 0,
+ # Temporal hotspot score (exponentially decayed churn)
+ "temporal_hotspot_score": 0.0,
+ }
+
+
+def _parse_per_file_log(
+ repo: Any,
+ file_path: str,
+ *,
+ commit_limit: int,
+ follow_renames: bool,
+) -> tuple[list[_CommitRec], str | None]:
+ """Run a per-file ``git log --numstat`` and parse it into commit records.
+
+ Returns ``(commits, original_path)``. Only used when no precomputed commit
+ index is available (i.e. when rename-tracking is on).
+ """
+ log_args: list[str] = []
+ if follow_renames:
+ log_args.append("--follow")
+ log_args += [
+ f"-{commit_limit}",
+ "--numstat",
+ "--format=%x00%H%x1f%an%x1f%ae%x1f%ct%x1f%P%x1f%s",
+ "--",
+ file_path,
+ ]
+ try:
+ raw = repo.git.log(*log_args)
+ except Exception:
+ return [], None
+
+ if not raw.strip():
+ return [], None
+
+ known_paths: set[str] = {file_path}
+ orig_path: str | None = None
+ if follow_renames:
+ orig_path = detect_original_path(repo, file_path, commit_limit)
+ if orig_path:
+ known_paths.add(orig_path)
+
+ commits: list[_CommitRec] = []
+ current: _CommitRec | None = None
+ for line in raw.splitlines():
+ if line.startswith("\x00"):
+ parts = line.lstrip("\x00").split("\x1f")
+ if len(parts) >= 6:
+ sha, an, ae, ct, parents, subj = parts[:6]
+ try:
+ ts = int(ct)
+ except ValueError:
+ ts = 0
+ current = _CommitRec(
+ sha=sha,
+ author_name=an or "unknown",
+ author_email=ae,
+ ts=ts,
+ is_merge=len(parents.split()) > 1,
+ subject=subj,
+ )
+ commits.append(current)
+ elif current is not None and line.strip():
+ numstat_parts = line.split("\t")
+ if len(numstat_parts) >= 3:
+ stat_path = numstat_parts[2]
+ match_path = stat_path
+ if "=>" in stat_path:
+ _old, _new = _extract_rename_paths(stat_path, known_paths)
+ match_path = _new or stat_path
+ if match_path in known_paths or match_path == file_path:
+ try:
+ current.added += (
+ int(numstat_parts[0]) if numstat_parts[0] != "-" else 0
+ )
+ current.deleted += (
+ int(numstat_parts[1]) if numstat_parts[1] != "-" else 0
+ )
+ except ValueError:
+ pass
+ return commits, orig_path
+
+
+def index_file(
+ repo: Any,
+ file_path: str,
+ *,
+ repo_path: Path,
+ commit_limit: int,
+ follow_renames: bool,
+ include_blame: bool = True,
+ precomputed_commits: list[_CommitRec] | None = None,
+) -> dict:
+ """Index a single file's git history. Runs in executor.
+
+ When *precomputed_commits* is provided (the default when called from the
+ batched full-repo path), the per-file ``git log`` subprocess is skipped
+ entirely — eliminating the dominant process-spawn cost on large repos.
+
+ *include_blame* gates the FULL-tier ``git blame`` ownership pass; the
+ ESSENTIAL tier sets it False and falls back to commit-author ownership.
+ """
+ now = datetime.now(UTC)
+ ninety_days_ago_ts = (now - timedelta(days=90)).timestamp()
+ thirty_days_ago_ts = (now - timedelta(days=30)).timestamp()
+
+ meta = new_meta(file_path)
+
+ orig_path: str | None = None
+ if precomputed_commits is not None:
+ commits: list[_CommitRec] = precomputed_commits
+ else:
+ commits, orig_path = _parse_per_file_log(
+ repo,
+ file_path,
+ commit_limit=commit_limit,
+ follow_renames=follow_renames,
+ )
+
+ if not commits:
+ return meta
+
+ meta["commit_count_total"] = len(commits)
+ meta["commit_count_capped"] = len(commits) >= commit_limit
+
+ try:
+ timestamps = [c.ts for c in commits if c.ts > 0]
+ if timestamps:
+ first_ts = min(timestamps)
+ last_ts = max(timestamps)
+ meta["first_commit_at"] = datetime.fromtimestamp(first_ts, tz=UTC)
+ meta["last_commit_at"] = datetime.fromtimestamp(last_ts, tz=UTC)
+ meta["age_days"] = (now - datetime.fromtimestamp(first_ts, tz=UTC)).days
+
+ author_counts: Counter[str] = Counter()
+ author_emails: dict[str, str] = {}
+ recent_author_counts: Counter[str] = Counter()
+
+ for c in commits:
+ if c.ts >= ninety_days_ago_ts:
+ meta["commit_count_90d"] += 1
+ recent_author_counts[c.author_name] += 1
+ meta["lines_added_90d"] += c.added
+ meta["lines_deleted_90d"] += c.deleted
+ if c.is_merge:
+ meta["merge_commit_count_90d"] += 1
+ if c.ts >= thirty_days_ago_ts:
+ meta["commit_count_30d"] += 1
+ author_counts[c.author_name] += 1
+ if c.author_name not in author_emails and c.author_email:
+ author_emails[c.author_name] = c.author_email
+
+ c90 = meta["commit_count_90d"]
+ total_churn = meta["lines_added_90d"] + meta["lines_deleted_90d"]
+ meta["avg_commit_size"] = total_churn / c90 if c90 > 0 else 0.0
+
+ # Temporal hotspot score: exponentially decayed per-commit churn.
+ _ln2 = math.log(2)
+ temporal_score = 0.0
+ for c in commits:
+ age_days = max((now.timestamp() - c.ts) / 86400.0, 0.0)
+ weight = math.exp(-_ln2 * age_days / HOTSPOT_HALFLIFE_DAYS)
+ lines = min((c.added + c.deleted) / 100.0, 3.0)
+ temporal_score += weight * lines
+ meta["temporal_hotspot_score"] = temporal_score
+
+ # Contributor count & bus factor
+ meta["contributor_count"] = len(author_counts)
+ total_commits = sum(author_counts.values())
+ if total_commits > 0:
+ threshold = total_commits * 0.8
+ running = 0
+ bus = 0
+ for _name, cnt in author_counts.most_common():
+ running += cnt
+ bus += 1
+ if running >= threshold:
+ break
+ meta["bus_factor"] = bus
+
+ # Top authors — keep a generous slice (50) so the API can show the
+ # full contributor surface for a file.
+ top_authors = []
+ for name, count in author_counts.most_common(_MAX_TOP_AUTHORS):
+ top_authors.append(
+ {
+ "name": name,
+ "email": author_emails.get(name, ""),
+ "commit_count": count,
+ }
+ )
+ meta["top_authors_json"] = json.dumps(top_authors)
+
+ if top_authors:
+ primary = top_authors[0]
+ meta["primary_owner_name"] = primary["name"]
+ meta["primary_owner_email"] = primary["email"]
+ meta["primary_owner_commit_pct"] = (
+ primary["commit_count"] / total_commits if total_commits > 0 else 0.0
+ )
+
+ # Recent owner (90d)
+ if recent_author_counts:
+ recent_top = recent_author_counts.most_common(1)[0]
+ meta["recent_owner_name"] = recent_top[0]
+ recent_total = sum(recent_author_counts.values())
+ meta["recent_owner_commit_pct"] = (
+ recent_top[1] / recent_total if recent_total > 0 else 0.0
+ )
+
+ # Blame ownership (FULL tier only; overrides author-based if available).
+ # Skipped for large files — git blame is O(lines) and can block the
+ # executor thread for many seconds on files > 100 KB.
+ if include_blame:
+ try:
+ file_size = (repo_path / file_path).stat().st_size
+ if file_size <= _MAX_BLAME_SIZE_BYTES:
+ blame_name, blame_email, blame_pct = get_blame_ownership(
+ repo, file_path
+ )
+ if blame_name:
+ meta["primary_owner_name"] = blame_name
+ meta["primary_owner_email"] = blame_email
+ meta["primary_owner_commit_pct"] = blame_pct
+ except Exception:
+ pass # blame is best-effort
+
+ # Significant commits + classification + PR extraction
+ sig_commits: list[dict[str, Any]] = []
+ sig_full = False
+ category_counts: Counter[str] = Counter()
+ for c in commits:
+ msg = c.subject[:200]
+ if not sig_full and is_significant_commit(msg, c.author_name):
+ entry: dict[str, Any] = {
+ "sha": c.sha[:8],
+ "date": datetime.fromtimestamp(c.ts, tz=UTC).isoformat()
+ if c.ts
+ else "",
+ "message": msg,
+ "author": c.author_name,
+ }
+ pr_match = _PR_NUMBER_RE.search(msg)
+ if pr_match:
+ pr_num = pr_match.group(1) or pr_match.group(2) or pr_match.group(3)
+ entry["pr_number"] = int(pr_num)
+ sig_commits.append(entry)
+ if len(sig_commits) >= _MAX_SIGNIFICANT_COMMITS:
+ sig_full = True
+ # Classify ALL commits for accurate category ratios
+ for cat, pattern in _COMMIT_CATEGORIES.items():
+ if pattern.search(msg):
+ category_counts[cat] += 1
+ break
+
+ meta["significant_commits_json"] = json.dumps(sig_commits)
+ meta["commit_categories_json"] = json.dumps(dict(category_counts))
+
+ # Original path detection (rename tracking) — reuse the result from the
+ # known_paths seeding above to avoid a duplicate subprocess.
+ if follow_renames and orig_path:
+ meta["original_path"] = orig_path
+
+ # Stable classification
+ if meta["commit_count_total"] > 10 and meta["commit_count_90d"] == 0:
+ meta["is_stable"] = True
+
+ except Exception:
+ logger.debug("git_indexer_partial_failure", file_path=file_path, exc_info=True)
+
+ return meta
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
new file mode 100644
index 000000000..480965e29
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -0,0 +1,377 @@
+"""GitIndexer — orchestrates per-file history + co-change accumulation.
+
+The class wires together the tier modules: :mod:`file_history` for per-file
+metadata, :mod:`co_change` for the repo-wide pair walk, and :mod:`enrich` for
+percentiles. The :class:`~.tiers.GitIndexTier` passed at construction decides
+which expensive signals run (blame, co-change); ESSENTIAL skips both.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from ._constants import (
+ _DEFAULT_CO_CHANGE_COMMIT_LIMIT,
+ _DEFAULT_CO_CHANGE_MIN_COUNT,
+ _DEFAULT_COMMIT_LIMIT,
+ _FILE_INDEX_TIMEOUT_SECS,
+)
+from .co_change import compute_co_changes
+from .enrich import compute_percentiles
+from .file_history import index_file
+from .records import GitIndexSummary, _CommitRec, _should_skip_index
+from .tiers import GitIndexTier
+
+logger = structlog.get_logger(__name__)
+
+__all__ = ["GitIndexer"]
+
+
+class GitIndexer:
+ """Mines git history into the git_metadata table.
+
+ Uses gitpython (already a dependency) for git operations.
+ Parallelizes per-file git log calls with asyncio.Semaphore(20).
+
+ Non-blocking: if git is unavailable or repo has no history, log a warning
+ and return an empty summary. All downstream features degrade gracefully.
+
+ *tier* selects indexing depth: ``FULL`` (default) preserves historical
+ behaviour; ``ESSENTIAL`` skips per-file blame and the co-change walk for
+ fast large-repo indexing (backfill the rest later — see :mod:`backfill`).
+ """
+
+ def __init__(
+ self,
+ repo_path: str | Path,
+ *,
+ commit_limit: int | None = None,
+ follow_renames: bool = False,
+ tier: GitIndexTier = GitIndexTier.FULL,
+ ) -> None:
+ self.repo_path = Path(repo_path)
+ self.commit_limit = commit_limit or _DEFAULT_COMMIT_LIMIT
+ self.follow_renames = follow_renames
+ self.tier = tier
+
+ async def index_repo(
+ self,
+ repo_id: str,
+ on_start: Callable[[int], None] | None = None,
+ on_file_done: Callable[[], None] | None = None,
+ on_commit_done: Callable[[], None] | None = None,
+ on_co_change_start: Callable[[int], None] | None = None,
+ on_co_change_done: Callable[[], None] | None = None,
+ ) -> tuple[GitIndexSummary, list[dict]]:
+ """Full index of all tracked files. Returns summary + list of metadata
+ dicts ready for bulk upsert.
+
+ Optional progress callbacks (all thread-safe, called from the event
+ loop or executor threads):
+ on_start(total) — fired once with the total number of tracked files
+ on_file_done() — fired after each file is indexed
+ on_co_change_start(total) — fired once with actual commit count for
+ co-change analysis
+ on_commit_done() — fired after each commit is processed during
+ co-change analysis
+ on_co_change_done() — fired the moment co-change accumulation
+ completes (BEFORE per-file git indexing
+ finishes).
+ """
+ start = time.monotonic()
+ repo = self._get_repo()
+ if repo is None:
+ return GitIndexSummary(0, 0, 0, 0.0), []
+
+ tracked_files = self._get_tracked_files(repo)
+ if not tracked_files:
+ return GitIndexSummary(0, 0, 0, 0.0), []
+
+ # Only run expensive per-file indexing (git log + blame) on code files.
+ indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]
+
+ if on_start is not None:
+ on_start(len(indexable_files))
+
+ from concurrent.futures import ThreadPoolExecutor
+
+ executor = ThreadPoolExecutor(max_workers=20, thread_name_prefix="git-idx")
+ semaphore = asyncio.Semaphore(20)
+ loop = asyncio.get_event_loop()
+
+ # Build a repo-wide commit index in ONE git log subprocess when
+ # rename-tracking is off (the default). Each per-file worker then reads
+ # its commits from this shared dict instead of spawning its own
+ # ``git log -- <file>`` — turning O(files) process spawns into O(1).
+ commit_index: dict[str, list[_CommitRec]] = {}
+ if not self.follow_renames:
+ from ..git_commit_index import load_commit_index
+
+ commit_index = load_commit_index(
+ repo, self.commit_limit, set(indexable_files)
+ )
+
+ include_blame = self.tier.includes_blame
+
+ def _index_one_sync(file_path: str) -> dict:
+ """Use a per-thread Repo to avoid shared-handle issues on Windows."""
+ try:
+ import git as gitpython
+
+ thread_repo = gitpython.Repo(
+ self.repo_path, search_parent_directories=True
+ )
+ try:
+ precomputed = commit_index.get(file_path) if commit_index else None
+ return index_file(
+ thread_repo,
+ file_path,
+ repo_path=self.repo_path,
+ commit_limit=self.commit_limit,
+ follow_renames=self.follow_renames,
+ include_blame=include_blame,
+ precomputed_commits=precomputed,
+ )
+ finally:
+ thread_repo.close()
+ except Exception:
+ return {"file_path": file_path}
+
+ async def index_one(file_path: str) -> dict:
+ async with semaphore:
+ try:
+ result = await asyncio.wait_for(
+ loop.run_in_executor(executor, _index_one_sync, file_path),
+ timeout=_FILE_INDEX_TIMEOUT_SECS,
+ )
+ except TimeoutError:
+ logger.debug(
+ "Git indexing timed out for file — using partial data",
+ path=file_path,
+ timeout=_FILE_INDEX_TIMEOUT_SECS,
+ )
+ result = {"file_path": file_path}
+ except Exception as exc:
+ logger.debug(
+ "Git indexing failed for file", path=file_path, error=str(exc)
+ )
+ result = {"file_path": file_path}
+ if on_file_done is not None:
+ on_file_done()
+ return result
+
+ file_tasks = [index_one(fp) for fp in indexable_files]
+
+ async def _co_change_task() -> dict[str, list[dict]]:
+ # ESSENTIAL tier defers co-change entirely (the expensive repo-wide
+ # walk) — return empty and let a FULL backfill fill it in.
+ if not self.tier.includes_co_change:
+ if on_co_change_done is not None:
+ with contextlib.suppress(Exception):
+ on_co_change_done()
+ return {}
+ result = await loop.run_in_executor(
+ executor,
+ compute_co_changes,
+ repo,
+ set(tracked_files),
+ max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),
+ _DEFAULT_CO_CHANGE_MIN_COUNT,
+ on_commit_done,
+ on_co_change_start,
+ )
+ if on_co_change_done is not None:
+ with contextlib.suppress(Exception):
+ on_co_change_done()
+ return result
+
+ metadata_list, co_changes = await asyncio.gather(
+ asyncio.gather(*file_tasks, return_exceptions=True),
+ _co_change_task(),
+ )
+
+ # Abandon any timed-out threads immediately instead of letting
+ # asyncio.run() block for minutes during default-executor cleanup.
+ executor.shutdown(wait=False, cancel_futures=True)
+
+ results: list[dict] = []
+ for r in metadata_list:
+ if isinstance(r, Exception):
+ logger.warning("Failed to index file", error=str(r))
+ else:
+ results.append(r)
+
+ # Merge co-change partners into per-file metadata
+ for meta in results:
+ fp = meta["file_path"]
+ if fp in co_changes:
+ meta["co_change_partners_json"] = json.dumps(co_changes[fp])
+
+ compute_percentiles(results)
+
+ duration = time.monotonic() - start
+ hotspots = sum(1 for m in results if m.get("is_hotspot", False))
+ stable = sum(1 for m in results if m.get("is_stable", False))
+
+ summary = GitIndexSummary(
+ files_indexed=len(results),
+ hotspots=hotspots,
+ stable_files=stable,
+ duration_seconds=duration,
+ )
+ repo.close()
+
+ logger.info(
+ "Git indexing complete",
+ tier=self.tier.value,
+ files=summary.files_indexed,
+ hotspots=summary.hotspots,
+ stable=summary.stable_files,
+ duration=f"{summary.duration_seconds:.1f}s",
+ )
+ return summary, results
+
+ async def index_changed_files(self, changed_file_paths: list[str]) -> list[dict]:
+ """Incremental update: re-index only changed files."""
+ repo = self._get_repo()
+ if repo is None:
+ return []
+
+ loop = asyncio.get_event_loop()
+ semaphore = asyncio.Semaphore(20)
+ include_blame = self.tier.includes_blame
+
+ def _index_one_sync(file_path: str) -> dict:
+ try:
+ import git as gitpython
+
+ thread_repo = gitpython.Repo(
+ self.repo_path, search_parent_directories=True
+ )
+ try:
+ return index_file(
+ thread_repo,
+ file_path,
+ repo_path=self.repo_path,
+ commit_limit=self.commit_limit,
+ follow_renames=self.follow_renames,
+ include_blame=include_blame,
+ )
+ finally:
+ thread_repo.close()
+ except Exception:
+ return {"file_path": file_path}
+
+ async def index_one(file_path: str) -> dict:
+ async with semaphore:
+ try:
+ return await asyncio.wait_for(
+ loop.run_in_executor(None, _index_one_sync, file_path),
+ timeout=_FILE_INDEX_TIMEOUT_SECS,
+ )
+ except (TimeoutError, Exception) as exc:
+ logger.debug(
+ "Git indexing failed for changed file",
+ path=file_path,
+ error=str(exc),
+ )
+ return {"file_path": file_path}
+
+ tasks = [index_one(fp) for fp in changed_file_paths]
+ results_raw = await asyncio.gather(*tasks, return_exceptions=True)
+
+ results: list[dict] = []
+ for r in results_raw:
+ if isinstance(r, Exception):
+ logger.warning("Failed to index changed file", error=str(r))
+ else:
+ results.append(r)
+
+ repo.close()
+ return results
+
+ # ------------------------------------------------------------------
+ # Internal methods
+ # ------------------------------------------------------------------
+
+ def _get_repo(self) -> Any | None:
+ try:
+ import git as gitpython
+
+ return gitpython.Repo(self.repo_path, search_parent_directories=True)
+ except Exception as exc:
+ logger.warning(
+ "Git unavailable or not a repository",
+ path=str(self.repo_path),
+ error=str(exc),
+ )
+ return None
+
+ def _get_tracked_files(self, repo: Any) -> list[str]:
+ try:
+ output = repo.git.ls_files()
+ return [f for f in output.splitlines() if f.strip()]
+ except Exception as exc:
+ logger.warning("Failed to list tracked files", error=str(exc))
+ return []
+
+ # ------------------------------------------------------------------
+ # Backward-compatible instance shims
+ #
+ # The per-tier logic now lives in module-level functions (file_history,
+ # co_change, enrich) so it can be unit tested directly. These thin methods
+ # preserve the historical instance API for existing callers and tests.
+ # ------------------------------------------------------------------
+
+ def _index_file(
+ self,
+ file_path: str,
+ repo: Any,
+ precomputed_commits: list[_CommitRec] | None = None,
+ ) -> dict:
+ return index_file(
+ repo,
+ file_path,
+ repo_path=self.repo_path,
+ commit_limit=self.commit_limit,
+ follow_renames=self.follow_renames,
+ include_blame=self.tier.includes_blame,
+ precomputed_commits=precomputed_commits,
+ )
+
+ def _get_blame_ownership(
+ self, file_path: str, repo: Any
+ ) -> tuple[str | None, str | None, float | None]:
+ from .enrich import get_blame_ownership
+
+ return get_blame_ownership(repo, file_path)
+
+ def _is_significant_commit(self, message: str, author: str) -> bool:
+ from .enrich import is_significant_commit
+
+ return is_significant_commit(message, author)
+
+ def _compute_co_changes(
+ self,
+ repo: Any,
+ all_files: set[str],
+ commit_limit: int = _DEFAULT_CO_CHANGE_COMMIT_LIMIT,
+ min_count: int = _DEFAULT_CO_CHANGE_MIN_COUNT,
+ on_commit_done: Callable[[], None] | None = None,
+ on_co_change_start: Callable[[int], None] | None = None,
+ ) -> dict[str, list[dict]]:
+ return compute_co_changes(
+ repo, all_files, commit_limit, min_count, on_commit_done, on_co_change_start
+ )
+
+ @staticmethod
+ def _compute_percentiles(metadata_list: list[dict]) -> None:
+ compute_percentiles(metadata_list)
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/records.py b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
new file mode 100644
index 000000000..dc3b97397
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
@@ -0,0 +1,76 @@
+"""Lightweight data records and path helpers shared across git-index tiers."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from ._constants import _CODE_EXTENSIONS
+
+__all__ = [
+ "GitIndexSummary",
+ "_CommitRec",
+ "_extract_rename_paths",
+ "_should_skip_index",
+]
+
+
+@dataclass
+class _CommitRec:
+ """Lightweight commit record parsed from ``git log --numstat``."""
+
+ sha: str
+ author_name: str
+ author_email: str
+ ts: int # unix epoch
+ is_merge: bool
+ subject: str
+ added: int = 0
+ deleted: int = 0
+
+
+@dataclass
+class GitIndexSummary:
+ files_indexed: int
+ hotspots: int
+ stable_files: int
+ duration_seconds: float = 0.0
+
+
+_RENAME_RE = re.compile(r"\{(.+?) => (.+?)\}")
+
+
+def _extract_rename_paths(
+ stat_path: str, known_paths: set[str]
+) -> tuple[str | None, str | None]:
+ """Extract old/new paths from a git numstat rename line and add to *known_paths*.
+
+ Git ``--numstat`` with ``--follow`` emits rename lines like::
+
+ 10\t5\t{old => new}/shared_suffix
+ 10\t5\told_dir/{old_name => new_name}.py
+
+ This helper parses both forms, adds both expanded paths to *known_paths*,
+ and returns ``(old_path, new_path)`` so the caller can attribute churn to
+ the correct file. Returns ``(None, None)`` if the pattern is not found.
+ """
+ m = _RENAME_RE.search(stat_path)
+ if m:
+ prefix = stat_path[: m.start()]
+ suffix = stat_path[m.end() :]
+ old_path = prefix + m.group(1) + suffix
+ new_path = prefix + m.group(2) + suffix
+ known_paths.add(old_path)
+ known_paths.add(new_path)
+ return old_path, new_path
+ return None, None
+
+
+def _should_skip_index(file_path: str) -> bool:
+ """Return True for files where per-file git indexing should be skipped.
+
+ Uses an allowlist: only files with known source-code extensions are indexed.
+ Everything else (data, config, markup, dotfiles, binaries) is skipped.
+ """
+ return Path(file_path).suffix.lower() not in _CODE_EXTENSIONS
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/tiers.py b/packages/core/src/repowise/core/ingestion/git_indexer/tiers.py
new file mode 100644
index 000000000..51ac3cec8
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/tiers.py
@@ -0,0 +1,41 @@
+"""Git-indexing depth tiers.
+
+``GitIndexTier`` is the public knob that lets large-repo callers trade
+completeness for speed. ``FULL`` (the default) preserves the historical
+behaviour exactly. ``ESSENTIAL`` runs only the cheap baseline — per-file
+commit history with no per-file ``git blame`` and no co-change accumulation —
+so the fast orchestrator path can index a 30k-file repo quickly and backfill
+the expensive signals later (see :mod:`backfill`).
+"""
+
+from __future__ import annotations
+
+import enum
+
+__all__ = ["GitIndexTier"]
+
+
+class GitIndexTier(enum.StrEnum):
+ """Public knob for git-indexing depth on large repositories."""
+
+ ESSENTIAL = "essential"
+ FULL = "full"
+
+ @property
+ def includes_blame(self) -> bool:
+ """Whether per-file ``git blame`` ownership runs in this tier.
+
+ Blame is O(lines) and the dominant per-file cost; ESSENTIAL skips it
+ and falls back to commit-author ownership.
+ """
+ return self is GitIndexTier.FULL
+
+ @property
+ def includes_co_change(self) -> bool:
+ """Whether co-change pair accumulation runs in this tier.
+
+ The repo-wide co-change walk is the other expensive signal; ESSENTIAL
+ skips it and leaves ``co_change_partners_json`` empty until a FULL
+ backfill fills it in.
+ """
+ return self is GitIndexTier.FULL
diff --git a/packages/core/src/repowise/core/ingestion/graph.py b/packages/core/src/repowise/core/ingestion/graph.py
deleted file mode 100644
index 26b246b3a..000000000
--- a/packages/core/src/repowise/core/ingestion/graph.py
+++ /dev/null
@@ -1,929 +0,0 @@
-"""Dependency graph builder for the repowise ingestion pipeline.
-
-GraphBuilder constructs a directed graph from ParsedFile objects with two
-tiers of nodes:
-
- File-level nodes:
- "file" — every source file
- "external" — third-party / unresolvable imports (prefix "external:")
-
- Symbol-level nodes:
- "symbol" — functions, classes, methods, interfaces, etc.
- keyed by Symbol.id (e.g. "src/app.py::main")
-
-Edge types:
- "imports" — file-to-file import relationship
- "defines" — file-to-symbol containment
- "has_method" — class-to-method ownership
- "calls" — symbol-to-symbol call relationship (with confidence)
-
-After calling build(), graph metrics are available:
- pagerank() — dict[path, float]
- strongly_connected_components() — list[frozenset[str]]
- betweenness_centrality() — dict[path, float]
-"""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import Any
-
-import networkx as nx
-import structlog
-
-from .models import ParsedFile
-from .resolvers import ResolverContext, resolve_import
-from .resolvers.go import read_go_module_path, read_go_modules
-from .type_ref_resolution import resolve_type_refs
-
-log = structlog.get_logger(__name__)
-
-_LARGE_REPO_THRESHOLD = 30_000 # nodes — above this, algorithms are expensive
-
-# Path segments that mark a file as low-value for stem-based import resolution.
-_LOW_VALUE_PATH_SEGMENTS = frozenset(
- {
- "tests",
- "test",
- "_tests",
- "__tests__",
- "testing",
- "test_apps",
- "testdata",
- "test_data",
- "fixtures",
- "examples",
- "example",
- "samples",
- "sample",
- "scripts",
- "benchmarks",
- "bench",
- "docs",
- "doc",
- }
-)
-
-
-def _stem_priority(path: str, stem: str) -> tuple[int, int, int, str]:
- """Sort key for choosing among files that share an import stem.
-
- Lower tuples sort first; callers take ``candidates[0]`` as the resolution.
- """
- path_obj = Path(path)
- parts = path_obj.parts
- if path_obj.name == "__init__.py":
- parent_match = 0
- else:
- parent_dir = parts[-2].lower() if len(parts) >= 2 else ""
- parent_match = 0 if parent_dir == stem else 1
- low_value = 1 if any(seg.lower() in _LOW_VALUE_PATH_SEGMENTS for seg in parts) else 0
- return (parent_match, low_value, len(parts), path)
-
-
-class GraphBuilder:
- """Build a dependency graph from a collection of ParsedFile objects.
-
- Usage::
-
- builder = GraphBuilder()
- for parsed in parsed_files:
- builder.add_file(parsed)
- graph = builder.build()
- pr = builder.pagerank()
- """
-
- def __init__(self, repo_path: Path | str | None = None) -> None:
- self._graph: nx.DiGraph = nx.DiGraph()
- self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile
- self._built = False
- self._repo_path: Path | None = Path(repo_path) if repo_path else None
- self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)
-
- # Community / flow / metric caches (invalidated on build)
- self._community_cache: dict[str, int] | None = None
- self._symbol_community_cache: dict[str, int] | None = None
- self._community_info_cache: dict[int, Any] | None = None
- self._community_algo: str = ""
- self._pagerank_cache: dict[str, float] | None = None
- self._betweenness_cache: dict[str, float] | None = None
- self._symbol_pagerank_cache: dict[str, float] | None = None
- self._symbol_betweenness_cache: dict[str, float] | None = None
- self._execution_flow_cache: Any | None = None
-
- def set_tsconfig_resolver(self, resolver: Any) -> None:
- """Attach a :class:`TsconfigResolver` for TS/JS path-alias resolution."""
- self._tsconfig_resolver = resolver
-
- # ------------------------------------------------------------------
- # Building
- # ------------------------------------------------------------------
-
- def add_file(self, parsed: ParsedFile) -> None:
- """Register one parsed file and its symbols in the graph."""
- path = parsed.file_info.path
- self._parsed_files[path] = parsed
- self._built = False # invalidate cached metrics
-
- # --- File node ---
- self._graph.add_node(
- path,
- node_type="file",
- language=parsed.file_info.language,
- symbol_count=len(parsed.symbols),
- has_error=bool(parsed.parse_errors),
- is_test=parsed.file_info.is_test,
- is_entry_point=parsed.file_info.is_entry_point,
- )
-
- # --- Symbol nodes ---
- for sym in parsed.symbols:
- self._graph.add_node(
- sym.id,
- node_type="symbol",
- kind=sym.kind,
- name=sym.name,
- qualified_name=sym.qualified_name,
- file_path=path,
- start_line=sym.start_line,
- end_line=sym.end_line,
- visibility=sym.visibility,
- is_async=sym.is_async,
- language=sym.language,
- parent_name=sym.parent_name,
- signature=sym.signature,
- decorators=sym.decorators,
- is_exported_symbol=sym.is_exported_symbol,
- )
-
- # DEFINES edge: file → symbol
- self._graph.add_edge(
- path,
- sym.id,
- edge_type="defines",
- )
-
- # HAS_METHOD edge: class/struct → method
- if sym.parent_name and sym.kind == "method":
- parent_id = f"{path}::{sym.parent_name}"
- if parent_id in self._graph:
- self._graph.add_edge(
- parent_id,
- sym.id,
- edge_type="has_method",
- )
-
- # --- Synthetic module-level symbol for top-level calls ---
- module_sym_id = f"{path}::__module__"
- self._graph.add_node(
- module_sym_id,
- node_type="symbol",
- kind="module",
- name="__module__",
- file_path=path,
- start_line=0,
- end_line=0,
- visibility="private",
- language=parsed.file_info.language,
- )
- self._graph.add_edge(path, module_sym_id, edge_type="defines")
-
- def build(self, progress: Any | None = None) -> nx.DiGraph:
- """Resolve imports and calls, add edges. Returns the finalized graph.
-
- *progress* is an optional ``ProgressCallback`` (duck-typed). When
- provided, sub-phase events ``graph.imports`` / ``graph.heritage`` /
- ``graph.calls`` are emitted so the CLI can surface per-file progress
- instead of a single opaque "0/1" bar over the whole build.
- """
- # Invalidate cached metrics
- self._community_cache = None
- self._symbol_community_cache = None
- self._community_info_cache = None
- self._community_algo = ""
- self._pagerank_cache = None
- self._betweenness_cache = None
- self._symbol_pagerank_cache = None
- self._symbol_betweenness_cache = None
- self._execution_flow_cache = None
-
- # Clear import/call edges but keep structural edges (defines, has_method)
- edges_to_remove = [
- (u, v)
- for u, v, d in self._graph.edges(data=True)
- if d.get("edge_type") not in ("defines", "has_method")
- ]
- self._graph.remove_edges_from(edges_to_remove)
-
- # Build lookup tables for import resolution
- path_set = set(self._parsed_files.keys())
- stem_map = self._build_stem_map(path_set)
-
- # Construct resolver context
- go_modules = read_go_modules(self._repo_path)
- ctx = ResolverContext(
- path_set=path_set,
- stem_map=stem_map,
- graph=self._graph,
- repo_path=self._repo_path,
- tsconfig_resolver=self._tsconfig_resolver,
- go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),
- go_modules=go_modules,
- has_sfc_files=any(
- p.endswith((".vue", ".svelte", ".astro")) for p in path_set
- ),
- parsed_files=self._parsed_files,
- )
-
- # --- Phase 1 prelude: language-specific warmups ---
- # Some languages need an expensive one-time index built before any
- # per-file import resolution can run. Doing it here, under a dedicated
- # phase event, keeps the cost out of ``graph.imports`` and surfaces
- # a meaningful progress label instead of a stuck per-file bar.
- from .graph_warmups import run_warmups
-
- run_warmups(self._parsed_files, ctx, progress=progress)
-
- # --- Phase 1: Resolve file-level imports ---
- import_targets: dict[str, set[str]] = {} # file → set of imported files
-
- # Per-language import-resolution timing — surfaces which language is
- # actually dominating the import loop on multi-language repos
- # (audit #30). Persisted to state.json for after-the-fact analysis.
- import time as _t
- lang_import_time: dict[str, float] = {}
-
- if progress:
- progress.on_phase_start("graph.imports", len(self._parsed_files))
- for path, parsed in self._parsed_files.items():
- _lang = parsed.file_info.language
- _t0 = _t.monotonic()
- file_imports: set[str] = set()
- for imp in parsed.imports:
- target = resolve_import(imp.module_path, path, parsed.file_info.language, ctx)
- if target:
- imp.resolved_file = target
- file_imports.add(target)
- # Aggregate imported_names on parallel edges
- if self._graph.has_edge(path, target):
- existing = self._graph[path][target].get("imported_names", [])
- merged = list(set(existing + imp.imported_names))
- self._graph[path][target]["imported_names"] = merged
- else:
- self._graph.add_edge(
- path,
- target,
- edge_type="imports",
- imported_names=list(imp.imported_names),
- )
- import_targets[path] = file_imports
- lang_import_time[_lang] = lang_import_time.get(_lang, 0.0) + (_t.monotonic() - _t0)
- if progress:
- progress.on_item_done("graph.imports")
- if progress:
- _phase_done = getattr(progress, "on_phase_done", None)
- if _phase_done is not None:
- _phase_done("graph.imports")
- if lang_import_time:
- log.info(
- "import_resolution_per_language",
- seconds={k: round(v, 2) for k, v in lang_import_time.items()},
- )
-
- # --- Phase 1b: Resolve non-import type references (e.g. C# ctor params) ---
- # Lives between import resolution and heritage so the type-use
- # edges feed into heritage's import_targets propagation if a
- # future language emits them, and so dead-code reachability
- # sees every edge before any analysis pass runs.
- if progress:
- progress.on_phase_start("graph.type_refs", len(self._parsed_files))
- type_use_counts = resolve_type_refs(self._parsed_files, ctx, self._graph)
- for path in self._parsed_files:
- for _, target, data in self._graph.out_edges(path, data=True):
- # Treat both real ``using`` imports and synthesised
- # type-use edges as import-like for downstream heritage
- # / call resolution. Without this, type-use edges would
- # only contribute to file-reachability — not to
- # ``import_targets`` which gates cross-file call /
- # heritage lookups.
- if data.get("edge_type") in ("imports", "type_use"):
- import_targets.setdefault(path, set()).add(target)
- if progress:
- _phase_done = getattr(progress, "on_phase_done", None)
- if _phase_done is not None:
- _phase_done("graph.type_refs")
- del type_use_counts # used only for logging inside resolve_type_refs
-
- # --- Phase 1c: Resolve C# member-access reads ---
- # Bind `var x = new T(...); ... x.Prop` style property reads
- # to T's defining file. Cuts the largest single bucket of C#
- # unused_export false positives (audit #23).
- self._resolve_member_reads(progress=progress)
-
- # --- Phase 2: Resolve heritage (extends/implements) ---
- self._resolve_heritage(import_targets, progress=progress)
-
- # --- Phase 3: Resolve symbol-level calls ---
- self._resolve_calls(import_targets, progress=progress)
-
- self._built = True
-
- # Count edge types for logging
- edge_counts: dict[str, int] = {}
- for _, _, d in self._graph.edges(data=True):
- et = d.get("edge_type", "imports")
- edge_counts[et] = edge_counts.get(et, 0) + 1
-
- file_nodes = sum(
- 1 for _, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"
- )
- symbol_nodes = sum(
- 1 for _, d in self._graph.nodes(data=True) if d.get("node_type") == "symbol"
- )
-
- log.info(
- "Graph built",
- file_nodes=file_nodes,
- symbol_nodes=symbol_nodes,
- edges=self._graph.number_of_edges(),
- edge_types=edge_counts,
- )
- return self._graph
-
- def _resolve_heritage(
- self,
- import_targets: dict[str, set[str]],
- progress: Any | None = None,
- ) -> None:
- """Resolve heritage relations and add EXTENDS/IMPLEMENTS edges."""
- from .heritage_resolver import HeritageResolver
-
- resolver = HeritageResolver(self._parsed_files, import_targets)
- total_resolved = 0
-
- files_with_heritage = [
- (p, pf) for p, pf in self._parsed_files.items() if pf.heritage
- ]
- if progress:
- progress.on_phase_start("graph.heritage", len(files_with_heritage))
- for path, parsed in files_with_heritage:
- resolved = resolver.resolve_file(path, parsed.heritage)
- for rh in resolved:
- if rh.child_id in self._graph and rh.parent_id in self._graph:
- if not self._graph.has_edge(rh.child_id, rh.parent_id):
- self._graph.add_edge(
- rh.child_id,
- rh.parent_id,
- edge_type=rh.edge_type,
- confidence=rh.confidence,
- )
- total_resolved += 1
- else:
- existing = self._graph[rh.child_id][rh.parent_id]
- if rh.confidence > existing.get("confidence", 0):
- existing["confidence"] = rh.confidence
- if progress:
- progress.on_item_done("graph.heritage")
-
- if progress:
- _phase_done = getattr(progress, "on_phase_done", None)
- if _phase_done is not None:
- _phase_done("graph.heritage")
- log.info("Heritage edges resolved", total=total_resolved)
-
- def _resolve_member_reads(self, progress: Any | None = None) -> None:
- """Phase 1c: emit ``reads`` edges for C# property / member access.
-
- Runs after type-use resolution so the dead-code analyser sees
- member access as evidence of reachability. The pass is C#-only
- today (the lever is largest there); the helper module is set
- up to receive other languages via additional strategies.
- """
- from .languages.csharp_member_reads import (
- build_csharp_type_to_file,
- collect_csharp_source_texts,
- resolve_csharp_member_reads,
- )
-
- has_csharp = any(
- pf.file_info.language == "csharp" for pf in self._parsed_files.values()
- )
- if not has_csharp:
- return
-
- phase = "graph.member_reads"
- if progress:
- progress.on_phase_start(phase, None)
- try:
- cs_texts = collect_csharp_source_texts(self._parsed_files)
- type_to_file = build_csharp_type_to_file(self._parsed_files)
- added = resolve_csharp_member_reads(self._graph, cs_texts, type_to_file)
- log.info("member_read_edges", language="csharp", added=added)
- except Exception as exc:
- log.warning("member_reads_failed", error=str(exc))
- finally:
- if progress:
- done = getattr(progress, "on_phase_done", None)
- if callable(done):
- done(phase)
-
- def _resolve_calls(
- self,
- import_targets: dict[str, set[str]],
- progress: Any | None = None,
- ) -> None:
- """Run three-tier call resolution and add CALLS edges to the graph."""
- from .call_resolver import CallResolver
-
- resolver = CallResolver(self._parsed_files, import_targets)
- total_resolved = 0
-
- files_with_calls = [
- (p, pf) for p, pf in self._parsed_files.items() if pf.calls
- ]
- if progress:
- progress.on_phase_start("graph.calls", len(files_with_calls))
- for path, parsed in files_with_calls:
- resolved = resolver.resolve_file(path, parsed.calls)
- for rc in resolved:
- if rc.caller_id in self._graph and rc.callee_id in self._graph:
- if not self._graph.has_edge(rc.caller_id, rc.callee_id):
- self._graph.add_edge(
- rc.caller_id,
- rc.callee_id,
- edge_type="calls",
- confidence=rc.confidence,
- )
- total_resolved += 1
- else:
- existing = self._graph[rc.caller_id][rc.callee_id]
- if rc.confidence > existing.get("confidence", 0):
- existing["confidence"] = rc.confidence
- if progress:
- progress.on_item_done("graph.calls")
-
- if progress:
- _phase_done = getattr(progress, "on_phase_done", None)
- if _phase_done is not None:
- _phase_done("graph.calls")
- log.info("Call edges resolved", total=total_resolved)
-
- def graph(self) -> nx.DiGraph:
- """Return the graph (building it first if necessary)."""
- if not self._built:
- self.build()
- return self._graph
-
- # ------------------------------------------------------------------
- # Graph metrics
- # ------------------------------------------------------------------
-
- def strongly_connected_components(self) -> list[frozenset[str]]:
- """Return SCCs as a list of frozensets."""
- return [frozenset(scc) for scc in nx.strongly_connected_components(self.file_subgraph())]
-
- def betweenness_centrality(self) -> dict[str, float]:
- """Return betweenness centrality for file nodes (cached)."""
- if self._betweenness_cache is not None:
- return self._betweenness_cache
- g = self.file_subgraph()
- n = g.number_of_nodes()
- if n == 0:
- self._betweenness_cache = {}
- return self._betweenness_cache
- if n > _LARGE_REPO_THRESHOLD:
- k = min(500, n)
- self._betweenness_cache = nx.betweenness_centrality(g, k=k, normalized=True)
- else:
- self._betweenness_cache = nx.betweenness_centrality(g, normalized=True)
- return self._betweenness_cache
-
- def community_detection(self) -> dict[str, int]:
- """Assign a community ID to each file node."""
- if self._community_cache is not None:
- return self._community_cache
-
- from repowise.core.analysis.communities import detect_file_communities
-
- try:
- assignment, info, algo = detect_file_communities(self._graph)
- self._community_cache = assignment
- self._community_info_cache = info
- self._community_algo = algo
- except Exception as exc:
- log.warning("community_detection_failed", error=str(exc))
- file_nodes = [
- n for n, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"
- ]
- self._community_cache = {n: 0 for n in file_nodes}
- self._community_info_cache = {}
- self._community_algo = "failed"
- return self._community_cache
-
- def symbol_communities(self) -> dict[str, int]:
- """Assign a community ID to each symbol node using call/heritage edges."""
- if self._symbol_community_cache is not None:
- return self._symbol_community_cache
-
- from repowise.core.analysis.communities import detect_symbol_communities
-
- try:
- self._symbol_community_cache = detect_symbol_communities(self._graph)
- except Exception as exc:
- log.warning("symbol_community_detection_failed", error=str(exc))
- self._symbol_community_cache = {}
- return self._symbol_community_cache
-
- def community_info(self) -> dict[int, Any]:
- """Return metadata for each file-level community."""
- if self._community_info_cache is None:
- self.community_detection()
- return self._community_info_cache or {}
-
- def execution_flows(self, config: Any | None = None) -> Any:
- """Trace execution flows from entry-point symbols (cached when ``config`` is None)."""
- from repowise.core.analysis.execution_flows import (
- ExecutionFlowReport,
- trace_execution_flows,
- )
-
- # Only the no-config path is cached — callers that pass custom
- # FlowConfig still get a fresh trace.
- if config is None and self._execution_flow_cache is not None:
- return self._execution_flow_cache
-
- file_cd = self.community_detection()
- merged_cd: dict[str, int] = dict(file_cd)
-
- sym_cd = self.symbol_communities()
- merged_cd.update(sym_cd)
-
- for node_id in self._graph.nodes():
- if node_id not in merged_cd and "::" in node_id:
- file_path = node_id.split("::")[0]
- if file_path in file_cd:
- merged_cd[node_id] = file_cd[file_path]
-
- try:
- report = trace_execution_flows(self._graph, merged_cd, config)
- except Exception as exc:
- log.warning("execution_flow_tracing_failed", error=str(exc))
- report = ExecutionFlowReport(
- total_entry_points_scored=0,
- total_flows=0,
- flows=[],
- )
- if config is None:
- self._execution_flow_cache = report
- return report
-
- async def compute_metrics_parallel(self) -> None:
- """Eagerly populate all metric caches with fan-out parallelism.
-
- Runs PageRank, betweenness, file/symbol community detection in
- parallel via ``asyncio.gather`` + ``asyncio.to_thread`` (the
- scipy- and igraph-backed kernels release the GIL during heavy
- compute, so true parallelism is achievable). Execution flows then
- run after, since they depend on the community caches.
-
- Calling this is optional — every metric falls back to lazy
- computation, so existing call sites keep working unchanged.
- """
- import asyncio as _asyncio
-
- await _asyncio.gather(
- _asyncio.to_thread(self.pagerank),
- _asyncio.to_thread(self.betweenness_centrality),
- _asyncio.to_thread(self.symbol_pagerank),
- _asyncio.to_thread(self.symbol_betweenness_centrality),
- _asyncio.to_thread(self.community_detection),
- _asyncio.to_thread(self.symbol_communities),
- )
- # execution_flows reads from the community caches just primed above.
- await _asyncio.to_thread(self.execution_flows)
-
- # ------------------------------------------------------------------
- # Serialisation
- # ------------------------------------------------------------------
-
- def to_json(self) -> dict[str, Any]:
- """Serialize the graph to a JSON-compatible dict (node-link format)."""
- return nx.node_link_data(self.graph())
-
- async def persist(self, db_path: Path, repo_id: str) -> None:
- """Persist the graph to an SQLite database."""
- import aiosqlite
- import sqlite3
-
- pr = self.pagerank()
- bc = self.betweenness_centrality()
- scc_map = self._build_scc_map()
- cd = self.community_detection()
- g = self.graph()
-
- async with aiosqlite.connect(db_path) as db:
- await db.executescript("""
- CREATE TABLE IF NOT EXISTS graph_nodes (
- repo_id TEXT NOT NULL,
- path TEXT NOT NULL,
- language TEXT,
- symbol_count INTEGER,
- has_error INTEGER,
- pagerank REAL,
- betweenness REAL,
- scc_id INTEGER,
- community_id INTEGER DEFAULT 0,
- PRIMARY KEY (repo_id, path)
- );
- CREATE TABLE IF NOT EXISTS graph_edges (
- repo_id TEXT NOT NULL,
- source_path TEXT NOT NULL,
- target_path TEXT NOT NULL,
- imported_names TEXT,
- PRIMARY KEY (repo_id, source_path, target_path)
- );
- """)
-
- try:
- await db.execute(
- "ALTER TABLE graph_nodes ADD COLUMN community_id INTEGER DEFAULT 0"
- )
- except sqlite3.OperationalError:
- pass
-
- node_rows = [
- (
- repo_id,
- path,
- data.get("language", ""),
- data.get("symbol_count", 0),
- int(data.get("has_error", False)),
- pr.get(path, 0.0),
- bc.get(path, 0.0),
- scc_map.get(path, 0),
- cd.get(path, cd.get(data.get("file_path", ""), 0)),
- )
- for path, data in g.nodes(data=True)
- ]
- await db.executemany(
- "INSERT OR REPLACE INTO graph_nodes VALUES (?,?,?,?,?,?,?,?,?)",
- node_rows,
- )
-
- edge_rows = [
- (
- repo_id,
- src,
- dst,
- json.dumps(data.get("imported_names", [])),
- )
- for src, dst, data in g.edges(data=True)
- ]
- await db.executemany(
- "INSERT OR REPLACE INTO graph_edges VALUES (?,?,?,?)",
- edge_rows,
- )
-
- await db.commit()
-
- log.info(
- "Graph persisted",
- db_path=str(db_path),
- nodes=len(node_rows),
- edges=len(edge_rows),
- )
-
- # ------------------------------------------------------------------
- # Internal helpers
- # ------------------------------------------------------------------
-
- def _build_stem_map(self, path_set: set[str]) -> dict[str, list[str]]:
- """Map import-stems to candidate file paths, sorted best-first."""
- buckets: dict[str, list[str]] = {}
- for p in path_set:
- path_obj = Path(p)
- if path_obj.name == "__init__.py":
- parent = path_obj.parent.name
- if not parent:
- continue
- stem = parent.lower()
- else:
- stem = path_obj.stem.lower()
- buckets.setdefault(stem, []).append(p)
-
- for stem, paths in buckets.items():
- paths.sort(key=lambda candidate: _stem_priority(candidate, stem))
- return buckets
-
- # ------------------------------------------------------------------
- # Co-change edges
- # ------------------------------------------------------------------
-
- def add_co_change_edges(self, git_meta_map: dict, min_count: int = 3) -> int:
- """Add co_changes edges from git metadata. Returns count of edges added."""
- count = 0
- seen: set[tuple[str, str]] = set()
-
- for file_path, meta in git_meta_map.items():
- co_json = meta.get("co_change_partners_json", "[]")
- if isinstance(co_json, str):
- try:
- partners = json.loads(co_json)
- except Exception:
- partners = []
- else:
- partners = co_json
-
- for partner in partners:
- partner_path = partner.get("file_path", "")
- co_count = partner.get("co_change_count", 0)
- if co_count < min_count:
- continue
- if partner_path not in self._graph:
- continue
-
- pair = tuple(sorted([file_path, partner_path]))
- if pair in seen:
- continue
- seen.add(pair)
-
- if not self._graph.has_edge(file_path, partner_path) and not self._graph.has_edge(
- partner_path, file_path
- ):
- self._graph.add_edge(
- file_path,
- partner_path,
- edge_type="co_changes",
- weight=co_count,
- imported_names=[],
- )
- count += 1
-
- log.info("Co-change edges added", count=count)
- return count
-
- def update_co_change_edges(self, updated_meta: dict, min_count: int = 3) -> None:
- """Remove old co_changes edges for updated files, add new ones."""
- edges_to_remove = []
- for u, v, data in self._graph.edges(data=True):
- if data.get("edge_type") == "co_changes" and (u in updated_meta or v in updated_meta):
- edges_to_remove.append((u, v))
- self._graph.remove_edges_from(edges_to_remove)
- self.add_co_change_edges(updated_meta, min_count)
-
- # ------------------------------------------------------------------
- # Dynamic-hint edges
- # ------------------------------------------------------------------
-
- def add_dynamic_edges(self, edges: list) -> None:
- """Add dynamic-hint edges to the graph. Each edge is a DynamicEdge."""
- for e in edges:
- if e.source not in self._graph:
- continue
- if e.target not in self._graph:
- self._graph.add_node(e.target)
- sub_type = e.edge_type or "dynamic"
- graph_edge_type = (
- sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"
- )
- self._graph.add_edge(
- e.source,
- e.target,
- edge_type=graph_edge_type,
- hint_source=e.hint_source,
- weight=e.weight,
- )
-
- # ------------------------------------------------------------------
- # Framework-aware synthetic edges
- # ------------------------------------------------------------------
-
- def add_framework_edges(self, tech_stack: list[str] | None = None) -> int:
- """Add synthetic edges for framework-mediated relationships.
-
- Returns the number of edges added.
- """
- from .framework_edges import add_framework_edges
-
- path_set = set(self._parsed_files.keys())
- stem_map = self._build_stem_map(path_set)
-
- go_modules = read_go_modules(self._repo_path)
- ctx = ResolverContext(
- path_set=path_set,
- stem_map=stem_map,
- graph=self._graph,
- repo_path=self._repo_path,
- tsconfig_resolver=self._tsconfig_resolver,
- go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),
- go_modules=go_modules,
- has_sfc_files=any(
- p.endswith((".vue", ".svelte", ".astro")) for p in path_set
- ),
- parsed_files=self._parsed_files,
- )
-
- count = add_framework_edges(self._graph, self._parsed_files, ctx, tech_stack)
- if count:
- log.info("Framework edges added", count=count)
- return count
-
- def file_subgraph(self) -> nx.DiGraph:
- """Return a subgraph containing only file-level nodes and import edges."""
- g = self.graph()
- file_nodes = [
- n for n, d in g.nodes(data=True) if d.get("node_type", "file") in ("file", "external")
- ]
- sub = g.subgraph(file_nodes).copy()
- edges_to_remove = [
- (u, v) for u, v, d in sub.edges(data=True) if d.get("edge_type") in ("co_changes",)
- ]
- sub.remove_edges_from(edges_to_remove)
- return sub
-
- def pagerank(self, alpha: float = 0.85) -> dict[str, float]:
- """Return PageRank scores for file nodes only (cached)."""
- if self._pagerank_cache is not None:
- return self._pagerank_cache
- filtered = self.file_subgraph()
- if filtered.number_of_nodes() == 0:
- self._pagerank_cache = {}
- return self._pagerank_cache
-
- try:
- self._pagerank_cache = nx.pagerank(filtered, alpha=alpha)
- except nx.PowerIterationFailedConvergence:
- log.warning("PageRank did not converge, using uniform scores")
- n = filtered.number_of_nodes()
- self._pagerank_cache = {node: 1.0 / n for node in filtered.nodes()}
- return self._pagerank_cache
-
- def symbol_subgraph(self) -> nx.DiGraph:
- """Return a subgraph of symbol nodes connected by call + heritage edges.
-
- File-to-symbol ``defines`` edges and class-to-method ``has_method``
- ownership edges are dropped so that the resulting centrality
- scores reflect call/heritage flow rather than containment.
- """
- g = self.graph()
- symbol_nodes = [
- n for n, d in g.nodes(data=True) if d.get("node_type") == "symbol"
- ]
- sub = g.subgraph(symbol_nodes).copy()
- edges_to_remove = [
- (u, v)
- for u, v, d in sub.edges(data=True)
- if d.get("edge_type") not in ("calls", "extends", "implements")
- ]
- sub.remove_edges_from(edges_to_remove)
- return sub
-
- def symbol_pagerank(self, alpha: float = 0.85) -> dict[str, float]:
- """Return PageRank scores for symbol nodes only (cached).
-
- Computed on the call/heritage symbol subgraph — this is what the
- UI's per-symbol "graph metrics" panel reads. Without it every
- symbol shows ``Not indexed in graph``.
- """
- if self._symbol_pagerank_cache is not None:
- return self._symbol_pagerank_cache
- sub = self.symbol_subgraph()
- if sub.number_of_nodes() == 0:
- self._symbol_pagerank_cache = {}
- return self._symbol_pagerank_cache
- try:
- self._symbol_pagerank_cache = nx.pagerank(sub, alpha=alpha)
- except nx.PowerIterationFailedConvergence:
- log.warning("Symbol PageRank did not converge, using uniform scores")
- n = sub.number_of_nodes()
- self._symbol_pagerank_cache = {node: 1.0 / n for node in sub.nodes()}
- return self._symbol_pagerank_cache
-
- def symbol_betweenness_centrality(self) -> dict[str, float]:
- """Return betweenness centrality for symbol nodes (cached)."""
- if self._symbol_betweenness_cache is not None:
- return self._symbol_betweenness_cache
- sub = self.symbol_subgraph()
- n = sub.number_of_nodes()
- if n == 0:
- self._symbol_betweenness_cache = {}
- return self._symbol_betweenness_cache
- if n > _LARGE_REPO_THRESHOLD:
- k = min(500, n)
- self._symbol_betweenness_cache = nx.betweenness_centrality(
- sub, k=k, normalized=True
- )
- else:
- self._symbol_betweenness_cache = nx.betweenness_centrality(
- sub, normalized=True
- )
- return self._symbol_betweenness_cache
-
- def _build_scc_map(self) -> dict[str, int]:
- """Assign a numeric SCC ID to each node."""
- result: dict[str, int] = {}
- for scc_id, scc in enumerate(nx.strongly_connected_components(self.graph())):
- for node in scc:
- result[node] = scc_id
- return result
diff --git a/packages/core/src/repowise/core/ingestion/graph/README.md b/packages/core/src/repowise/core/ingestion/graph/README.md
new file mode 100644
index 000000000..33147a499
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/README.md
@@ -0,0 +1,71 @@
+# `graph`
+
+Builds the directed dependency graph for a repository from `ParsedFile`
+objects and exposes the graph metrics (PageRank, betweenness, communities,
+degrees) that downstream selection, generation, and risk analysis consume.
+
+## Purpose
+
+`GraphBuilder` is the analysis stage between parsing and generation. It adds
+file/symbol nodes, resolves imports, heritage, member reads, and calls into a
+NetworkX `DiGraph`, then computes centrality and community metrics over
+file- and symbol-level subgraphs.
+
+## Public API
+
+```python
+from repowise.core.ingestion.graph import GraphBuilder
+
+b = GraphBuilder(repo_path)
+for parsed in parsed_files:
+ b.add_file(parsed)
+b.build()
+pr = b.pagerank() # dict[path, float]
+bc = b.betweenness_centrality()
+cd = b.community_detection()
+ind = b.in_degree() # dict[path, int]
+```
+
+Edge augmentation: `add_co_change_edges`, `add_dynamic_edges`,
+`add_framework_edges`. Serialisation: `to_json`, `persist`.
+
+### Large-repo SQL metric routing
+
+For large repos the file-level metrics are materialized to the `graph_metrics`
+table (see `persistence.crud.batch_upsert_graph_metrics`). After that:
+
+- `b.file_metrics_snapshot()` returns the `node_id → {pagerank, betweenness,
+ community_id, in_degree, out_degree}` dict written to SQL.
+- `b.load_metrics_from_sql(rows)` pre-fills the metric caches from the
+ materialized snapshot, so the expensive NetworkX kernels are never recomputed
+ — reads are served from SQL.
+- `b.release_graph()` drops the in-memory NetworkX object once metrics are
+ loaded, for callers that no longer need traversal (e.g. the fast-mode
+ pipeline, which generates no docs).
+
+This is gated by the pipeline knob `OrchestratorMode` / `sql_backed_metrics`
+(default off in standard mode; on in fast mode). Standard generation keeps the
+graph live because it traverses it for context assembly.
+
+## Internal layout
+
+- `builder.py` — `GraphBuilder` core: node/edge construction + `build()`.
+- `_metrics.py` — `MetricsMixin`: centrality, communities, degrees, SQL routing.
+- `_resolvers.py` — `ResolveMixin`: heritage / member-read / call passes.
+- `_edges.py` — `EdgesMixin`: co-change / dynamic / framework edges.
+- `_serialize.py` — `SerializeMixin`: node-link JSON + SQLite export.
+- `_stem.py` — import-stem priority + stem-map construction.
+
+## Extension points
+
+- New edge kind: add a method to `EdgesMixin` or a resolution pass to
+ `ResolveMixin`, then call it from `build()`.
+- New metric: add a cached method to `MetricsMixin` and include it in
+ `file_metrics_snapshot()` if it should be materialized.
+
+## Tests
+
+- `tests/unit/ingestion/test_graph.py`
+- `tests/unit/ingestion/test_graph_metrics_sql.py`
+- `tests/integration/persistence/test_graph_metrics_pg.py`
+"""
diff --git a/packages/core/src/repowise/core/ingestion/graph/__init__.py b/packages/core/src/repowise/core/ingestion/graph/__init__.py
new file mode 100644
index 000000000..0772cdfca
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/__init__.py
@@ -0,0 +1,40 @@
+"""Dependency graph builder for the repowise ingestion pipeline.
+
+GraphBuilder constructs a directed graph from ParsedFile objects with two
+tiers of nodes:
+
+ File-level nodes:
+ "file" — every source file
+ "external" — third-party / unresolvable imports (prefix "external:")
+
+ Symbol-level nodes:
+ "symbol" — functions, classes, methods, interfaces, etc.
+ keyed by Symbol.id (e.g. "src/app.py::main")
+
+Edge types:
+ "imports" — file-to-file import relationship
+ "defines" — file-to-symbol containment
+ "has_method" — class-to-method ownership
+ "calls" — symbol-to-symbol call relationship (with confidence)
+
+After calling build(), graph metrics are available:
+ pagerank() — dict[path, float]
+ strongly_connected_components() — list[frozenset[str]]
+ betweenness_centrality() — dict[path, float]
+ in_degree() / out_degree() — dict[path, int]
+
+Internal layout (kept under the 400-line ceiling):
+ builder.py — GraphBuilder core (nodes, edges, build lifecycle)
+ _metrics.py — centrality / community / degree metrics (+ SQL routing)
+ _resolvers.py — heritage / member-read / call resolution passes
+ _edges.py — co-change / dynamic / framework edges
+ _serialize.py — node-link JSON + SQLite export
+ _stem.py — import-stem resolution helpers
+"""
+
+from __future__ import annotations
+
+from ._stem import _stem_priority, build_stem_map
+from .builder import GraphBuilder
+
+__all__ = ["GraphBuilder", "_stem_priority", "build_stem_map"]
diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.py
new file mode 100644
index 000000000..f159acd98
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py
@@ -0,0 +1,121 @@
+"""Co-change, dynamic-hint, and framework-aware edge construction.
+
+Mixed into :class:`GraphBuilder`. These passes add edges that static import
+resolution cannot infer: git co-change relations, dynamic-dispatch hints, and
+framework-mediated wiring.
+"""
+
+from __future__ import annotations
+
+import json
+
+import structlog
+
+from ..resolvers import ResolverContext
+from ..resolvers.go import read_go_module_path, read_go_modules
+from ._stem import build_stem_map
+
+log = structlog.get_logger(__name__)
+
+
+class EdgesMixin:
+ """Non-import edge construction for :class:`GraphBuilder`."""
+
+ def add_co_change_edges(self, git_meta_map: dict, min_count: int = 3) -> int:
+ """Add co_changes edges from git metadata. Returns count of edges added."""
+ count = 0
+ seen: set[tuple[str, str]] = set()
+
+ for file_path, meta in git_meta_map.items():
+ co_json = meta.get("co_change_partners_json", "[]")
+ if isinstance(co_json, str):
+ try:
+ partners = json.loads(co_json)
+ except Exception:
+ partners = []
+ else:
+ partners = co_json
+
+ for partner in partners:
+ partner_path = partner.get("file_path", "")
+ co_count = partner.get("co_change_count", 0)
+ if co_count < min_count:
+ continue
+ if partner_path not in self._graph:
+ continue
+
+ pair = tuple(sorted([file_path, partner_path]))
+ if pair in seen:
+ continue
+ seen.add(pair)
+
+ if not self._graph.has_edge(file_path, partner_path) and not self._graph.has_edge(
+ partner_path, file_path
+ ):
+ self._graph.add_edge(
+ file_path,
+ partner_path,
+ edge_type="co_changes",
+ weight=co_count,
+ imported_names=[],
+ )
+ count += 1
+
+ log.info("Co-change edges added", count=count)
+ return count
+
+ def update_co_change_edges(self, updated_meta: dict, min_count: int = 3) -> None:
+ """Remove old co_changes edges for updated files, add new ones."""
+ edges_to_remove = []
+ for u, v, data in self._graph.edges(data=True):
+ if data.get("edge_type") == "co_changes" and (u in updated_meta or v in updated_meta):
+ edges_to_remove.append((u, v))
+ self._graph.remove_edges_from(edges_to_remove)
+ self.add_co_change_edges(updated_meta, min_count)
+
+ def add_dynamic_edges(self, edges: list) -> None:
+ """Add dynamic-hint edges to the graph. Each edge is a DynamicEdge."""
+ for e in edges:
+ if e.source not in self._graph:
+ continue
+ if e.target not in self._graph:
+ self._graph.add_node(e.target)
+ sub_type = e.edge_type or "dynamic"
+ graph_edge_type = (
+ sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"
+ )
+ self._graph.add_edge(
+ e.source,
+ e.target,
+ edge_type=graph_edge_type,
+ hint_source=e.hint_source,
+ weight=e.weight,
+ )
+
+ def add_framework_edges(self, tech_stack: list[str] | None = None) -> int:
+ """Add synthetic edges for framework-mediated relationships.
+
+ Returns the number of edges added.
+ """
+ from ..framework_edges import add_framework_edges
+
+ path_set = set(self._parsed_files.keys())
+ stem_map = build_stem_map(path_set)
+
+ go_modules = read_go_modules(self._repo_path)
+ ctx = ResolverContext(
+ path_set=path_set,
+ stem_map=stem_map,
+ graph=self._graph,
+ repo_path=self._repo_path,
+ tsconfig_resolver=self._tsconfig_resolver,
+ go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),
+ go_modules=go_modules,
+ has_sfc_files=any(p.endswith((".vue", ".svelte", ".astro")) for p in path_set),
+ parsed_files=self._parsed_files,
+ )
+
+ count = add_framework_edges(self._graph, self._parsed_files, ctx, tech_stack)
+ if count:
+ log.info("Framework edges added", count=count)
+ return count
diff --git a/packages/core/src/repowise/core/ingestion/graph/_metrics.py b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
new file mode 100644
index 000000000..052a8de7f
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
@@ -0,0 +1,325 @@
+"""Graph metric computation (PageRank, betweenness, communities, degrees).
+
+These methods are mixed into :class:`GraphBuilder`. They read the metric
+caches and ``_graph`` set up in the builder's ``__init__``.
+
+Large-repo SQL routing
+----------------------
+When metric values have been materialized to SQL (the ``graph_metrics``
+table), :meth:`load_metrics_from_sql` pre-fills the file-level caches so the
+expensive NetworkX kernels (notably betweenness on 30k+ nodes) are never
+recomputed — subsequent reads are served straight from the materialized
+snapshot. The structural graph stays available for traversal.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import networkx as nx
+import structlog
+
+log = structlog.get_logger(__name__)
+
+_LARGE_REPO_THRESHOLD = 30_000 # nodes — above this, algorithms are expensive
+
+
+class MetricsMixin:
+ """Centrality + community + degree metrics for :class:`GraphBuilder`."""
+
+ # ------------------------------------------------------------------
+ # SQL-backed metric routing (large repos)
+ # ------------------------------------------------------------------
+
+ def load_metrics_from_sql(self, metrics: dict[str, dict[str, Any]]) -> None:
+ """Pre-fill the file-level metric caches from a materialized snapshot.
+
+ *metrics* maps ``node_id`` → a dict with any of ``pagerank``,
+ ``betweenness``, ``community_id``, ``in_degree``, ``out_degree``.
+ After this call, the corresponding metric methods return the snapshot
+ values without recomputing them via NetworkX.
+ """
+ self._pagerank_cache = {n: float(m.get("pagerank", 0.0)) for n, m in metrics.items()}
+ self._betweenness_cache = {
+ n: float(m.get("betweenness", 0.0)) for n, m in metrics.items()
+ }
+ self._community_cache = {n: int(m.get("community_id", 0)) for n, m in metrics.items()}
+ self._in_degree_cache = {n: int(m.get("in_degree", 0)) for n, m in metrics.items()}
+ self._out_degree_cache = {n: int(m.get("out_degree", 0)) for n, m in metrics.items()}
+ log.info("graph.metrics_loaded_from_sql", nodes=len(metrics))
+
+ def file_metrics_snapshot(self) -> dict[str, dict[str, Any]]:
+ """Return the file-level metrics as a ``node_id → metrics`` dict.
+
+ Used to materialize the ``graph_metrics`` table. Computes every metric
+ from NetworkX (or returns the cached/SQL-loaded values when present).
+ """
+ pr = self.pagerank()
+ bc = self.betweenness_centrality()
+ cd = self.community_detection()
+ ind = self.in_degree()
+ outd = self.out_degree()
+ nodes = set(pr) | set(bc) | set(cd) | set(ind) | set(outd)
+ return {
+ n: {
+ "pagerank": pr.get(n, 0.0),
+ "betweenness": bc.get(n, 0.0),
+ "community_id": cd.get(n, 0),
+ "in_degree": ind.get(n, 0),
+ "out_degree": outd.get(n, 0),
+ }
+ for n in nodes
+ }
+
+ # ------------------------------------------------------------------
+ # Subgraphs
+ # ------------------------------------------------------------------
+
+ def file_subgraph(self) -> nx.DiGraph:
+ """Return a subgraph containing only file-level nodes and import edges."""
+ g = self.graph()
+ file_nodes = [
+ n for n, d in g.nodes(data=True) if d.get("node_type", "file") in ("file", "external")
+ ]
+ sub = g.subgraph(file_nodes).copy()
+ edges_to_remove = [
+ (u, v) for u, v, d in sub.edges(data=True) if d.get("edge_type") in ("co_changes",)
+ ]
+ sub.remove_edges_from(edges_to_remove)
+ return sub
+
+ def symbol_subgraph(self) -> nx.DiGraph:
+ """Return a subgraph of symbol nodes connected by call + heritage edges.
+
+ File-to-symbol ``defines`` edges and class-to-method ``has_method``
+ ownership edges are dropped so that the resulting centrality
+ scores reflect call/heritage flow rather than containment.
+ """
+ g = self.graph()
+ symbol_nodes = [n for n, d in g.nodes(data=True) if d.get("node_type") == "symbol"]
+ sub = g.subgraph(symbol_nodes).copy()
+ edges_to_remove = [
+ (u, v)
+ for u, v, d in sub.edges(data=True)
+ if d.get("edge_type") not in ("calls", "extends", "implements")
+ ]
+ sub.remove_edges_from(edges_to_remove)
+ return sub
+
+ # ------------------------------------------------------------------
+ # File-level metrics
+ # ------------------------------------------------------------------
+
+ def strongly_connected_components(self) -> list[frozenset[str]]:
+ """Return SCCs as a list of frozensets."""
+ return [frozenset(scc) for scc in nx.strongly_connected_components(self.file_subgraph())]
+
+ def pagerank(self, alpha: float = 0.85) -> dict[str, float]:
+ """Return PageRank scores for file nodes only (cached)."""
+ if self._pagerank_cache is not None:
+ return self._pagerank_cache
+ filtered = self.file_subgraph()
+ if filtered.number_of_nodes() == 0:
+ self._pagerank_cache = {}
+ return self._pagerank_cache
+
+ try:
+ self._pagerank_cache = nx.pagerank(filtered, alpha=alpha)
+ except nx.PowerIterationFailedConvergence:
+ log.warning("PageRank did not converge, using uniform scores")
+ n = filtered.number_of_nodes()
+ self._pagerank_cache = {node: 1.0 / n for node in filtered.nodes()}
+ return self._pagerank_cache
+
+ def betweenness_centrality(self) -> dict[str, float]:
+ """Return betweenness centrality for file nodes (cached)."""
+ if self._betweenness_cache is not None:
+ return self._betweenness_cache
+ g = self.file_subgraph()
+ n = g.number_of_nodes()
+ if n == 0:
+ self._betweenness_cache = {}
+ return self._betweenness_cache
+ if n > _LARGE_REPO_THRESHOLD:
+ k = min(500, n)
+ self._betweenness_cache = nx.betweenness_centrality(g, k=k, normalized=True)
+ else:
+ self._betweenness_cache = nx.betweenness_centrality(g, normalized=True)
+ return self._betweenness_cache
+
+ def in_degree(self) -> dict[str, int]:
+ """Return in-degree (number of importers) for each file node (cached)."""
+ if self._in_degree_cache is not None:
+ return self._in_degree_cache
+ g = self.file_subgraph()
+ self._in_degree_cache = {n: int(d) for n, d in g.in_degree()}
+ return self._in_degree_cache
+
+ def out_degree(self) -> dict[str, int]:
+ """Return out-degree (number of dependencies) for each file node (cached)."""
+ if self._out_degree_cache is not None:
+ return self._out_degree_cache
+ g = self.file_subgraph()
+ self._out_degree_cache = {n: int(d) for n, d in g.out_degree()}
+ return self._out_degree_cache
+
+ # ------------------------------------------------------------------
+ # Communities
+ # ------------------------------------------------------------------
+
+ def community_detection(self) -> dict[str, int]:
+ """Assign a community ID to each file node."""
+ if self._community_cache is not None:
+ return self._community_cache
+
+ from repowise.core.analysis.communities import detect_file_communities
+
+ try:
+ assignment, info, algo = detect_file_communities(self._graph)
+ self._community_cache = assignment
+ self._community_info_cache = info
+ self._community_algo = algo
+ except Exception as exc:
+ log.warning("community_detection_failed", error=str(exc))
+ file_nodes = [
+ n for n, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"
+ ]
+ self._community_cache = {n: 0 for n in file_nodes}
+ self._community_info_cache = {}
+ self._community_algo = "failed"
+ return self._community_cache
+
+ def symbol_communities(self) -> dict[str, int]:
+ """Assign a community ID to each symbol node using call/heritage edges."""
+ if self._symbol_community_cache is not None:
+ return self._symbol_community_cache
+
+ from repowise.core.analysis.communities import detect_symbol_communities
+
+ try:
+ self._symbol_community_cache = detect_symbol_communities(self._graph)
+ except Exception as exc:
+ log.warning("symbol_community_detection_failed", error=str(exc))
+ self._symbol_community_cache = {}
+ return self._symbol_community_cache
+
+ def community_info(self) -> dict[int, Any]:
+ """Return metadata for each file-level community."""
+ if self._community_info_cache is None:
+ self.community_detection()
+ return self._community_info_cache or {}
+
+ # ------------------------------------------------------------------
+ # Symbol-level metrics
+ # ------------------------------------------------------------------
+
+ def symbol_pagerank(self, alpha: float = 0.85) -> dict[str, float]:
+ """Return PageRank scores for symbol nodes only (cached).
+
+ Computed on the call/heritage symbol subgraph — this is what the
+ UI's per-symbol "graph metrics" panel reads. Without it every
+ symbol shows ``Not indexed in graph``.
+ """
+ if self._symbol_pagerank_cache is not None:
+ return self._symbol_pagerank_cache
+ sub = self.symbol_subgraph()
+ if sub.number_of_nodes() == 0:
+ self._symbol_pagerank_cache = {}
+ return self._symbol_pagerank_cache
+ try:
+ self._symbol_pagerank_cache = nx.pagerank(sub, alpha=alpha)
+ except nx.PowerIterationFailedConvergence:
+ log.warning("Symbol PageRank did not converge, using uniform scores")
+ n = sub.number_of_nodes()
+ self._symbol_pagerank_cache = {node: 1.0 / n for node in sub.nodes()}
+ return self._symbol_pagerank_cache
+
+ def symbol_betweenness_centrality(self) -> dict[str, float]:
+ """Return betweenness centrality for symbol nodes (cached)."""
+ if self._symbol_betweenness_cache is not None:
+ return self._symbol_betweenness_cache
+ sub = self.symbol_subgraph()
+ n = sub.number_of_nodes()
+ if n == 0:
+ self._symbol_betweenness_cache = {}
+ return self._symbol_betweenness_cache
+ if n > _LARGE_REPO_THRESHOLD:
+ k = min(500, n)
+ self._symbol_betweenness_cache = nx.betweenness_centrality(sub, k=k, normalized=True)
+ else:
+ self._symbol_betweenness_cache = nx.betweenness_centrality(sub, normalized=True)
+ return self._symbol_betweenness_cache
+
+ # ------------------------------------------------------------------
+ # Execution flows + bulk priming
+ # ------------------------------------------------------------------
+
+ def execution_flows(self, config: Any | None = None) -> Any:
+ """Trace execution flows from entry-point symbols (cached when ``config`` is None)."""
+ from repowise.core.analysis.execution_flows import (
+ ExecutionFlowReport,
+ trace_execution_flows,
+ )
+
+ # Only the no-config path is cached — callers that pass custom
+ # FlowConfig still get a fresh trace.
+ if config is None and self._execution_flow_cache is not None:
+ return self._execution_flow_cache
+
+ file_cd = self.community_detection()
+ merged_cd: dict[str, int] = dict(file_cd)
+
+ sym_cd = self.symbol_communities()
+ merged_cd.update(sym_cd)
+
+ for node_id in self._graph.nodes():
+ if node_id not in merged_cd and "::" in node_id:
+ file_path = node_id.split("::")[0]
+ if file_path in file_cd:
+ merged_cd[node_id] = file_cd[file_path]
+
+ try:
+ report = trace_execution_flows(self._graph, merged_cd, config)
+ except Exception as exc:
+ log.warning("execution_flow_tracing_failed", error=str(exc))
+ report = ExecutionFlowReport(
+ total_entry_points_scored=0,
+ total_flows=0,
+ flows=[],
+ )
+ if config is None:
+ self._execution_flow_cache = report
+ return report
+
+ async def compute_metrics_parallel(self) -> None:
+ """Eagerly populate all metric caches with fan-out parallelism.
+
+ Runs PageRank, betweenness, file/symbol community detection in
+ parallel via ``asyncio.gather`` + ``asyncio.to_thread`` (the
+ scipy- and igraph-backed kernels release the GIL during heavy
+ compute, so true parallelism is achievable). Execution flows then
+ run after, since they depend on the community caches.
+
+ Calling this is optional — every metric falls back to lazy
+ computation, so existing call sites keep working unchanged.
+ """
+ import asyncio as _asyncio
+
+ await _asyncio.gather(
+ _asyncio.to_thread(self.pagerank),
+ _asyncio.to_thread(self.betweenness_centrality),
+ _asyncio.to_thread(self.symbol_pagerank),
+ _asyncio.to_thread(self.symbol_betweenness_centrality),
+ _asyncio.to_thread(self.community_detection),
+ _asyncio.to_thread(self.symbol_communities),
+ )
+ # execution_flows reads from the community caches just primed above.
+ await _asyncio.to_thread(self.execution_flows)
+
+ def _build_scc_map(self) -> dict[str, int]:
+ """Assign a numeric SCC ID to each node."""
+ result: dict[str, int] = {}
+ for scc_id, scc in enumerate(nx.strongly_connected_components(self.graph())):
+ for node in scc:
+ result[node] = scc_id
+ return result
diff --git a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
new file mode 100644
index 000000000..164f86243
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py
@@ -0,0 +1,135 @@
+"""Heritage / member-read / call edge resolution for :class:`GraphBuilder`.
+
+Each pass reads ``self._parsed_files`` and mutates ``self._graph`` in place,
+emitting EXTENDS/IMPLEMENTS, ``reads``, and ``calls`` edges respectively.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+
+class ResolveMixin:
+ """Symbol-level edge resolution passes run during ``build()``."""
+
+ def _resolve_heritage(
+ self,
+ import_targets: dict[str, set[str]],
+ progress: Any | None = None,
+ ) -> None:
+ """Resolve heritage relations and add EXTENDS/IMPLEMENTS edges."""
+ from ..heritage_resolver import HeritageResolver
+
+ resolver = HeritageResolver(self._parsed_files, import_targets)
+ total_resolved = 0
+
+ files_with_heritage = [
+ (p, pf) for p, pf in self._parsed_files.items() if pf.heritage
+ ]
+ if progress:
+ progress.on_phase_start("graph.heritage", len(files_with_heritage))
+ for path, parsed in files_with_heritage:
+ resolved = resolver.resolve_file(path, parsed.heritage)
+ for rh in resolved:
+ if rh.child_id in self._graph and rh.parent_id in self._graph:
+ if not self._graph.has_edge(rh.child_id, rh.parent_id):
+ self._graph.add_edge(
+ rh.child_id,
+ rh.parent_id,
+ edge_type=rh.edge_type,
+ confidence=rh.confidence,
+ )
+ total_resolved += 1
+ else:
+ existing = self._graph[rh.child_id][rh.parent_id]
+ if rh.confidence > existing.get("confidence", 0):
+ existing["confidence"] = rh.confidence
+ if progress:
+ progress.on_item_done("graph.heritage")
+
+ if progress:
+ _phase_done = getattr(progress, "on_phase_done", None)
+ if _phase_done is not None:
+ _phase_done("graph.heritage")
+ log.info("Heritage edges resolved", total=total_resolved)
+
+ def _resolve_member_reads(self, progress: Any | None = None) -> None:
+ """Phase 1c: emit ``reads`` edges for C# property / member access.
+
+ Runs after type-use resolution so the dead-code analyser sees
+ member access as evidence of reachability. The pass is C#-only
+ today (the lever is largest there); the helper module is set
+ up to receive other languages via additional strategies.
+ """
+ from ..languages.csharp_member_reads import (
+ build_csharp_type_to_file,
+ collect_csharp_source_texts,
+ resolve_csharp_member_reads,
+ )
+
+ has_csharp = any(
+ pf.file_info.language == "csharp" for pf in self._parsed_files.values()
+ )
+ if not has_csharp:
+ return
+
+ phase = "graph.member_reads"
+ if progress:
+ progress.on_phase_start(phase, None)
+ try:
+ cs_texts = collect_csharp_source_texts(self._parsed_files)
+ type_to_file = build_csharp_type_to_file(self._parsed_files)
+ added = resolve_csharp_member_reads(self._graph, cs_texts, type_to_file)
+ log.info("member_read_edges", language="csharp", added=added)
+ except Exception as exc:
+ log.warning("member_reads_failed", error=str(exc))
+ finally:
+ if progress:
+ done = getattr(progress, "on_phase_done", None)
+ if callable(done):
+ done(phase)
+
+ def _resolve_calls(
+ self,
+ import_targets: dict[str, set[str]],
+ progress: Any | None = None,
+ ) -> None:
+ """Run three-tier call resolution and add CALLS edges to the graph."""
+ from ..call_resolver import CallResolver
+
+ resolver = CallResolver(self._parsed_files, import_targets)
+ total_resolved = 0
+
+ files_with_calls = [
+ (p, pf) for p, pf in self._parsed_files.items() if pf.calls
+ ]
+ if progress:
+ progress.on_phase_start("graph.calls", len(files_with_calls))
+ for path, parsed in files_with_calls:
+ resolved = resolver.resolve_file(path, parsed.calls)
+ for rc in resolved:
+ if rc.caller_id in self._graph and rc.callee_id in self._graph:
+ if not self._graph.has_edge(rc.caller_id, rc.callee_id):
+ self._graph.add_edge(
+ rc.caller_id,
+ rc.callee_id,
+ edge_type="calls",
+ confidence=rc.confidence,
+ )
+ total_resolved += 1
+ else:
+ existing = self._graph[rc.caller_id][rc.callee_id]
+ if rc.confidence > existing.get("confidence", 0):
+ existing["confidence"] = rc.confidence
+ if progress:
+ progress.on_item_done("graph.calls")
+
+ if progress:
+ _phase_done = getattr(progress, "on_phase_done", None)
+ if _phase_done is not None:
+ _phase_done("graph.calls")
+ log.info("Call edges resolved", total=total_resolved)
diff --git a/packages/core/src/repowise/core/ingestion/graph/_serialize.py b/packages/core/src/repowise/core/ingestion/graph/_serialize.py
new file mode 100644
index 000000000..2f2724470
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_serialize.py
@@ -0,0 +1,108 @@
+"""Graph serialisation: node-link JSON + standalone SQLite persistence.
+
+Mixed into :class:`GraphBuilder`. The SQLite ``persist`` path is the
+self-contained writer used by the standalone graph export; the main pipeline
+persists through ``persistence.crud`` instead.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+from pathlib import Path
+from typing import Any
+
+import networkx as nx
+import structlog
+
+log = structlog.get_logger(__name__)
+
+
+class SerializeMixin:
+ """JSON + SQLite serialisation for :class:`GraphBuilder`."""
+
+ def to_json(self) -> dict[str, Any]:
+ """Serialize the graph to a JSON-compatible dict (node-link format)."""
+ return nx.node_link_data(self.graph())
+
+ async def persist(self, db_path: Path, repo_id: str) -> None:
+ """Persist the graph to an SQLite database."""
+ import sqlite3
+
+ import aiosqlite
+
+ pr = self.pagerank()
+ bc = self.betweenness_centrality()
+ scc_map = self._build_scc_map()
+ cd = self.community_detection()
+ g = self.graph()
+
+ async with aiosqlite.connect(db_path) as db:
+ await db.executescript("""
+ CREATE TABLE IF NOT EXISTS graph_nodes (
+ repo_id TEXT NOT NULL,
+ path TEXT NOT NULL,
+ language TEXT,
+ symbol_count INTEGER,
+ has_error INTEGER,
+ pagerank REAL,
+ betweenness REAL,
+ scc_id INTEGER,
+ community_id INTEGER DEFAULT 0,
+ PRIMARY KEY (repo_id, path)
+ );
+ CREATE TABLE IF NOT EXISTS graph_edges (
+ repo_id TEXT NOT NULL,
+ source_path TEXT NOT NULL,
+ target_path TEXT NOT NULL,
+ imported_names TEXT,
+ PRIMARY KEY (repo_id, source_path, target_path)
+ );
+ """)
+
+ with contextlib.suppress(sqlite3.OperationalError):
+ await db.execute(
+ "ALTER TABLE graph_nodes ADD COLUMN community_id INTEGER DEFAULT 0"
+ )
+
+ node_rows = [
+ (
+ repo_id,
+ path,
+ data.get("language", ""),
+ data.get("symbol_count", 0),
+ int(data.get("has_error", False)),
+ pr.get(path, 0.0),
+ bc.get(path, 0.0),
+ scc_map.get(path, 0),
+ cd.get(path, cd.get(data.get("file_path", ""), 0)),
+ )
+ for path, data in g.nodes(data=True)
+ ]
+ await db.executemany(
+ "INSERT OR REPLACE INTO graph_nodes VALUES (?,?,?,?,?,?,?,?,?)",
+ node_rows,
+ )
+
+ edge_rows = [
+ (
+ repo_id,
+ src,
+ dst,
+ json.dumps(data.get("imported_names", [])),
+ )
+ for src, dst, data in g.edges(data=True)
+ ]
+ await db.executemany(
+ "INSERT OR REPLACE INTO graph_edges VALUES (?,?,?,?)",
+ edge_rows,
+ )
+
+ await db.commit()
+
+ log.info(
+ "Graph persisted",
+ db_path=str(db_path),
+ nodes=len(node_rows),
+ edges=len(edge_rows),
+ )
diff --git a/packages/core/src/repowise/core/ingestion/graph/_stem.py b/packages/core/src/repowise/core/ingestion/graph/_stem.py
new file mode 100644
index 000000000..6a37c5777
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/_stem.py
@@ -0,0 +1,64 @@
+"""Import-stem resolution helpers for the dependency graph builder."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+# Path segments that mark a file as low-value for stem-based import resolution.
+_LOW_VALUE_PATH_SEGMENTS = frozenset(
+ {
+ "tests",
+ "test",
+ "_tests",
+ "__tests__",
+ "testing",
+ "test_apps",
+ "testdata",
+ "test_data",
+ "fixtures",
+ "examples",
+ "example",
+ "samples",
+ "sample",
+ "scripts",
+ "benchmarks",
+ "bench",
+ "docs",
+ "doc",
+ }
+)
+
+
+def _stem_priority(path: str, stem: str) -> tuple[int, int, int, str]:
+ """Sort key for choosing among files that share an import stem.
+
+ Lower tuples sort first; callers take ``candidates[0]`` as the resolution.
+ """
+ path_obj = Path(path)
+ parts = path_obj.parts
+ if path_obj.name == "__init__.py":
+ parent_match = 0
+ else:
+ parent_dir = parts[-2].lower() if len(parts) >= 2 else ""
+ parent_match = 0 if parent_dir == stem else 1
+ low_value = 1 if any(seg.lower() in _LOW_VALUE_PATH_SEGMENTS for seg in parts) else 0
+ return (parent_match, low_value, len(parts), path)
+
+
+def build_stem_map(path_set: set[str]) -> dict[str, list[str]]:
+ """Map import-stems to candidate file paths, sorted best-first."""
+ buckets: dict[str, list[str]] = {}
+ for p in path_set:
+ path_obj = Path(p)
+ if path_obj.name == "__init__.py":
+ parent = path_obj.parent.name
+ if not parent:
+ continue
+ stem = parent.lower()
+ else:
+ stem = path_obj.stem.lower()
+ buckets.setdefault(stem, []).append(p)
+
+ for stem, paths in buckets.items():
+ paths.sort(key=lambda candidate: _stem_priority(candidate, stem))
+ return buckets
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
new file mode 100644
index 000000000..14fc94246
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -0,0 +1,325 @@
+"""GraphBuilder — constructs the directed dependency graph from ParsedFiles.
+
+This module holds the structural core (node/edge construction, import
+resolution driver, lifecycle). Metrics, edge augmentation, resolution passes,
+and serialisation are provided by mixins in sibling modules to keep every
+file under the project's 400-line ceiling.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import networkx as nx
+import structlog
+
+from ..models import ParsedFile
+from ..resolvers import ResolverContext, resolve_import
+from ..resolvers.go import read_go_module_path, read_go_modules
+from ..type_ref_resolution import resolve_type_refs
+from ._edges import EdgesMixin
+from ._metrics import MetricsMixin
+from ._resolvers import ResolveMixin
+from ._serialize import SerializeMixin
+from ._stem import build_stem_map
+
+log = structlog.get_logger(__name__)
+
+
+class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin):
+ """Build a dependency graph from a collection of ParsedFile objects.
+
+ Usage::
+
+ builder = GraphBuilder()
+ for parsed in parsed_files:
+ builder.add_file(parsed)
+ graph = builder.build()
+ pr = builder.pagerank()
+ """
+
+ def __init__(self, repo_path: Path | str | None = None) -> None:
+ self._graph: nx.DiGraph = nx.DiGraph()
+ self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile
+ self._built = False
+ self._repo_path: Path | None = Path(repo_path) if repo_path else None
+ self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)
+
+ # Community / flow / metric caches (invalidated on build)
+ self._community_cache: dict[str, int] | None = None
+ self._symbol_community_cache: dict[str, int] | None = None
+ self._community_info_cache: dict[int, Any] | None = None
+ self._community_algo: str = ""
+ self._pagerank_cache: dict[str, float] | None = None
+ self._betweenness_cache: dict[str, float] | None = None
+ self._in_degree_cache: dict[str, int] | None = None
+ self._out_degree_cache: dict[str, int] | None = None
+ self._symbol_pagerank_cache: dict[str, float] | None = None
+ self._symbol_betweenness_cache: dict[str, float] | None = None
+ self._execution_flow_cache: Any | None = None
+
+ def set_tsconfig_resolver(self, resolver: Any) -> None:
+ """Attach a :class:`TsconfigResolver` for TS/JS path-alias resolution."""
+ self._tsconfig_resolver = resolver
+
+ def _invalidate_metric_caches(self) -> None:
+ """Clear every cached metric (called on build + add_file)."""
+ self._community_cache = None
+ self._symbol_community_cache = None
+ self._community_info_cache = None
+ self._community_algo = ""
+ self._pagerank_cache = None
+ self._betweenness_cache = None
+ self._in_degree_cache = None
+ self._out_degree_cache = None
+ self._symbol_pagerank_cache = None
+ self._symbol_betweenness_cache = None
+ self._execution_flow_cache = None
+
+ def release_graph(self) -> None:
+ """Drop the in-memory NetworkX object after metrics are materialized.
+
+ For large-repo scale work: once file-level metrics have been loaded
+ from SQL via :meth:`load_metrics_from_sql`, callers that no longer need
+ graph traversal (e.g. the fast-mode pipeline, which generates no docs)
+ can release the structural graph to free memory. Metric reads continue
+ to be served from the loaded caches.
+
+ ``_built`` stays True so a subsequent ``graph()`` call returns the
+ empty graph rather than silently (and expensively) rebuilding the
+ structure we deliberately dropped.
+ """
+ self._graph = nx.DiGraph()
+ self._built = True
+
+ # ------------------------------------------------------------------
+ # Building
+ # ------------------------------------------------------------------
+
+ def add_file(self, parsed: ParsedFile) -> None:
+ """Register one parsed file and its symbols in the graph."""
+ path = parsed.file_info.path
+ self._parsed_files[path] = parsed
+ self._built = False # invalidate cached metrics
+
+ # --- File node ---
+ self._graph.add_node(
+ path,
+ node_type="file",
+ language=parsed.file_info.language,
+ symbol_count=len(parsed.symbols),
+ has_error=bool(parsed.parse_errors),
+ is_test=parsed.file_info.is_test,
+ is_entry_point=parsed.file_info.is_entry_point,
+ )
+
+ # --- Symbol nodes ---
+ for sym in parsed.symbols:
+ self._graph.add_node(
+ sym.id,
+ node_type="symbol",
+ kind=sym.kind,
+ name=sym.name,
+ qualified_name=sym.qualified_name,
+ file_path=path,
+ start_line=sym.start_line,
+ end_line=sym.end_line,
+ visibility=sym.visibility,
+ is_async=sym.is_async,
+ language=sym.language,
+ parent_name=sym.parent_name,
+ signature=sym.signature,
+ decorators=sym.decorators,
+ is_exported_symbol=sym.is_exported_symbol,
+ )
+
+ # DEFINES edge: file → symbol
+ self._graph.add_edge(
+ path,
+ sym.id,
+ edge_type="defines",
+ )
+
+ # HAS_METHOD edge: class/struct → method
+ if sym.parent_name and sym.kind == "method":
+ parent_id = f"{path}::{sym.parent_name}"
+ if parent_id in self._graph:
+ self._graph.add_edge(
+ parent_id,
+ sym.id,
+ edge_type="has_method",
+ )
+
+ # --- Synthetic module-level symbol for top-level calls ---
+ module_sym_id = f"{path}::__module__"
+ self._graph.add_node(
+ module_sym_id,
+ node_type="symbol",
+ kind="module",
+ name="__module__",
+ file_path=path,
+ start_line=0,
+ end_line=0,
+ visibility="private",
+ language=parsed.file_info.language,
+ )
+ self._graph.add_edge(path, module_sym_id, edge_type="defines")
+
+ def build(self, progress: Any | None = None) -> nx.DiGraph:
+ """Resolve imports and calls, add edges. Returns the finalized graph.
+
+ *progress* is an optional ``ProgressCallback`` (duck-typed). When
+ provided, sub-phase events ``graph.imports`` / ``graph.heritage`` /
+ ``graph.calls`` are emitted so the CLI can surface per-file progress
+ instead of a single opaque "0/1" bar over the whole build.
+ """
+ self._invalidate_metric_caches()
+
+ # Clear import/call edges but keep structural edges (defines, has_method)
+ edges_to_remove = [
+ (u, v)
+ for u, v, d in self._graph.edges(data=True)
+ if d.get("edge_type") not in ("defines", "has_method")
+ ]
+ self._graph.remove_edges_from(edges_to_remove)
+
+ # Build lookup tables for import resolution
+ path_set = set(self._parsed_files.keys())
+ stem_map = build_stem_map(path_set)
+
+ # Construct resolver context
+ go_modules = read_go_modules(self._repo_path)
+ ctx = ResolverContext(
+ path_set=path_set,
+ stem_map=stem_map,
+ graph=self._graph,
+ repo_path=self._repo_path,
+ tsconfig_resolver=self._tsconfig_resolver,
+ go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),
+ go_modules=go_modules,
+ has_sfc_files=any(p.endswith((".vue", ".svelte", ".astro")) for p in path_set),
+ parsed_files=self._parsed_files,
+ )
+
+ # --- Phase 1 prelude: language-specific warmups ---
+ # Some languages need an expensive one-time index built before any
+ # per-file import resolution can run. Doing it here, under a dedicated
+ # phase event, keeps the cost out of ``graph.imports`` and surfaces
+ # a meaningful progress label instead of a stuck per-file bar.
+ from ..graph_warmups import run_warmups
+
+ run_warmups(self._parsed_files, ctx, progress=progress)
+
+ # --- Phase 1: Resolve file-level imports ---
+ import_targets: dict[str, set[str]] = {} # file → set of imported files
+
+ # Per-language import-resolution timing — surfaces which language is
+ # actually dominating the import loop on multi-language repos
+ # (audit #30). Persisted to state.json for after-the-fact analysis.
+ import time as _t
+
+ lang_import_time: dict[str, float] = {}
+
+ if progress:
+ progress.on_phase_start("graph.imports", len(self._parsed_files))
+ for path, parsed in self._parsed_files.items():
+ _lang = parsed.file_info.language
+ _t0 = _t.monotonic()
+ file_imports: set[str] = set()
+ for imp in parsed.imports:
+ target = resolve_import(imp.module_path, path, parsed.file_info.language, ctx)
+ if target:
+ imp.resolved_file = target
+ file_imports.add(target)
+ # Aggregate imported_names on parallel edges
+ if self._graph.has_edge(path, target):
+ existing = self._graph[path][target].get("imported_names", [])
+ merged = list(set(existing + imp.imported_names))
+ self._graph[path][target]["imported_names"] = merged
+ else:
+ self._graph.add_edge(
+ path,
+ target,
+ edge_type="imports",
+ imported_names=list(imp.imported_names),
+ )
+ import_targets[path] = file_imports
+ lang_import_time[_lang] = lang_import_time.get(_lang, 0.0) + (_t.monotonic() - _t0)
+ if progress:
+ progress.on_item_done("graph.imports")
+ if progress:
+ _phase_done = getattr(progress, "on_phase_done", None)
+ if _phase_done is not None:
+ _phase_done("graph.imports")
+ if lang_import_time:
+ log.info(
+ "import_resolution_per_language",
+ seconds={k: round(v, 2) for k, v in lang_import_time.items()},
+ )
+
+ # --- Phase 1b: Resolve non-import type references (e.g. C# ctor params) ---
+ # Lives between import resolution and heritage so the type-use
+ # edges feed into heritage's import_targets propagation if a
+ # future language emits them, and so dead-code reachability
+ # sees every edge before any analysis pass runs.
+ if progress:
+ progress.on_phase_start("graph.type_refs", len(self._parsed_files))
+ type_use_counts = resolve_type_refs(self._parsed_files, ctx, self._graph)
+ for path in self._parsed_files:
+ for _, target, data in self._graph.out_edges(path, data=True):
+ # Treat both real ``using`` imports and synthesised
+ # type-use edges as import-like for downstream heritage
+ # / call resolution. Without this, type-use edges would
+ # only contribute to file-reachability — not to
+ # ``import_targets`` which gates cross-file call /
+ # heritage lookups.
+ if data.get("edge_type") in ("imports", "type_use"):
+ import_targets.setdefault(path, set()).add(target)
+ if progress:
+ _phase_done = getattr(progress, "on_phase_done", None)
+ if _phase_done is not None:
+ _phase_done("graph.type_refs")
+ del type_use_counts # used only for logging inside resolve_type_refs
+
+ # --- Phase 1c: Resolve C# member-access reads ---
+ # Bind `var x = new T(...); ... x.Prop` style property reads
+ # to T's defining file. Cuts the largest single bucket of C#
+ # unused_export false positives (audit #23).
+ self._resolve_member_reads(progress=progress)
+
+ # --- Phase 2: Resolve heritage (extends/implements) ---
+ self._resolve_heritage(import_targets, progress=progress)
+
+ # --- Phase 3: Resolve symbol-level calls ---
+ self._resolve_calls(import_targets, progress=progress)
+
+ self._built = True
+
+ # Count edge types for logging
+ edge_counts: dict[str, int] = {}
+ for _, _, d in self._graph.edges(data=True):
+ et = d.get("edge_type", "imports")
+ edge_counts[et] = edge_counts.get(et, 0) + 1
+
+ file_nodes = sum(
+ 1 for _, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"
+ )
+ symbol_nodes = sum(
+ 1 for _, d in self._graph.nodes(data=True) if d.get("node_type") == "symbol"
+ )
+
+ log.info(
+ "Graph built",
+ file_nodes=file_nodes,
+ symbol_nodes=symbol_nodes,
+ edges=self._graph.number_of_edges(),
+ edge_types=edge_counts,
+ )
+ return self._graph
+
+ def graph(self) -> nx.DiGraph:
+ """Return the graph (building it first if necessary)."""
+ if not self._built:
+ self.build()
+ return self._graph
diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.py
index ed8af4092..5d974bf2c 100644
--- a/packages/core/src/repowise/core/persistence/__init__.py
+++ b/packages/core/src/repowise/core/persistence/__init__.py
@@ -18,12 +18,11 @@ from repowise.core.providers.embedding.base import Embedder, MockEmbedder
from .crud import (
batch_upsert_graph_edges,
+ batch_upsert_graph_metrics,
batch_upsert_graph_nodes,
batch_upsert_symbols,
bulk_upsert_decisions,
bulk_upsert_external_systems,
- link_graph_nodes_to_external_systems,
- list_external_systems,
count_chat_messages,
create_chat_message,
create_conversation,
@@ -43,6 +42,7 @@ from .crud import (
get_git_metadata,
get_git_metadata_bulk,
get_graph_edges_for_node,
+ get_graph_metrics,
get_graph_node,
get_graph_nodes_by_ids,
get_node_degree_counts,
@@ -53,9 +53,11 @@ from .crud import (
get_stale_decisions,
get_stale_pages,
get_top_entry_points,
+ link_graph_nodes_to_external_systems,
list_chat_messages,
list_conversations,
list_decisions,
+ list_external_systems,
list_page_ids,
list_pages,
load_prior_pages,
@@ -99,6 +101,7 @@ from .models import (
GenerationJob,
GitMetadata,
GraphEdge,
+ GraphMetric,
GraphNode,
Page,
PageVersion,
@@ -132,6 +135,7 @@ __all__ = [
"GenerationJob",
"GitMetadata",
"GraphEdge",
+ "GraphMetric",
"GraphNode",
# vector store
"InMemoryVectorStore",
@@ -148,25 +152,15 @@ __all__ = [
"async_sessionmaker",
# crud
"batch_upsert_graph_edges",
+ "batch_upsert_graph_metrics",
"batch_upsert_graph_nodes",
"batch_upsert_symbols",
# decision crud
"bulk_upsert_decisions",
# external systems crud (C4 L1)
"bulk_upsert_external_systems",
- "link_graph_nodes_to_external_systems",
- "list_external_systems",
# chat crud
"count_chat_messages",
- # graph read-side queries
- "get_all_file_metrics",
- "get_community_members",
- "get_cross_community_edges",
- "get_graph_edges_for_node",
- "get_graph_node",
- "get_graph_nodes_by_ids",
- "get_node_degree_counts",
- "get_top_entry_points",
"create_chat_message",
"create_conversation",
"create_engine",
@@ -174,10 +168,14 @@ __all__ = [
"delete_conversation",
"delete_decision",
"delete_repository",
+ # graph read-side queries
+ "get_all_file_metrics",
# git metadata crud
"get_all_git_metadata",
+ "get_community_members",
"get_configured_db_url",
"get_conversation",
+ "get_cross_community_edges",
"get_db_url",
# dead code crud
"get_dead_code_findings",
@@ -187,6 +185,11 @@ __all__ = [
"get_generation_job",
"get_git_metadata",
"get_git_metadata_bulk",
+ "get_graph_edges_for_node",
+ "get_graph_metrics",
+ "get_graph_node",
+ "get_graph_nodes_by_ids",
+ "get_node_degree_counts",
"get_page",
"get_page_versions",
"get_repo_db_path",
@@ -195,10 +198,13 @@ __all__ = [
"get_session",
"get_stale_decisions",
"get_stale_pages",
+ "get_top_entry_points",
"init_db",
+ "link_graph_nodes_to_external_systems",
"list_chat_messages",
"list_conversations",
"list_decisions",
+ "list_external_systems",
"list_page_ids",
"list_pages",
"load_prior_pages",
diff --git a/packages/core/src/repowise/core/persistence/crud.py b/packages/core/src/repowise/core/persistence/crud.py
index 1f5e387c5..a43213c07 100644
--- a/packages/core/src/repowise/core/persistence/crud.py
+++ b/packages/core/src/repowise/core/persistence/crud.py
@@ -31,6 +31,7 @@ from .models import (
GenerationJob,
GitMetadata,
GraphEdge,
+ GraphMetric,
GraphNode,
HealthFileMetric,
HealthFinding,
@@ -553,6 +554,69 @@ async def batch_upsert_graph_edges(
await session.flush()
+async def batch_upsert_graph_metrics(
+ session: AsyncSession,
+ repository_id: str,
+ metrics: dict[str, dict],
+) -> None:
+ """Materialize the file-level metrics snapshot into ``graph_metrics``.
+
+ *metrics* maps ``node_id`` → a dict with ``pagerank``, ``betweenness``,
+ ``community_id``, ``in_degree``, ``out_degree``. Additive to
+ ``graph_nodes`` — this is the snapshot read back by
+ ``GraphBuilder.load_metrics_from_sql`` on large repos. SELECT-then-write
+ for dialect portability (SQLite + Postgres).
+ """
+ fields = ("pagerank", "betweenness", "community_id", "in_degree", "out_degree")
+ for node_id, m in metrics.items():
+ result = await session.execute(
+ select(GraphMetric).where(
+ GraphMetric.repository_id == repository_id,
+ GraphMetric.node_id == node_id,
+ )
+ )
+ existing = result.scalar_one_or_none()
+ if existing is not None:
+ for key in fields:
+ if key in m:
+ setattr(existing, key, m[key])
+ else:
+ session.add(
+ GraphMetric(
+ id=_new_uuid(),
+ repository_id=repository_id,
+ node_id=node_id,
+ pagerank=float(m.get("pagerank", 0.0)),
+ betweenness=float(m.get("betweenness", 0.0)),
+ community_id=int(m.get("community_id", 0)),
+ in_degree=int(m.get("in_degree", 0)),
+ out_degree=int(m.get("out_degree", 0)),
+ )
+ )
+
+ await session.flush()
+
+
+async def get_graph_metrics(
+ session: AsyncSession,
+ repository_id: str,
+) -> dict[str, dict]:
+ """Read the materialized ``graph_metrics`` snapshot as ``node_id → metrics``."""
+ result = await session.execute(
+ select(GraphMetric).where(GraphMetric.repository_id == repository_id)
+ )
+ return {
+ row.node_id: {
+ "pagerank": row.pagerank,
+ "betweenness": row.betweenness,
+ "community_id": row.community_id,
+ "in_degree": row.in_degree,
+ "out_degree": row.out_degree,
+ }
+ for row in result.scalars().all()
+ }
+
+
# ---------------------------------------------------------------------------
# ExternalSystem CRUD
# ---------------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.py
index ec96e286e..18bf0f0ca 100644
--- a/packages/core/src/repowise/core/persistence/models.py
+++ b/packages/core/src/repowise/core/persistence/models.py
@@ -100,7 +100,7 @@ class Page(Base):
page_type: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(Text, nullable=False)
content: Mapped[str] = mapped_column(Text, nullable=False)
- # 1–3 sentence purpose blurb. Always populated (LLM-extracted from content
+ # 1-3 sentence purpose blurb. Always populated (LLM-extracted from content
# for full mode, deterministic structure summary for index-only mode).
# Surfaced by get_context as the default narrative; content is gated
# behind include=["full_doc"] to keep MCP responses small.
@@ -243,6 +243,34 @@ class GraphEdge(Base):
)
+class GraphMetric(Base):
+ """Materialized file-level graph metrics snapshot (large-repo scale).
+
+ Lets metric reads be served from SQL without recomputing the expensive
+ NetworkX centrality kernels on big graphs. Written after the graph is
+ built (additive to ``graph_nodes``) and read back into a GraphBuilder via
+ ``load_metrics_from_sql``.
+ """
+
+ __tablename__ = "graph_metrics"
+
+ id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_new_uuid)
+ repository_id: Mapped[str] = mapped_column(
+ String(32), ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True
+ )
+ node_id: Mapped[str] = mapped_column(Text, nullable=False)
+ pagerank: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+ betweenness: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+ community_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ in_degree: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ out_degree: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, default=_now_utc
+ )
+
+ __table_args__ = (UniqueConstraint("repository_id", "node_id", name="uq_graph_metric"),)
+
+
class WebhookEvent(Base):
__tablename__ = "webhook_events"
diff --git a/packages/core/src/repowise/core/persistence/vector_store.py b/packages/core/src/repowise/core/persistence/vector_store.py
deleted file mode 100644
index 6d61559eb..000000000
--- a/packages/core/src/repowise/core/persistence/vector_store.py
+++ /dev/null
@@ -1,536 +0,0 @@
-"""Vector store abstraction and implementations for repowise semantic search.
-
-Three implementations are provided:
-
-InMemoryVectorStore
- Pure Python, no external dependencies. Cosine similarity search over
- an in-memory dict. Suitable for tests and development.
-
-LanceDBVectorStore
- Embedded vector database stored in a local directory. Requires the
- ``repowise-core[search]`` extra (lancedb>=0.12).
-
-PgVectorStore
- Stores embeddings in the ``wiki_pages.embedding`` pgvector column.
- Requires the ``repowise-core[pgvector]`` extra and a running PostgreSQL
- with the ``vector`` extension enabled.
-
-All stores accept an :class:`Embedder` at construction time and handle
-embedding internally — callers pass raw text, not pre-computed vectors.
-"""
-
-from __future__ import annotations
-
-import math
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
-
-from repowise.core.providers.embedding.base import Embedder
-
-from .search import SearchResult
-
-if TYPE_CHECKING:
- from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-
-__all__ = [
- "InMemoryVectorStore",
- "LanceDBVectorStore",
- "PgVectorStore",
- "VectorStore",
-]
-
-
-# ---------------------------------------------------------------------------
-# Abstract base
-# ---------------------------------------------------------------------------
-
-
-class VectorStore(ABC):
- """Abstract vector store. All methods are async."""
-
- @abstractmethod
- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
- """Embed *text* and upsert the vector under *page_id*."""
- ...
-
- @abstractmethod
- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
- """Embed *query* and return the *limit* nearest pages."""
- ...
-
- @abstractmethod
- async def delete(self, page_id: str) -> None:
- """Remove the vector for *page_id* from the store."""
- ...
-
- @abstractmethod
- async def close(self) -> None:
- """Release any resources held by the store."""
- ...
-
- async def list_page_ids(self) -> set[str]:
- """Return the set of page IDs currently stored.
-
- Used by ``repowise doctor --repair`` to detect three-store
- inconsistencies. Implementations may override for efficiency.
- """
- return set() # default: empty (subclasses should override)
-
- async def get_page_summary_by_path(self, path: str) -> dict | None:
- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
-
- Used for RAG context injection during doc generation: when generating page B
- that imports A, we fetch A's previously-generated summary and feed it to the LLM.
- """
- return None # default: no-op (subclasses should override)
-
- async def get_page_summaries_by_paths(
- self, paths: list[str]
- ) -> dict[str, dict]:
- """Batch variant of :meth:`get_page_summary_by_path`.
-
- Returns a mapping of resolved paths → summary dict for every
- input path that produced a non-None result. The default
- implementation fires all per-path calls concurrently via
- ``asyncio.gather`` so callers don't have to await each one
- sequentially — backends that can do a single SQL/index scan
- should override this for the obvious efficiency gain.
- """
- import asyncio as _asyncio
-
- if not paths:
- return {}
- coros = [self.get_page_summary_by_path(p) for p in paths]
- results = await _asyncio.gather(*coros, return_exceptions=True)
- out: dict[str, dict] = {}
- for path, result in zip(paths, results, strict=False):
- if isinstance(result, dict) and result.get("summary"):
- out[path] = result
- return out
-
-
-# ---------------------------------------------------------------------------
-# InMemoryVectorStore
-# ---------------------------------------------------------------------------
-
-
-def _cosine(a: list[float], b: list[float]) -> float:
- """Cosine similarity between two vectors (returns 0.0 for zero vectors)."""
- dot = sum(x * y for x, y in zip(a, b, strict=False))
- norm_a = math.sqrt(sum(x * x for x in a))
- norm_b = math.sqrt(sum(x * x for x in b))
- denom = norm_a * norm_b
- return dot / denom if denom > 0 else 0.0
-
-
-class InMemoryVectorStore(VectorStore):
- """Cosine-similarity vector store backed by a plain Python dict.
-
- Suitable for unit tests and small-scale development use.
- No external dependencies beyond the Embedder.
- """
-
- def __init__(self, embedder: Embedder) -> None:
- self._embedder = embedder
- # page_id → (vector, metadata)
- self._store: dict[str, tuple[list[float], dict]] = {}
-
- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
- vectors = await self._embedder.embed([text])
- self._store[page_id] = (vectors[0], dict(metadata))
-
- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
- if not self._store:
- return []
- q_vecs = await self._embedder.embed([query])
- q_vec = q_vecs[0]
-
- scored: list[tuple[float, str, dict]] = []
- for pid, (vec, meta) in self._store.items():
- score = _cosine(q_vec, vec)
- scored.append((score, pid, meta))
-
- scored.sort(key=lambda t: t[0], reverse=True)
-
- results = []
- for score, pid, meta in scored[:limit]:
- content = meta.get("content", "")
- results.append(
- SearchResult(
- page_id=pid,
- title=str(meta.get("title", "")),
- page_type=str(meta.get("page_type", "")),
- target_path=str(meta.get("target_path", "")),
- score=score,
- snippet=str(content)[:200].rstrip(),
- search_type="vector",
- )
- )
- return results
-
- async def delete(self, page_id: str) -> None:
- self._store.pop(page_id, None)
-
- async def close(self) -> None:
- self._store.clear()
-
- async def list_page_ids(self) -> set[str]:
- return set(self._store.keys())
-
- async def get_page_summary_by_path(self, path: str) -> dict | None:
- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
-
- Used for RAG context injection during doc generation: when generating page B
- that imports A, we fetch A's previously-generated summary and feed it to the LLM.
-
- Implementation note: reads 'summary' from metadata if present (set by the
- generation pipeline), else falls back to the first 500 chars of 'content'.
- 'key_exports' reads the 'exports' metadata field if present, else [].
- """
- for _pid, (_, meta) in self._store.items():
- if meta.get("target_path") == path:
- summary = meta.get("summary") or str(meta.get("content", ""))[:500]
- key_exports = meta.get("exports") or []
- return {"summary": summary, "key_exports": list(key_exports)}
- return None
-
- async def get_page_summaries_by_paths(
- self, paths: list[str]
- ) -> dict[str, dict]:
- """Single-pass scan over the store — avoids N full scans when
- many paths are queried at once.
- """
- if not paths:
- return {}
- wanted = set(paths)
- out: dict[str, dict] = {}
- for _pid, (_, meta) in self._store.items():
- tp = meta.get("target_path")
- if tp in wanted and tp not in out:
- summary = meta.get("summary") or str(meta.get("content", ""))[:500]
- out[str(tp)] = {
- "summary": summary,
- "key_exports": list(meta.get("exports") or []),
- }
- if len(out) == len(wanted):
- break
- return out
-
- def __len__(self) -> int:
- return len(self._store)
-
-
-# ---------------------------------------------------------------------------
-# LanceDBVectorStore
-# ---------------------------------------------------------------------------
-
-
-class LanceDBVectorStore(VectorStore):
- """Vector store backed by LanceDB (embedded, local file storage).
-
- Requires the ``repowise-core[search]`` extra:
- pip install repowise-core[search]
-
- Data is stored in *db_path* (e.g. ``.repowise/lancedb/``).
- The LanceDB table is created lazily on the first call to
- :meth:`embed_and_upsert`.
- """
-
- _TABLE_NAME = "wiki_pages"
-
- def __init__(self, db_path: str, embedder: Embedder, table_name: str | None = None) -> None:
- self._db_path = db_path
- self._embedder = embedder
- self._table_name = table_name or self._TABLE_NAME
- self._db = None
- self._table = None
-
- async def _ensure_connected(self) -> None:
- if self._db is not None:
- return
- try:
- import lancedb # type: ignore[import]
- except ImportError as exc:
- raise RuntimeError(
- "LanceDB is not installed. Install it with: pip install repowise-core[search]"
- ) from exc
-
- self._db = await lancedb.connect_async(self._db_path)
- table_names = await self._db.table_names()
- if self._table_name in table_names:
- self._table = await self._db.open_table(self._table_name)
- else:
- self._table = None # will be created on first upsert
-
- async def _ensure_table(self, sample_vector: list[float]) -> None:
- """Create the LanceDB table if it does not exist yet."""
- if self._table is not None:
- return
-
- try:
- import pyarrow as pa # type: ignore[import]
- except ImportError as exc:
- raise RuntimeError(
- "pyarrow is required for LanceDBVectorStore. "
- "It is installed automatically with lancedb."
- ) from exc
-
- dim = len(sample_vector)
- schema = pa.schema(
- [
- pa.field("page_id", pa.string()),
- pa.field("vector", pa.list_(pa.float32(), dim)),
- pa.field("title", pa.string()),
- pa.field("page_type", pa.string()),
- pa.field("target_path", pa.string()),
- pa.field("content_snippet", pa.string()),
- ]
- )
- self._table = await self._db.create_table( # type: ignore[union-attr]
- self._table_name, schema=schema, exist_ok=True
- )
-
- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
- await self._ensure_connected()
- vectors = await self._embedder.embed([text])
- vector = vectors[0]
- await self._ensure_table(vector)
-
- content = str(metadata.get("content", text))
- row = {
- "page_id": page_id,
- "vector": [float(v) for v in vector],
- "title": str(metadata.get("title", "")),
- "page_type": str(metadata.get("page_type", "")),
- "target_path": str(metadata.get("target_path", "")),
- "content_snippet": content[:200],
- }
-
- # merge_insert: upsert by page_id (LanceDB 0.12+)
- try:
- await (
- self._table.merge_insert("page_id")
- .when_matched_update_all()
- .when_not_matched_insert_all()
- .execute([row])
- ) # type: ignore[union-attr]
- except AttributeError:
- # Fallback for older LanceDB versions: delete + add
- safe_id = page_id.replace("'", "''")
- await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]
- await self._table.add([row]) # type: ignore[union-attr]
-
- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
- await self._ensure_connected()
- if self._table is None:
- return []
-
- q_vecs = await self._embedder.embed([query])
- q_vec = [float(v) for v in q_vecs[0]]
-
- raw = (
- await self._table.query() # type: ignore[union-attr]
- .nearest_to(q_vec)
- .limit(limit)
- .to_list()
- )
-
- return [
- SearchResult(
- page_id=r["page_id"],
- title=r.get("title", ""),
- page_type=r.get("page_type", ""),
- target_path=r.get("target_path", ""),
- score=float(r.get("_distance", 0.0)),
- snippet=r.get("content_snippet", ""),
- search_type="vector",
- )
- for r in raw
- ]
-
- async def delete(self, page_id: str) -> None:
- await self._ensure_connected()
- if self._table is not None:
- safe_id = page_id.replace("'", "''")
- await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]
-
- async def close(self) -> None:
- self._table = None
- self._db = None
-
- async def list_page_ids(self) -> set[str]:
- await self._ensure_connected()
- if self._table is None:
- return set()
- rows = await self._table.query().select(["page_id"]).to_list() # type: ignore[union-attr]
- return {r["page_id"] for r in rows}
-
- async def get_page_summary_by_path(self, path: str) -> dict | None:
- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
-
- Used for RAG context injection during doc generation: when generating page B
- that imports A, we fetch A's previously-generated summary and feed it to the LLM.
-
- Implementation note: LanceDB stores up to 200 chars of content in 'content_snippet'.
- We use that as the summary. 'key_exports' is not stored in the LanceDB schema, so
- we return [] — the caller only uses the text summary for prompt injection.
- """
- await self._ensure_connected()
- if self._table is None:
- return None
-
- safe_path = path.replace("'", "''")
- try:
- rows = (
- await self._table.query() # type: ignore[union-attr]
- .where(f"target_path = '{safe_path}'")
- .select(["content_snippet"])
- .limit(1)
- .to_list()
- )
- except Exception:
- return None
-
- if not rows:
- return None
-
- summary = rows[0].get("content_snippet") or ""
- return {"summary": str(summary), "key_exports": []}
-
-
-# ---------------------------------------------------------------------------
-# PgVectorStore
-# ---------------------------------------------------------------------------
-
-
-class PgVectorStore(VectorStore):
- """Vector store that writes embeddings to the ``wiki_pages.embedding`` column.
-
- Requires:
- - PostgreSQL with the ``vector`` extension.
- - The Alembic migration ``0001_initial_schema`` has been applied.
- - The ``repowise-core[pgvector]`` extra.
-
- Uses raw SQL to avoid importing ``pgvector.sqlalchemy.Vector`` at module
- level (keeps the base package installable without the extra).
- """
-
- def __init__(
- self,
- session_factory: async_sessionmaker[AsyncSession],
- embedder: Embedder,
- ) -> None:
- self._session_factory = session_factory
- self._embedder = embedder
-
- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
- vectors = await self._embedder.embed([text])
- vector = vectors[0]
- # pgvector expects a list encoded as a string like "[0.1, 0.2, ...]"
- vec_str = "[" + ",".join(str(v) for v in vector) + "]"
-
- from sqlalchemy.sql import text as sa_text
-
- async with self._session_factory() as session:
- await session.execute(
- sa_text("UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"),
- {"emb": vec_str, "pid": page_id},
- )
- await session.commit()
-
- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
- q_vecs = await self._embedder.embed([query])
- q_vec = q_vecs[0]
- vec_str = "[" + ",".join(str(v) for v in q_vec) + "]"
-
- from sqlalchemy.sql import text as sa_text
-
- async with self._session_factory() as session:
- rows = await session.execute(
- sa_text(
- "SELECT id, title, content, page_type, target_path, "
- " 1 - (embedding <=> CAST(:q AS vector)) AS score "
- "FROM wiki_pages "
- "WHERE embedding IS NOT NULL "
- "ORDER BY embedding <=> CAST(:q AS vector) "
- "LIMIT :lim"
- ),
- {"q": vec_str, "lim": limit},
- )
- raw = rows.fetchall()
-
- return [
- SearchResult(
- page_id=r[0],
- title=r[1],
- page_type=r[3],
- target_path=r[4],
- score=float(r[5]),
- snippet=str(r[2])[:200].rstrip(),
- search_type="vector",
- )
- for r in raw
- ]
-
- async def delete(self, page_id: str) -> None:
- from sqlalchemy.sql import text as sa_text
-
- async with self._session_factory() as session:
- await session.execute(
- sa_text("UPDATE wiki_pages SET embedding = NULL WHERE id = :pid"),
- {"pid": page_id},
- )
- await session.commit()
-
- async def close(self) -> None:
- pass # session_factory manages connection lifecycle
-
- async def list_page_ids(self) -> set[str]:
- from sqlalchemy.sql import text as sa_text
-
- async with self._session_factory() as session:
- rows = await session.execute(
- sa_text("SELECT id FROM wiki_pages WHERE embedding IS NOT NULL")
- )
- return {r[0] for r in rows.fetchall()}
-
- async def get_page_summary_by_path(self, path: str) -> dict | None:
- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
-
- Used for RAG context injection during doc generation: when generating page B
- that imports A, we fetch A's previously-generated summary and feed it to the LLM.
-
- Implementation note: reads the 'content' column (first 500 chars) from the
- wiki_pages table matched by target_path. 'key_exports' is derived from the
- page's exports if stored in a metadata JSON column; otherwise returns [].
- """
- from sqlalchemy.sql import text as sa_text
-
- async with self._session_factory() as session:
- rows = await session.execute(
- sa_text(
- "SELECT content, metadata FROM wiki_pages "
- "WHERE target_path = :path "
- "LIMIT 1"
- ),
- {"path": path},
- )
- row = rows.fetchone()
-
- if row is None:
- return None
-
- content = str(row[0] or "")[:500]
- # Extract key_exports from metadata JSON column if present
- key_exports: list[str] = []
- if row[1] and isinstance(row[1], dict):
- key_exports = list(row[1].get("exports", []))
- elif row[1] and isinstance(row[1], str):
- import json
- try:
- meta = json.loads(row[1])
- key_exports = list(meta.get("exports", []))
- except (json.JSONDecodeError, AttributeError):
- pass
-
- return {"summary": content, "key_exports": key_exports}
diff --git a/packages/core/src/repowise/core/persistence/vector_store/README.md b/packages/core/src/repowise/core/persistence/vector_store/README.md
new file mode 100644
index 000000000..a9ac9d49a
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/vector_store/README.md
@@ -0,0 +1,52 @@
+# `vector_store`
+
+Pluggable semantic-search backends for repowise wiki pages.
+
+## Purpose
+
+Embed generated wiki pages into vectors and answer nearest-neighbour
+queries. Callers pass raw text; each store owns an `Embedder` and handles
+embedding internally.
+
+## Public API
+
+Imported as before via `repowise.core.persistence.vector_store`:
+
+- `VectorStore` — abstract base class.
+- `InMemoryVectorStore` — pure-Python dict, no extra deps (tests / dev).
+- `LanceDBVectorStore` — embedded LanceDB (`repowise-core[search]` extra).
+- `PgVectorStore` — `wiki_pages.embedding` pgvector column (`[pgvector]` extra).
+- `cosine_similarity(a, b)` — shared helper.
+
+### Methods
+
+- `embed_and_upsert(page_id, text, metadata)` — single-item write.
+- `embed_batch(items: list[tuple[id, text, metadata]])` — embed a whole batch
+ in one model call. The ABC provides a correct sequential default; each
+ concrete store overrides it to make a single `embed()` call plus a bulk
+ upsert. Use this on the hot indexing path; keep `embed_and_upsert` for
+ incremental single-page updates.
+- `search`, `delete`, `close`, `list_page_ids`,
+ `get_page_summary_by_path`, `get_page_summaries_by_paths`.
+
+## Internal layout
+
+| Module | Contents |
+|--------|----------|
+| `_base.py` | `VectorStore` ABC + `cosine_similarity` |
+| `in_memory.py` | `InMemoryVectorStore` |
+| `lancedb_store.py` | `LanceDBVectorStore` |
+| `pgvector_store.py` | `PgVectorStore` |
+
+The package `__init__` re-exports every public name so the historical
+`from repowise.core.persistence.vector_store import ...` path is unchanged.
+
+## Extension points
+
+Author a new backend by subclassing `VectorStore` and implementing the four
+abstract methods. Override `embed_batch` if your backend can amortise batched
+writes; otherwise the default sequential implementation is used.
+
+## Tests
+
+`tests/unit/persistence/test_vector_store.py`.
diff --git a/packages/core/src/repowise/core/persistence/vector_store/__init__.py b/packages/core/src/repowise/core/persistence/vector_store/__init__.py
new file mode 100644
index 000000000..84716d669
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/vector_store/__init__.py
@@ -0,0 +1,40 @@
+"""Vector store abstraction and implementations for repowise semantic search.
+
+Three implementations are provided:
+
+InMemoryVectorStore
+ Pure Python, no external dependencies. Cosine similarity search over
+ an in-memory dict. Suitable for tests and development.
+
+LanceDBVectorStore
+ Embedded vector database stored in a local directory. Requires the
+ ``repowise-core[search]`` extra (lancedb>=0.12).
+
+PgVectorStore
+ Stores embeddings in the ``wiki_pages.embedding`` pgvector column.
+ Requires the ``repowise-core[pgvector]`` extra and a running PostgreSQL
+ with the ``vector`` extension enabled.
+
+All stores accept an :class:`Embedder` at construction time and handle
+embedding internally — callers pass raw text, not pre-computed vectors.
+
+Every store implements :meth:`VectorStore.embed_batch`, which embeds a whole
+list of pages in a single model call. Large-repo indexing uses the batch path
+to avoid one embedding round-trip per page; the single-item
+:meth:`VectorStore.embed_and_upsert` path is preserved for incremental updates.
+"""
+
+from __future__ import annotations
+
+from ._base import VectorStore, cosine_similarity
+from .in_memory import InMemoryVectorStore
+from .lancedb_store import LanceDBVectorStore
+from .pgvector_store import PgVectorStore
+
+__all__ = [
+ "InMemoryVectorStore",
+ "LanceDBVectorStore",
+ "PgVectorStore",
+ "VectorStore",
+ "cosine_similarity",
+]
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.py
new file mode 100644
index 000000000..95507eb2c
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py
@@ -0,0 +1,101 @@
+"""Vector store abstract base class and shared helpers.
+
+The concrete implementations live in sibling modules
+(:mod:`in_memory`, :mod:`lancedb_store`, :mod:`pgvector_store`) and are
+re-exported from the package ``__init__`` so the historical import path
+``repowise.core.persistence.vector_store`` keeps working unchanged.
+"""
+
+from __future__ import annotations
+
+import math
+from abc import ABC, abstractmethod
+
+from ..search import SearchResult
+
+__all__ = ["VectorStore", "cosine_similarity"]
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+ """Cosine similarity between two vectors (returns 0.0 for zero vectors)."""
+ dot = sum(x * y for x, y in zip(a, b, strict=False))
+ norm_a = math.sqrt(sum(x * x for x in a))
+ norm_b = math.sqrt(sum(x * x for x in b))
+ denom = norm_a * norm_b
+ return dot / denom if denom > 0 else 0.0
+
+
+class VectorStore(ABC):
+ """Abstract vector store. All methods are async."""
+
+ @abstractmethod
+ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
+ """Embed *text* and upsert the vector under *page_id*."""
+ ...
+
+ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
+ """Embed and upsert many ``(page_id, text, metadata)`` items at once.
+
+ The default implementation processes items sequentially via
+ :meth:`embed_and_upsert`, so any backend gets correct behaviour for
+ free. Backends that can embed a whole batch in a single model call
+ (the common case) override this to amortise the network / GPU
+ round-trip — see the bundled stores. Callers may always use this
+ path; it never has worse semantics than calling
+ :meth:`embed_and_upsert` in a loop.
+ """
+ for page_id, text, metadata in items:
+ await self.embed_and_upsert(page_id, text, metadata)
+
+ @abstractmethod
+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+ """Embed *query* and return the *limit* nearest pages."""
+ ...
+
+ @abstractmethod
+ async def delete(self, page_id: str) -> None:
+ """Remove the vector for *page_id* from the store."""
+ ...
+
+ @abstractmethod
+ async def close(self) -> None:
+ """Release any resources held by the store."""
+ ...
+
+ async def list_page_ids(self) -> set[str]:
+ """Return the set of page IDs currently stored.
+
+ Used by ``repowise doctor --repair`` to detect three-store
+ inconsistencies. Implementations may override for efficiency.
+ """
+ return set() # default: empty (subclasses should override)
+
+ async def get_page_summary_by_path(self, path: str) -> dict | None:
+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
+
+ Used for RAG context injection during doc generation: when generating page B
+ that imports A, we fetch A's previously-generated summary and feed it to the LLM.
+ """
+ return None # default: no-op (subclasses should override)
+
+ async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:
+ """Batch variant of :meth:`get_page_summary_by_path`.
+
+ Returns a mapping of resolved paths → summary dict for every
+ input path that produced a non-None result. The default
+ implementation fires all per-path calls concurrently via
+ ``asyncio.gather`` so callers don't have to await each one
+ sequentially — backends that can do a single SQL/index scan
+ should override this for the obvious efficiency gain.
+ """
+ import asyncio as _asyncio
+
+ if not paths:
+ return {}
+ coros = [self.get_page_summary_by_path(p) for p in paths]
+ results = await _asyncio.gather(*coros, return_exceptions=True)
+ out: dict[str, dict] = {}
+ for path, result in zip(paths, results, strict=False):
+ if isinstance(result, dict) and result.get("summary"):
+ out[path] = result
+ return out
diff --git a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
new file mode 100644
index 000000000..b97e04633
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
@@ -0,0 +1,110 @@
+"""Pure-Python in-memory vector store (tests + small-scale dev)."""
+
+from __future__ import annotations
+
+from repowise.core.providers.embedding.base import Embedder
+
+from ..search import SearchResult
+from ._base import VectorStore, cosine_similarity
+
+__all__ = ["InMemoryVectorStore"]
+
+
+class InMemoryVectorStore(VectorStore):
+ """Cosine-similarity vector store backed by a plain Python dict.
+
+ Suitable for unit tests and small-scale development use.
+ No external dependencies beyond the Embedder.
+ """
+
+ def __init__(self, embedder: Embedder) -> None:
+ self._embedder = embedder
+ # page_id → (vector, metadata)
+ self._store: dict[str, tuple[list[float], dict]] = {}
+
+ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
+ vectors = await self._embedder.embed([text])
+ self._store[page_id] = (vectors[0], dict(metadata))
+
+ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
+ if not items:
+ return
+ texts = [text for _, text, _ in items]
+ vectors = await self._embedder.embed(texts)
+ for (page_id, _text, metadata), vector in zip(items, vectors, strict=True):
+ self._store[page_id] = (vector, dict(metadata))
+
+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+ if not self._store:
+ return []
+ q_vecs = await self._embedder.embed([query])
+ q_vec = q_vecs[0]
+
+ scored: list[tuple[float, str, dict]] = []
+ for pid, (vec, meta) in self._store.items():
+ score = cosine_similarity(q_vec, vec)
+ scored.append((score, pid, meta))
+
+ scored.sort(key=lambda t: t[0], reverse=True)
+
+ results = []
+ for score, pid, meta in scored[:limit]:
+ content = meta.get("content", "")
+ results.append(
+ SearchResult(
+ page_id=pid,
+ title=str(meta.get("title", "")),
+ page_type=str(meta.get("page_type", "")),
+ target_path=str(meta.get("target_path", "")),
+ score=score,
+ snippet=str(content)[:200].rstrip(),
+ search_type="vector",
+ )
+ )
+ return results
+
+ async def delete(self, page_id: str) -> None:
+ self._store.pop(page_id, None)
+
+ async def close(self) -> None:
+ self._store.clear()
+
+ async def list_page_ids(self) -> set[str]:
+ return set(self._store.keys())
+
+ async def get_page_summary_by_path(self, path: str) -> dict | None:
+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
+
+ Reads 'summary' from metadata if present (set by the generation
+ pipeline), else falls back to the first 500 chars of 'content'.
+ 'key_exports' reads the 'exports' metadata field if present, else [].
+ """
+ for _pid, (_, meta) in self._store.items():
+ if meta.get("target_path") == path:
+ summary = meta.get("summary") or str(meta.get("content", ""))[:500]
+ key_exports = meta.get("exports") or []
+ return {"summary": summary, "key_exports": list(key_exports)}
+ return None
+
+ async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:
+ """Single-pass scan over the store — avoids N full scans when
+ many paths are queried at once.
+ """
+ if not paths:
+ return {}
+ wanted = set(paths)
+ out: dict[str, dict] = {}
+ for _pid, (_, meta) in self._store.items():
+ tp = meta.get("target_path")
+ if tp in wanted and tp not in out:
+ summary = meta.get("summary") or str(meta.get("content", ""))[:500]
+ out[str(tp)] = {
+ "summary": summary,
+ "key_exports": list(meta.get("exports") or []),
+ }
+ if len(out) == len(wanted):
+ break
+ return out
+
+ def __len__(self) -> int:
+ return len(self._store)
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
new file mode 100644
index 000000000..3d131088f
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
@@ -0,0 +1,201 @@
+"""LanceDB-backed vector store (embedded, local file storage)."""
+
+from __future__ import annotations
+
+from repowise.core.providers.embedding.base import Embedder
+
+from ..search import SearchResult
+from ._base import VectorStore
+
+__all__ = ["LanceDBVectorStore"]
+
+
+class LanceDBVectorStore(VectorStore):
+ """Vector store backed by LanceDB (embedded, local file storage).
+
+ Requires the ``repowise-core[search]`` extra:
+ pip install repowise-core[search]
+
+ Data is stored in *db_path* (e.g. ``.repowise/lancedb/``).
+ The LanceDB table is created lazily on the first call to
+ :meth:`embed_and_upsert`.
+ """
+
+ _TABLE_NAME = "wiki_pages"
+
+ def __init__(
+ self, db_path: str, embedder: Embedder, table_name: str | None = None
+ ) -> None:
+ self._db_path = db_path
+ self._embedder = embedder
+ self._table_name = table_name or self._TABLE_NAME
+ self._db = None
+ self._table = None
+
+ async def _ensure_connected(self) -> None:
+ if self._db is not None:
+ return
+ try:
+ import lancedb # type: ignore[import]
+ except ImportError as exc:
+ raise RuntimeError(
+ "LanceDB is not installed. Install it with: pip install repowise-core[search]"
+ ) from exc
+
+ self._db = await lancedb.connect_async(self._db_path)
+ table_names = await self._db.table_names()
+ if self._table_name in table_names:
+ self._table = await self._db.open_table(self._table_name)
+ else:
+ self._table = None # will be created on first upsert
+
+ async def _ensure_table(self, sample_vector: list[float]) -> None:
+ """Create the LanceDB table if it does not exist yet."""
+ if self._table is not None:
+ return
+
+ try:
+ import pyarrow as pa # type: ignore[import]
+ except ImportError as exc:
+ raise RuntimeError(
+ "pyarrow is required for LanceDBVectorStore. "
+ "It is installed automatically with lancedb."
+ ) from exc
+
+ dim = len(sample_vector)
+ schema = pa.schema(
+ [
+ pa.field("page_id", pa.string()),
+ pa.field("vector", pa.list_(pa.float32(), dim)),
+ pa.field("title", pa.string()),
+ pa.field("page_type", pa.string()),
+ pa.field("target_path", pa.string()),
+ pa.field("content_snippet", pa.string()),
+ ]
+ )
+ self._table = await self._db.create_table( # type: ignore[union-attr]
+ self._table_name, schema=schema, exist_ok=True
+ )
+
+ @staticmethod
+ def _row(page_id: str, vector: list[float], metadata: dict) -> dict:
+ content = str(metadata.get("content", ""))
+ return {
+ "page_id": page_id,
+ "vector": [float(v) for v in vector],
+ "title": str(metadata.get("title", "")),
+ "page_type": str(metadata.get("page_type", "")),
+ "target_path": str(metadata.get("target_path", "")),
+ "content_snippet": content[:200],
+ }
+
+ async def _upsert_rows(self, rows: list[dict]) -> None:
+ # merge_insert: upsert by page_id (LanceDB 0.12+)
+ try:
+ await (
+ self._table.merge_insert("page_id") # type: ignore[union-attr]
+ .when_matched_update_all()
+ .when_not_matched_insert_all()
+ .execute(rows)
+ )
+ except AttributeError:
+ # Fallback for older LanceDB versions: delete + add
+ for row in rows:
+ safe_id = str(row["page_id"]).replace("'", "''")
+ await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]
+ await self._table.add(rows) # type: ignore[union-attr]
+
+ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
+ await self._ensure_connected()
+ vectors = await self._embedder.embed([text])
+ vector = vectors[0]
+ await self._ensure_table(vector)
+ meta = {"content": text, **metadata}
+ await self._upsert_rows([self._row(page_id, vector, meta)])
+
+ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
+ if not items:
+ return
+ await self._ensure_connected()
+ texts = [text for _, text, _ in items]
+ vectors = await self._embedder.embed(texts)
+ await self._ensure_table(vectors[0])
+ rows = [
+ self._row(page_id, vector, {"content": text, **metadata})
+ for (page_id, text, metadata), vector in zip(items, vectors, strict=True)
+ ]
+ await self._upsert_rows(rows)
+
+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+ await self._ensure_connected()
+ if self._table is None:
+ return []
+
+ q_vecs = await self._embedder.embed([query])
+ q_vec = [float(v) for v in q_vecs[0]]
+
+ raw = (
+ await self._table.query() # type: ignore[union-attr]
+ .nearest_to(q_vec)
+ .limit(limit)
+ .to_list()
+ )
+
+ return [
+ SearchResult(
+ page_id=r["page_id"],
+ title=r.get("title", ""),
+ page_type=r.get("page_type", ""),
+ target_path=r.get("target_path", ""),
+ score=float(r.get("_distance", 0.0)),
+ snippet=r.get("content_snippet", ""),
+ search_type="vector",
+ )
+ for r in raw
+ ]
+
+ async def delete(self, page_id: str) -> None:
+ await self._ensure_connected()
+ if self._table is not None:
+ safe_id = page_id.replace("'", "''")
+ await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]
+
+ async def close(self) -> None:
+ self._table = None
+ self._db = None
+
+ async def list_page_ids(self) -> set[str]:
+ await self._ensure_connected()
+ if self._table is None:
+ return set()
+ rows = await self._table.query().select(["page_id"]).to_list() # type: ignore[union-attr]
+ return {r["page_id"] for r in rows}
+
+ async def get_page_summary_by_path(self, path: str) -> dict | None:
+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
+
+ LanceDB stores up to 200 chars of content in 'content_snippet'; we use
+ that as the summary. 'key_exports' is not stored in the schema, so we
+ return [] — the caller only uses the text summary for prompt injection.
+ """
+ await self._ensure_connected()
+ if self._table is None:
+ return None
+
+ safe_path = path.replace("'", "''")
+ try:
+ rows = (
+ await self._table.query() # type: ignore[union-attr]
+ .where(f"target_path = '{safe_path}'")
+ .select(["content_snippet"])
+ .limit(1)
+ .to_list()
+ )
+ except Exception:
+ return None
+
+ if not rows:
+ return None
+
+ summary = rows[0].get("content_snippet") or ""
+ return {"summary": str(summary), "key_exports": []}
diff --git a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
new file mode 100644
index 000000000..4a5f7f40e
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
@@ -0,0 +1,169 @@
+"""PostgreSQL/pgvector-backed vector store."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from repowise.core.providers.embedding.base import Embedder
+
+from ..search import SearchResult
+from ._base import VectorStore
+
+if TYPE_CHECKING:
+ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+__all__ = ["PgVectorStore"]
+
+
+def _encode(vector: list[float]) -> str:
+ """Encode a vector as the pgvector literal ``"[0.1,0.2,...]"``."""
+ return "[" + ",".join(str(v) for v in vector) + "]"
+
+
+class PgVectorStore(VectorStore):
+ """Vector store that writes embeddings to the ``wiki_pages.embedding`` column.
+
+ Requires:
+ - PostgreSQL with the ``vector`` extension.
+ - The Alembic migration ``0001_initial_schema`` has been applied.
+ - The ``repowise-core[pgvector]`` extra.
+
+ Uses raw SQL to avoid importing ``pgvector.sqlalchemy.Vector`` at module
+ level (keeps the base package installable without the extra).
+ """
+
+ def __init__(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ embedder: Embedder,
+ ) -> None:
+ self._session_factory = session_factory
+ self._embedder = embedder
+
+ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:
+ vectors = await self._embedder.embed([text])
+ vec_str = _encode(vectors[0])
+
+ from sqlalchemy.sql import text as sa_text
+
+ async with self._session_factory() as session:
+ await session.execute(
+ sa_text(
+ "UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"
+ ),
+ {"emb": vec_str, "pid": page_id},
+ )
+ await session.commit()
+
+ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
+ if not items:
+ return
+ texts = [text for _, text, _ in items]
+ vectors = await self._embedder.embed(texts)
+ params = [
+ {"emb": _encode(vector), "pid": page_id}
+ for (page_id, _text, _meta), vector in zip(items, vectors, strict=True)
+ ]
+
+ from sqlalchemy.sql import text as sa_text
+
+ stmt = sa_text(
+ "UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"
+ )
+ async with self._session_factory() as session:
+ for p in params:
+ await session.execute(stmt, p)
+ await session.commit()
+
+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+ q_vecs = await self._embedder.embed([query])
+ vec_str = _encode(q_vecs[0])
+
+ from sqlalchemy.sql import text as sa_text
+
+ async with self._session_factory() as session:
+ rows = await session.execute(
+ sa_text(
+ "SELECT id, title, content, page_type, target_path, "
+ " 1 - (embedding <=> CAST(:q AS vector)) AS score "
+ "FROM wiki_pages "
+ "WHERE embedding IS NOT NULL "
+ "ORDER BY embedding <=> CAST(:q AS vector) "
+ "LIMIT :lim"
+ ),
+ {"q": vec_str, "lim": limit},
+ )
+ raw = rows.fetchall()
+
+ return [
+ SearchResult(
+ page_id=r[0],
+ title=r[1],
+ page_type=r[3],
+ target_path=r[4],
+ score=float(r[5]),
+ snippet=str(r[2])[:200].rstrip(),
+ search_type="vector",
+ )
+ for r in raw
+ ]
+
+ async def delete(self, page_id: str) -> None:
+ from sqlalchemy.sql import text as sa_text
+
+ async with self._session_factory() as session:
+ await session.execute(
+ sa_text("UPDATE wiki_pages SET embedding = NULL WHERE id = :pid"),
+ {"pid": page_id},
+ )
+ await session.commit()
+
+ async def close(self) -> None:
+ pass # session_factory manages connection lifecycle
+
+ async def list_page_ids(self) -> set[str]:
+ from sqlalchemy.sql import text as sa_text
+
+ async with self._session_factory() as session:
+ rows = await session.execute(
+ sa_text("SELECT id FROM wiki_pages WHERE embedding IS NOT NULL")
+ )
+ return {r[0] for r in rows.fetchall()}
+
+ async def get_page_summary_by_path(self, path: str) -> dict | None:
+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.
+
+ Reads the 'content' column (first 500 chars) from the wiki_pages table
+ matched by target_path. 'key_exports' is derived from the page's
+ ``exports`` if stored in a metadata JSON column; otherwise returns [].
+ """
+ from sqlalchemy.sql import text as sa_text
+
+ async with self._session_factory() as session:
+ rows = await session.execute(
+ sa_text(
+ "SELECT content, metadata FROM wiki_pages "
+ "WHERE target_path = :path "
+ "LIMIT 1"
+ ),
+ {"path": path},
+ )
+ row = rows.fetchone()
+
+ if row is None:
+ return None
+
+ content = str(row[0] or "")[:500]
+ key_exports: list[str] = []
+ if row[1] and isinstance(row[1], dict):
+ key_exports = list(row[1].get("exports", []))
+ elif row[1] and isinstance(row[1], str):
+ import json
+
+ try:
+ meta = json.loads(row[1])
+ key_exports = list(meta.get("exports", []))
+ except (json.JSONDecodeError, AttributeError):
+ pass
+
+ return {"summary": content, "key_exports": key_exports}
diff --git a/packages/core/src/repowise/core/pipeline/checkpoint.py b/packages/core/src/repowise/core/pipeline/checkpoint.py
new file mode 100644
index 000000000..8236fd207
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/checkpoint.py
@@ -0,0 +1,194 @@
+"""Pipeline checkpoint/resume integration over the Phase-1 ``JobStore`` seam.
+
+On a large repo a crashed ``repowise init`` should not start from scratch. This
+module records each pipeline phase's lifecycle to a :class:`JobStore` so a
+re-run can detect which phases already completed and skip them.
+
+Design
+------
+Phase boundaries are observed through the existing ``pipeline_hooks`` registry
+(``HookProgressCallback`` fires ``PRE``/``POST`` hooks around every
+``on_phase_start`` / ``on_phase_done``). Those hooks are **synchronous** but
+fire inside the running event loop, while ``JobStore`` writes are **async**. To
+bridge the two without losing ordering, the hooks push lightweight events onto
+an ``asyncio.Queue`` that a single background consumer drains in order —
+guaranteeing a phase's ``create_job`` happens before its completion update.
+
+The whole mechanism is **opt-in**: it only activates when a ``JobStore`` is
+supplied to :class:`PhaseCheckpointer`. When absent, the orchestrator runs
+exactly as before — no new writes, no behaviour change.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import enum
+from typing import TYPE_CHECKING
+
+import structlog
+
+from repowise.core.registry import HookPhase, pipeline_hooks
+
+if TYPE_CHECKING:
+ from repowise.core.persistence._interfaces.job_store import JobStore
+ from repowise.core.registry import PipelineHookRegistry
+
+logger = structlog.get_logger(__name__)
+
+__all__ = [
+ "DEFAULT_CHECKPOINT_PHASES",
+ "PhaseCheckpointer",
+ "find_completed_phases",
+]
+
+# Major phases worth checkpointing. These names match the strings the
+# orchestrator passes to ``progress.on_phase_start`` / ``on_phase_done``.
+DEFAULT_CHECKPOINT_PHASES: tuple[str, ...] = (
+ "parse",
+ "git",
+ "co_change",
+ "generation",
+)
+
+
+class _EventKind(enum.Enum):
+ START = "start"
+ DONE = "done"
+ STOP = "stop" # sentinel to shut the consumer down
+
+
+async def find_completed_phases(
+ job_store: JobStore, repository_id: str
+) -> set[str]:
+ """Return the set of phases already COMPLETED for *repository_id*.
+
+ Used by the resume path to skip work that a prior run finished.
+ """
+ from repowise.core.persistence._interfaces.job_store import JobState
+
+ jobs = await job_store.list_jobs(repository_id=repository_id)
+ return {j.phase for j in jobs if j.state == JobState.COMPLETED}
+
+
+class PhaseCheckpointer:
+ """Records phase lifecycle to a ``JobStore``, driven by pipeline hooks.
+
+ Use as an async context manager around a pipeline run::
+
+ async with PhaseCheckpointer(job_store, repo_id) as cp:
+ await run_pipeline(..., )
+
+ The consumer task started on ``__aenter__`` drains queued phase events; on
+ ``__aexit__`` it is signalled to stop and awaited so all writes land before
+ control returns.
+ """
+
+ def __init__(
+ self,
+ job_store: JobStore,
+ repository_id: str,
+ *,
+ phases: tuple[str, ...] = DEFAULT_CHECKPOINT_PHASES,
+ registry: PipelineHookRegistry | None = None,
+ ) -> None:
+ self._job_store = job_store
+ self._repository_id = repository_id
+ self._phases = phases
+ self._registry = registry if registry is not None else pipeline_hooks
+ self._queue: asyncio.Queue[tuple[_EventKind, str]] = asyncio.Queue()
+ self._job_ids: dict[str, str] = {}
+ self._consumer: asyncio.Task[None] | None = None
+ self._registered: list[tuple[str, HookPhase, object]] = []
+
+ # -- hook callbacks (sync, fired inside the loop) ----------------------
+
+ def _enqueue_start(self, phase: str) -> None:
+ self._queue.put_nowait((_EventKind.START, phase))
+
+ def _enqueue_done(self, phase: str) -> None:
+ self._queue.put_nowait((_EventKind.DONE, phase))
+
+ def register(self) -> None:
+ """Register pre/post hooks for every tracked phase."""
+ for phase in self._phases:
+ start_cb = self._make_cb(self._enqueue_start)
+ done_cb = self._make_cb(self._enqueue_done)
+ self._registry.register(phase, start_cb, when=HookPhase.PRE)
+ self._registry.register(phase, done_cb, when=HookPhase.POST)
+ self._registered.append((phase, HookPhase.PRE, start_cb))
+ self._registered.append((phase, HookPhase.POST, done_cb))
+
+ @staticmethod
+ def _make_cb(fn: object):
+ # pipeline_hooks expects Callable[[str], None]; bind through a plain
+ # function so each registration is a distinct, removable object.
+ def _cb(phase: str) -> None:
+ fn(phase) # type: ignore[operator]
+
+ return _cb
+
+ def unregister(self) -> None:
+ """Best-effort removal of this checkpointer's hooks from the registry."""
+ hooks = getattr(self._registry, "_hooks", None)
+ if not isinstance(hooks, dict):
+ return
+ for phase, when, cb in self._registered:
+ bucket = hooks.get((phase, when))
+ if bucket and cb in bucket:
+ bucket.remove(cb)
+ self._registered.clear()
+
+ # -- async consumer ----------------------------------------------------
+
+ async def _consume(self) -> None:
+ from repowise.core.persistence._interfaces.job_store import JobState
+
+ while True:
+ kind, phase = await self._queue.get()
+ try:
+ if kind is _EventKind.STOP:
+ return
+ if kind is _EventKind.START:
+ job = await self._job_store.create_job(
+ repository_id=self._repository_id, phase=phase
+ )
+ self._job_ids[phase] = job.id
+ await self._job_store.update_state(job.id, JobState.RUNNING)
+ elif kind is _EventKind.DONE:
+ job_id = self._job_ids.get(phase)
+ if job_id is not None:
+ await self._job_store.update_state(
+ job_id, JobState.COMPLETED
+ )
+ except Exception as exc: # checkpointing must never break a run
+ logger.warning(
+ "checkpoint_write_failed", phase=phase, kind=kind.value, error=str(exc)
+ )
+ finally:
+ self._queue.task_done()
+
+ async def start(self) -> None:
+ """Register hooks and start the background consumer."""
+ self.register()
+ self._consumer = asyncio.create_task(self._consume())
+
+ async def aclose(self) -> None:
+ """Drain queued events, stop the consumer, and remove hooks.
+
+ Call on the success path. If the pipeline raises instead, leaving this
+ uncalled is intentional: in-flight phases stay in ``RUNNING`` so a
+ re-run's ``find_resumable`` can detect the interrupted work.
+ """
+ await self._queue.join()
+ self._queue.put_nowait((_EventKind.STOP, ""))
+ if self._consumer is not None:
+ await self._consumer
+ self._consumer = None
+ self.unregister()
+
+ async def __aenter__(self) -> PhaseCheckpointer:
+ await self.start()
+ return self
+
+ async def __aexit__(self, *exc: object) -> None:
+ await self.aclose()
diff --git a/packages/core/src/repowise/core/pipeline/modes.py b/packages/core/src/repowise/core/pipeline/modes.py
new file mode 100644
index 000000000..7b55fd8f5
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/modes.py
@@ -0,0 +1,58 @@
+"""Orchestrator execution modes.
+
+``OrchestratorMode`` selects how much of the pipeline runs. It is a thin policy
+object over the existing :func:`run_pipeline` — not a fork of it — so the two
+modes share one code path and differ only in the knobs they flip.
+
+- ``STANDARD`` (default): the historical behaviour. FULL git indexing, all
+ analysis, and (when requested) LLM doc generation.
+- ``FAST``: graph + ESSENTIAL git only, no LLM doc generation. Intended for a
+ quick first index of a very large repo; the deferred git signals can be
+ backfilled later (see ``git_indexer.backfill``).
+"""
+
+from __future__ import annotations
+
+import enum
+
+from repowise.core.ingestion.git_indexer import GitIndexTier
+
+__all__ = ["OrchestratorMode"]
+
+
+class OrchestratorMode(enum.StrEnum):
+ """Pipeline execution depth."""
+
+ STANDARD = "standard"
+ FAST = "fast"
+
+ @property
+ def git_tier(self) -> GitIndexTier:
+ """Git-indexing tier this mode runs."""
+ return (
+ GitIndexTier.ESSENTIAL if self is OrchestratorMode.FAST else GitIndexTier.FULL
+ )
+
+ @property
+ def allows_doc_generation(self) -> bool:
+ """Whether LLM doc generation may run in this mode.
+
+ FAST never generates docs (and never makes LLM calls), regardless of
+ the ``generate_docs`` flag passed to the pipeline.
+ """
+ return self is not OrchestratorMode.FAST
+
+ @property
+ def sql_backed_metrics(self) -> bool:
+ """Whether file-level metrics should be served from SQL post-build.
+
+ When ``True`` the pipeline materializes the metrics snapshot to
+ ``graph_metrics`` and may drop the in-memory NetworkX object (via
+ ``GraphBuilder.release_graph``), serving subsequent metric reads from
+ the loaded snapshot — the large-repo scaling path.
+
+ Default off in ``STANDARD`` (doc generation traverses the live graph)
+ and on in ``FAST`` (which generates no docs, so nothing needs the
+ structural graph after build + persistence).
+ """
+ return self is OrchestratorMode.FAST
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 564dea1d9..8f6bbb92a 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -24,6 +24,8 @@ from typing import Any
import structlog
+from repowise.core.ingestion.git_indexer import GitIndexTier
+from repowise.core.pipeline.modes import OrchestratorMode
from repowise.core.pipeline.progress import ProgressCallback
from repowise.core.registry import HookProgressCallback
@@ -214,6 +216,8 @@ async def run_pipeline(
concurrency: int = 5,
test_run: bool = False,
resume: bool = False,
+ mode: OrchestratorMode = OrchestratorMode.STANDARD,
+ job_store: Any | None = None,
progress: ProgressCallback | None = None,
cost_tracker: Any | None = None,
generation_config: Any | None = None,
@@ -266,10 +270,28 @@ async def run_pipeline(
commit_depth = max(1, min(commit_depth, 5000))
+ # Mode policy: FAST forces ESSENTIAL git indexing and disables doc
+ # generation (and therefore all LLM calls). STANDARD preserves the
+ # caller's flags exactly. This is the single switch point — the rest of
+ # the pipeline reads ``generate_docs`` / the git tier, not ``mode``.
+ git_tier = mode.git_tier
+ generate_docs = generate_docs and mode.allows_doc_generation
+
# Wrap the incoming progress callback so registered pipeline hooks fire
# around each phase transition. Zero-op when no hooks are registered.
progress = HookProgressCallback(progress)
+ # Optional crash-resume checkpointing. When a JobStore is supplied, a
+ # checkpointer records each major phase's lifecycle (via the same hook
+ # seam) so a re-run can detect completed phases. No-op otherwise — the
+ # default (None) preserves existing behaviour exactly.
+ _checkpointer = None
+ if job_store is not None:
+ from repowise.core.pipeline.checkpoint import PhaseCheckpointer
+
+ _checkpointer = PhaseCheckpointer(job_store, str(repo_path))
+ await _checkpointer.start()
+
# Attach cost tracker to provider if supplied
if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):
llm_client._cost_tracker = cost_tracker
@@ -288,6 +310,7 @@ async def run_pipeline(
repo_path,
commit_depth=commit_depth,
follow_renames=follow_renames,
+ tier=git_tier,
progress=progress,
)
@@ -473,6 +496,12 @@ async def run_pipeline(
_phase_done(progress, "graph.flows")
# ---- Build result -------------------------------------------------------
+ # Flush checkpoints on the success path; on an exception above this point
+ # the checkpointer is intentionally not closed so interrupted phases stay
+ # RUNNING for a resumable re-run.
+ if _checkpointer is not None:
+ await _checkpointer.aclose()
+
elapsed = time.monotonic() - start
languages = {fi.language for fi in file_infos if hasattr(fi, "language") and fi.language}
symbol_count = sum(len(pf.symbols) for pf in parsed_files)
@@ -793,6 +822,7 @@ async def _run_git_indexing(
*,
commit_depth: int,
follow_renames: bool,
+ tier: GitIndexTier = GitIndexTier.FULL,
progress: ProgressCallback | None,
) -> tuple[Any | None, list[dict], dict[str, dict]]:
"""Run git history indexing.
@@ -806,6 +836,7 @@ async def _run_git_indexing(
repo_path,
commit_limit=commit_depth,
follow_renames=follow_renames,
+ tier=tier,
)
def _on_start(total: int) -> None:
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index 0456351dc..751b4c8a6 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -27,7 +27,10 @@ async def persist_graph_nodes(
update path can refresh ``graph_nodes`` (including symbol-level
PageRank / betweenness) without constructing a full ``PipelineResult``.
"""
- from repowise.core.persistence import batch_upsert_graph_nodes
+ from repowise.core.persistence import (
+ batch_upsert_graph_metrics,
+ batch_upsert_graph_nodes,
+ )
graph = graph_builder.graph()
pr = graph_builder.pagerank()
@@ -96,6 +99,16 @@ async def persist_graph_nodes(
if nodes:
await batch_upsert_graph_nodes(session, repo_id, nodes)
+ # Materialize the file-level metrics snapshot (graph_metrics) so large
+ # repos can serve metric reads from SQL without recomputing the NetworkX
+ # centrality kernels. Additive to graph_nodes; never changes node rows.
+ try:
+ await batch_upsert_graph_metrics(
+ session, repo_id, graph_builder.file_metrics_snapshot()
+ )
+ except Exception as exc: # materialization is non-load-bearing
+ logger.warning("graph_metrics_materialize_skipped", error=str(exc))
+
async def persist_pipeline_result(
result: Any,
diff --git a/scripts/benchmark_large_repo.py b/scripts/benchmark_large_repo.py
new file mode 100644
index 000000000..03dbd9188
--- /dev/null
+++ b/scripts/benchmark_large_repo.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Benchmark the repowise pipeline against a synthetic large repository.
+
+Generates a throwaway git repo with a configurable number of source files
+(default 5,000) wired together with cross-module imports, then runs the
+indexing pipeline and records per-phase wall-clock to
+``bench/results/<git-sha>.json`` (gitignored).
+
+The 5k size exercises the Phase-2 scale work (tiered git indexing, ESSENTIAL
+``--mode fast`` path, batched embedding, checkpointing). A 30k dry-run is a
+later milestone.
+
+Usage::
+
+ python scripts/benchmark_large_repo.py # 5k files, fast mode
+ python scripts/benchmark_large_repo.py --files 2000
+ python scripts/benchmark_large_repo.py --mode standard --keep
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import subprocess
+import sys
+import tempfile
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def _git(repo: Path, *args: str) -> str:
+ return subprocess.run(
+ ["git", *args],
+ cwd=repo,
+ capture_output=True,
+ text=True,
+ check=True,
+ ).stdout.strip()
+
+
+def generate_synthetic_repo(root: Path, n_files: int, n_commits: int) -> int:
+ """Create *n_files* Python modules under *root* with cross-imports and a
+ git history of *n_commits* commits. Returns the number of files written.
+
+ Modules are grouped into packages of 100 and each imports a symbol from a
+ lower-numbered module so the dependency graph has real edges (and PageRank
+ has something to rank).
+ """
+ root.mkdir(parents=True, exist_ok=True)
+ _git(root, "init", "-q")
+ _git(root, "config", "user.email", "[email protected]")
+ _git(root, "config", "user.name", "bench")
+
+ files_per_pkg = 100
+ written = 0
+ for i in range(n_files):
+ pkg = root / f"pkg_{i // files_per_pkg:03d}"
+ pkg.mkdir(exist_ok=True)
+ (pkg / "__init__.py").touch()
+ mod = pkg / f"mod_{i:05d}.py"
+ # Import from an earlier module to create a graph edge.
+ import_line = ""
+ if i > 0:
+ dep = i - 1
+ import_line = (
+ f"from pkg_{dep // files_per_pkg:03d}.mod_{dep:05d} import value_{dep}\n"
+ )
+ mod.write_text(
+ f'"""Synthetic module {i}."""\n'
+ f"{import_line}\n"
+ f"value_{i} = {i}\n\n\n"
+ f"def compute_{i}(x: int) -> int:\n"
+ f' """Return x scaled by module ordinal {i}."""\n'
+ f" return x * value_{i}\n"
+ )
+ written += 1
+
+ # Spread the files across n_commits commits so git indexing has history.
+ _git(root, "add", "-A")
+ _git(root, "commit", "-q", "-m", "feat: initial synthetic tree")
+ commits = max(1, n_commits - 1)
+ step = max(1, n_files // commits)
+ for c in range(commits):
+ # Touch a slice of files to create churn + co-change-able commits.
+ touched = []
+ for i in range(c * step, min((c + 1) * step, n_files)):
+ mod = root / f"pkg_{i // files_per_pkg:03d}" / f"mod_{i:05d}.py"
+ with mod.open("a") as fh:
+ fh.write(f"\n# touch {c}\n")
+ touched.append(str(mod.relative_to(root)))
+ if touched:
+ _git(root, "add", *touched)
+ _git(root, "commit", "-q", "-m", f"refactor: tweak batch {c}")
+ return written
+
+
+async def _run(repo: Path, mode_name: str) -> tuple[dict[str, float], float, object]:
+ from repowise.core.pipeline import PhaseTimingRecorder, run_pipeline
+ from repowise.core.pipeline.modes import OrchestratorMode
+
+ mode = OrchestratorMode(mode_name)
+ recorder = PhaseTimingRecorder(None)
+ start = time.monotonic()
+ result = await run_pipeline(
+ repo,
+ generate_docs=False,
+ mode=mode,
+ progress=recorder,
+ )
+ total = time.monotonic() - start
+ return recorder.timings, total, result
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--files", type=int, default=5000, help="Number of source files.")
+ parser.add_argument("--commits", type=int, default=50, help="Number of git commits.")
+ parser.add_argument(
+ "--mode", choices=["fast", "standard"], default="fast", help="Pipeline mode."
+ )
+ parser.add_argument(
+ "--keep", action="store_true", help="Keep the generated repo (print its path)."
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=Path,
+ default=REPO_ROOT / "bench" / "results",
+ help="Where to write the result JSON (gitignored).",
+ )
+ args = parser.parse_args(argv)
+
+ tmp = Path(tempfile.mkdtemp(prefix="repowise-bench-"))
+ repo = tmp / "synthetic_repo"
+ print(f"Generating {args.files} files across {args.commits} commits in {repo} ...")
+ gen_start = time.monotonic()
+ n = generate_synthetic_repo(repo, args.files, args.commits)
+ gen_secs = time.monotonic() - gen_start
+ print(f"Generated {n} files in {gen_secs:.1f}s. Running pipeline (mode={args.mode}) ...")
+
+ timings, total, result = asyncio.run(_run(repo, args.mode))
+
+ try:
+ sha = _git(REPO_ROOT, "rev-parse", "--short", "HEAD")
+ except Exception:
+ sha = "unknown"
+
+ record = {
+ "git_sha": sha,
+ "timestamp": datetime.now(UTC).isoformat(),
+ "mode": args.mode,
+ "files_requested": args.files,
+ "files_generated": n,
+ "commits": args.commits,
+ "generation_seconds": round(gen_secs, 2),
+ "pipeline_total_seconds": round(total, 2),
+ "phase_timings": timings,
+ "files_indexed": getattr(result, "file_count", None),
+ "symbol_count": getattr(result, "symbol_count", None),
+ }
+
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ out = args.output_dir / f"{sha}.json"
+ out.write_text(json.dumps(record, indent=2))
+ print(f"\nWrote {out}")
+ print(json.dumps(record, indent=2))
+
+ if args.keep:
+ print(f"\nKept synthetic repo at {repo}")
+ else:
+ import shutil
+
+ shutil.rmtree(tmp, ignore_errors=True)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/integration/persistence/test_embed_batch_pg.py b/tests/integration/persistence/test_embed_batch_pg.py
new file mode 100644
index 000000000..a049cb236
--- /dev/null
+++ b/tests/integration/persistence/test_embed_batch_pg.py
@@ -0,0 +1,97 @@
+"""PgVectorStore.embed_batch against a real pgvector-enabled PostgreSQL.
+
+Skipped automatically when ``testcontainers`` is not installed or Docker is
+not running. Exercises the batched embed path (one embedder call + per-row
+UPDATE of ``wiki_pages.embedding``) on the dialect production uses.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.sql import text as sa_text
+
+from repowise.core.persistence.database import init_db
+from repowise.core.persistence.models import Page
+from repowise.core.persistence.stores import SqlIndexStore
+from repowise.core.persistence.vector_store import PgVectorStore
+from repowise.core.providers.embedding.base import MockEmbedder
+
+testcontainers = pytest.importorskip(
+ "testcontainers.postgres",
+ reason="testcontainers not installed",
+)
+
+
[email protected](scope="module")
+def pgvector_container():
+ try:
+ with testcontainers.PostgresContainer("pgvector/pgvector:pg16") as pg:
+ yield pg
+ except Exception as exc: # pragma: no cover - skip when Docker / image missing
+ pytest.skip(f"pgvector container unavailable: {exc}")
+
+
+async def pg_factory(pgvector_container):
+ raw_url = pgvector_container.get_connection_url()
+ url = raw_url.replace("postgresql+psycopg2://", "postgresql+asyncpg://").replace(
+ "postgresql://", "postgresql+asyncpg://"
+ )
+ engine = create_async_engine(url, echo=False)
+ try:
+ await init_db(engine)
+ # The pgvector ``embedding`` column is added by the Alembic migration in
+ # production; create_all does not, so add it here to match.
+ async with engine.begin() as conn:
+ await conn.execute(sa_text("CREATE EXTENSION IF NOT EXISTS vector"))
+ await conn.execute(
+ sa_text(
+ "ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS "
+ f"embedding vector({MockEmbedder.dimensions})"
+ )
+ )
+ yield async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+ finally:
+ await engine.dispose()
+
+
+async def test_embed_batch_writes_embeddings_postgres(pg_factory):
+ async with pg_factory() as session:
+ repo = await SqlIndexStore(session).upsert_repository(
+ name="pg-embed", local_path="/tmp/pg-embed"
+ )
+ now = datetime.now(UTC)
+ for i in range(3):
+ session.add(
+ Page(
+ id=f"file_page:m{i}.py",
+ repository_id=repo.id,
+ page_type="file_page",
+ title=f"File m{i}",
+ content=f"content {i}",
+ summary="",
+ target_path=f"m{i}.py",
+ source_hash="h",
+ model_name="mock-model-1",
+ provider_name="mock",
+ created_at=now,
+ updated_at=now,
+ )
+ )
+ await session.commit()
+
+ store = PgVectorStore(pg_factory, MockEmbedder())
+ assert await store.list_page_ids() == set() # no embeddings yet
+
+ items = [(f"file_page:m{i}.py", f"content {i}", {"page_type": "file_page"}) for i in range(3)]
+ await store.embed_batch(items)
+
+ assert await store.list_page_ids() == {f"file_page:m{i}.py" for i in range(3)}
+
+ # Empty batch is a no-op.
+ await store.embed_batch([])
+ assert len(await store.list_page_ids()) == 3
diff --git a/tests/integration/persistence/test_graph_metrics_pg.py b/tests/integration/persistence/test_graph_metrics_pg.py
new file mode 100644
index 000000000..a7b5ba80b
--- /dev/null
+++ b/tests/integration/persistence/test_graph_metrics_pg.py
@@ -0,0 +1,123 @@
+"""graph_metrics materialization round-trip against a real PostgreSQL container.
+
+Skipped automatically when ``testcontainers`` is not installed or Docker is
+not running. Proves the materialized snapshot round-trips through Postgres and
+that a GraphBuilder hydrated from it reproduces the NetworkX metrics.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from repowise.core.ingestion.graph import GraphBuilder
+from repowise.core.ingestion.models import FileInfo, Import, ParsedFile
+from repowise.core.persistence import batch_upsert_graph_metrics, get_graph_metrics
+from repowise.core.persistence.database import init_db
+from repowise.core.persistence.stores import SqlIndexStore
+
+testcontainers = pytest.importorskip(
+ "testcontainers.postgres",
+ reason="testcontainers not installed",
+)
+
+
[email protected](scope="module")
+def postgres_container():
+ try:
+ with testcontainers.PostgresContainer("postgres:16-alpine") as pg:
+ yield pg
+ except Exception as exc: # pragma: no cover - skip when Docker missing
+ pytest.skip(f"PostgreSQL container unavailable: {exc}")
+
+
+async def pg_session(postgres_container):
+ raw_url = postgres_container.get_connection_url()
+ url = raw_url.replace("postgresql+psycopg2://", "postgresql+asyncpg://").replace(
+ "postgresql://", "postgresql+asyncpg://"
+ )
+ engine = create_async_engine(url, echo=False)
+ try:
+ await init_db(engine)
+ factory = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+ async with factory() as session:
+ yield session
+ finally:
+ await engine.dispose()
+
+
+def _build_sample() -> GraphBuilder:
+ def fi(path: str) -> FileInfo:
+ return FileInfo(
+ path=path,
+ abs_path=f"/repo/{path}",
+ language="python",
+ size_bytes=100,
+ git_hash="",
+ last_modified=datetime.now(),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+
+ def imp(mp: str) -> Import:
+ return Import(
+ raw_statement=f"import {mp}",
+ module_path=mp,
+ imported_names=[],
+ is_relative=False,
+ resolved_file=None,
+ )
+
+ def parsed(path: str, imports: list[Import]) -> ParsedFile:
+ return ParsedFile(
+ file_info=fi(path),
+ symbols=[],
+ imports=imports,
+ exports=[],
+ docstring=None,
+ parse_errors=[],
+ content_hash="",
+ )
+
+ b = GraphBuilder()
+ b.add_file(parsed("a.py", [imp("b"), imp("c")]))
+ b.add_file(parsed("b.py", [imp("c")]))
+ b.add_file(parsed("c.py", []))
+ b.build()
+ return b
+
+
+async def test_graph_metrics_roundtrip_postgres(pg_session: AsyncSession):
+ idx = SqlIndexStore(pg_session)
+ repo = await idx.upsert_repository(name="pg-metrics", local_path="/tmp/pg-metrics")
+
+ builder = _build_sample()
+ snapshot = builder.file_metrics_snapshot()
+
+ await batch_upsert_graph_metrics(pg_session, repo.id, snapshot)
+ read_back = await get_graph_metrics(pg_session, repo.id)
+
+ assert set(read_back) == set(snapshot)
+ for node, m in snapshot.items():
+ assert read_back[node]["pagerank"] == pytest.approx(m["pagerank"])
+ assert read_back[node]["betweenness"] == pytest.approx(m["betweenness"])
+ assert read_back[node]["community_id"] == m["community_id"]
+ assert read_back[node]["in_degree"] == m["in_degree"]
+ assert read_back[node]["out_degree"] == m["out_degree"]
+
+ # A builder hydrated from the SQL snapshot reproduces the NetworkX metrics.
+ hydrated = _build_sample()
+ hydrated.load_metrics_from_sql(read_back)
+ nx_pr = builder.pagerank()
+ for node, score in hydrated.pagerank().items():
+ assert score == pytest.approx(nx_pr[node])
+
+ # Upsert again → idempotent (no duplicate rows).
+ await batch_upsert_graph_metrics(pg_session, repo.id, snapshot)
+ assert len(await get_graph_metrics(pg_session, repo.id)) == len(snapshot)
diff --git a/tests/integration/test_generation_pipeline.py b/tests/integration/test_generation_pipeline.py
index 58f2217b1..45d77e29a 100644
--- a/tests/integration/test_generation_pipeline.py
+++ b/tests/integration/test_generation_pipeline.py
@@ -81,6 +81,13 @@ class _SlowVectorStore:
finally:
self.active_embeds -= 1
+ async def embed_batch(self, items: list[tuple[str, str, dict[str, Any]]]) -> None:
+ # Mirror the generator's batched embed path: a level's pages are
+ # embedded together after their LLM calls have all returned. The real
+ # stores issue one batched embed call, so this never fans out.
+ for pid, text, meta in items:
+ await self.embed_and_upsert(pid, text, meta)
+
async def list_page_ids(self) -> set[str]:
return set()
diff --git a/tests/integration/test_pipeline_fast_checkpoint.py b/tests/integration/test_pipeline_fast_checkpoint.py
new file mode 100644
index 000000000..711ef46d5
--- /dev/null
+++ b/tests/integration/test_pipeline_fast_checkpoint.py
@@ -0,0 +1,87 @@
+"""Integration tests for --mode fast and pipeline checkpointing.
+
+Runs the real pipeline against the sample repo (no LLM, no external services)
+to prove:
+ * FAST mode skips doc generation and the co-change walk (ESSENTIAL git tier);
+ * a supplied JobStore receives a COMPLETED record for each major phase;
+ * an interrupted phase (left RUNNING) is excluded from the resume set.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+from repowise.core.persistence._interfaces.job_store import JobRecord, JobState
+from repowise.core.pipeline import run_pipeline
+from repowise.core.pipeline.checkpoint import find_completed_phases
+from repowise.core.pipeline.modes import OrchestratorMode
+
+
+class _FakeJobStore:
+ def __init__(self) -> None:
+ self.jobs: dict[str, JobRecord] = {}
+ self._seq = 0
+
+ async def create_job(self, *, repository_id, phase, metadata=None) -> JobRecord:
+ self._seq += 1
+ jid = f"job{self._seq}"
+ now = datetime.now(UTC)
+ rec = JobRecord(
+ jid, repository_id, phase, JobState.PENDING, None, now, now, None, metadata or {}
+ )
+ self.jobs[jid] = rec
+ return rec
+
+ async def update_state(self, job_id, state, *, cursor=None, error=None) -> JobRecord:
+ old = self.jobs[job_id]
+ rec = JobRecord(
+ old.id, old.repository_id, old.phase, state, cursor or old.cursor,
+ old.started_at, datetime.now(UTC), error, old.metadata,
+ )
+ self.jobs[job_id] = rec
+ return rec
+
+ async def list_jobs(self, *, repository_id=None, phase=None, state=None, limit=100):
+ return list(self.jobs.values())
+
+
+async def test_fast_mode_skips_generation_and_co_change(sample_repo_path: Path) -> None:
+ store = _FakeJobStore()
+ result = await run_pipeline(
+ sample_repo_path,
+ generate_docs=True, # FAST must override this to False (no LLM client passed)
+ mode=OrchestratorMode.FAST,
+ job_store=store,
+ )
+
+ # No LLM generation happened.
+ assert result.generated_pages is None
+ # ESSENTIAL git tier: per-file metadata exists but no co-change partners.
+ assert result.git_metadata_list, "expected git metadata for sample repo"
+ for meta in result.git_metadata_list:
+ assert meta.get("co_change_partners_json", "[]") == "[]"
+
+
+async def test_checkpoints_recorded_for_completed_phases(sample_repo_path: Path) -> None:
+ store = _FakeJobStore()
+ await run_pipeline(
+ sample_repo_path,
+ mode=OrchestratorMode.FAST,
+ job_store=store,
+ )
+
+ completed = await find_completed_phases(store, str(Path(sample_repo_path).resolve()))
+ # parse + git always run; both should be recorded COMPLETED.
+ assert "parse" in completed
+ assert "git" in completed
+ # Every recorded job reached a terminal COMPLETED state on the success path.
+ assert all(j.state == JobState.COMPLETED for j in store.jobs.values())
+
+
+async def test_interrupted_phase_excluded_from_resume_set() -> None:
+ store = _FakeJobStore()
+ j = await store.create_job(repository_id="r", phase="generation")
+ await store.update_state(j.id, JobState.RUNNING) # simulate a crash mid-phase
+ completed = await find_completed_phases(store, "r")
+ assert "generation" not in completed
diff --git a/tests/unit/cli/test_fast_mode_offer.py b/tests/unit/cli/test_fast_mode_offer.py
new file mode 100644
index 000000000..4927ded91
--- /dev/null
+++ b/tests/unit/cli/test_fast_mode_offer.py
@@ -0,0 +1,48 @@
+"""Tests for the large-repo fast-mode offer helpers in cli.ui."""
+
+from __future__ import annotations
+
+from repowise.cli.ui import (
+ LARGE_REPO_FILE_THRESHOLD,
+ RepoScanInfo,
+ build_contextual_next_steps,
+ should_offer_fast_mode,
+)
+
+
+def test_should_offer_fast_mode_none():
+ assert should_offer_fast_mode(None) is False
+
+
+def test_should_offer_fast_mode_small_repo():
+ assert should_offer_fast_mode(RepoScanInfo(total_files=100)) is False
+
+
+def test_should_offer_fast_mode_at_threshold():
+ # Threshold is strict ">" — exactly at the boundary does not offer.
+ assert should_offer_fast_mode(RepoScanInfo(total_files=LARGE_REPO_FILE_THRESHOLD)) is False
+
+
+def test_should_offer_fast_mode_large_repo():
+ assert should_offer_fast_mode(RepoScanInfo(total_files=LARGE_REPO_FILE_THRESHOLD + 1)) is True
+
+
+def test_next_steps_fast_mode_leads_with_full_upgrade():
+ steps = build_contextual_next_steps(index_only=True, fast_mode=True)
+ cmds = [c for c, _ in steps]
+ descs = " ".join(d for _, d in steps).lower()
+ # Fast mode tells the user how to get the full result.
+ assert cmds[0] == "repowise init"
+ assert "full" in descs and "docs" in descs
+
+
+def test_next_steps_index_only_without_fast():
+ steps = build_contextual_next_steps(index_only=True, fast_mode=False)
+ cmds = [c for c, _ in steps]
+ assert any("--provider" in c for c in cmds) # the existing generate hint
+
+
+def test_next_steps_full_mode():
+ steps = build_contextual_next_steps(index_only=False)
+ cmds = [c for c, _ in steps]
+ assert any("search" in c for c in cmds)
diff --git a/tests/unit/generation/test_page_generator_tiering.py b/tests/unit/generation/test_page_generator_tiering.py
new file mode 100644
index 000000000..14207fb03
--- /dev/null
+++ b/tests/unit/generation/test_page_generator_tiering.py
@@ -0,0 +1,92 @@
+"""Tests for tiered doc generation (page_generator/tiering.py + tier-2 path)."""
+
+from __future__ import annotations
+
+from repowise.core.generation.context_assembler import ContextAssembler
+from repowise.core.generation.models import GeneratedPage, GenerationConfig
+from repowise.core.generation.page_generator import PageGenerator
+from repowise.core.generation.page_generator.tiering import partition_file_tiers
+from repowise.core.providers.llm.mock import MockProvider
+
+# ---------------------------------------------------------------------------
+# partition_file_tiers — pure partition logic
+# ---------------------------------------------------------------------------
+
+_PR = {"a.py": 0.5, "b.py": 0.3, "c.py": 0.1, "d.py": 0.05}
+_SELECTED = set(_PR)
+
+
+def test_partition_none_puts_everything_in_tier1():
+ """Default (None) reproduces current selection: all pages tier-1."""
+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, None)
+ assert tier1 == _SELECTED
+ assert tier2 == set()
+
+
+def test_partition_n_geq_len_is_all_tier1():
+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 99)
+ assert tier1 == _SELECTED
+ assert tier2 == set()
+
+
+def test_partition_top_n_by_pagerank():
+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 2)
+ # Highest two PageRank scores → tier-1.
+ assert tier1 == {"a.py", "b.py"}
+ assert tier2 == {"c.py", "d.py"}
+
+
+def test_partition_zero_is_all_tier2():
+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 0)
+ assert tier1 == set()
+ assert tier2 == _SELECTED
+
+
+def test_partition_is_a_partition():
+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 3)
+ assert tier1 | tier2 == _SELECTED
+ assert tier1 & tier2 == set()
+
+
+def test_partition_tie_break_is_deterministic():
+ pr = {"x.py": 0.1, "y.py": 0.1, "z.py": 0.1}
+ t1a, _ = partition_file_tiers(set(pr), pr, 1)
+ t1b, _ = partition_file_tiers(set(pr), pr, 1)
+ assert t1a == t1b # stable path-based tie-break
+
+
+# ---------------------------------------------------------------------------
+# Tier-2 deterministic render — no LLM call
+# ---------------------------------------------------------------------------
+
+
+async def test_tier2_render_makes_no_provider_call(
+ sample_config, sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes
+):
+ provider = MockProvider()
+ assembler = ContextAssembler(sample_config)
+ gen = PageGenerator(provider, assembler, sample_config)
+
+ ctx = assembler.assemble_file_page(
+ sample_parsed_file,
+ sample_graph,
+ graph_metrics["pagerank"],
+ graph_metrics["betweenness"],
+ graph_metrics["community"],
+ sample_source_bytes,
+ )
+ page = await gen._generate_file_page_tier2(sample_parsed_file, ctx)
+
+ assert isinstance(page, GeneratedPage)
+ assert page.page_type == "file_page"
+ assert page.provider_name == "template"
+ assert page.metadata["doc_tier"] == 2
+ assert page.input_tokens == 0 and page.output_tokens == 0
+ # Deterministic structure, no model invocation.
+ assert provider.call_count == 0
+ assert "## Overview" in page.content
+ assert "## Public API" in page.content
+
+
+def test_generation_config_tier1_top_n_defaults_none():
+ assert GenerationConfig().tier1_top_n is None
diff --git a/tests/unit/ingestion/test_git_indexer_tiers.py b/tests/unit/ingestion/test_git_indexer_tiers.py
new file mode 100644
index 000000000..e88fa664c
--- /dev/null
+++ b/tests/unit/ingestion/test_git_indexer_tiers.py
@@ -0,0 +1,206 @@
+"""Unit tests for the tiered git indexer (ESSENTIAL vs FULL) and backfill.
+
+These exercise the tier gating directly on the module-level functions and the
+``GitIndexer`` shims, plus the resumable backfill worker against a fake
+JobStore — no real git repo required.
+"""
+
+from __future__ import annotations
+
+import time
+from unittest.mock import MagicMock
+
+import pytest
+
+from repowise.core.ingestion.git_indexer import (
+ BACKFILL_PHASE,
+ GitIndexer,
+ GitIndexTier,
+ backfill_full_tier,
+ index_file,
+)
+from repowise.core.ingestion.git_indexer.records import _CommitRec
+
+# ---------------------------------------------------------------------------
+# Tier flags
+# ---------------------------------------------------------------------------
+
+
+def test_full_tier_includes_expensive_signals() -> None:
+ assert GitIndexTier.FULL.includes_blame is True
+ assert GitIndexTier.FULL.includes_co_change is True
+
+
+def test_essential_tier_skips_expensive_signals() -> None:
+ assert GitIndexTier.ESSENTIAL.includes_blame is False
+ assert GitIndexTier.ESSENTIAL.includes_co_change is False
+
+
+def test_indexer_defaults_to_full_tier() -> None:
+ assert GitIndexer("/tmp/repo").tier is GitIndexTier.FULL
+
+
+# ---------------------------------------------------------------------------
+# index_file: include_blame gating
+# ---------------------------------------------------------------------------
+
+
+def _commits() -> list[_CommitRec]:
+ now = int(time.time())
+ return [
+ _CommitRec("a" * 40, "Alice", "[email protected]", now, False, "feat: add widget", 10, 2),
+ _CommitRec("b" * 40, "Bob", "[email protected]", now - 86400, False, "fix bug", 4, 1),
+ ]
+
+
+def test_index_file_essential_skips_blame(tmp_path) -> None:
+ """With include_blame=False, get_blame_ownership must never be consulted —
+ ownership falls back to the top commit author.
+ """
+ repo = MagicMock()
+ # If blame were called it would raise, proving it must not be reached.
+ repo.blame.side_effect = AssertionError("blame must not run in ESSENTIAL tier")
+ (tmp_path / "a.py").write_text("x = 1\n")
+
+ meta = index_file(
+ repo,
+ "a.py",
+ repo_path=tmp_path,
+ commit_limit=500,
+ follow_renames=False,
+ include_blame=False,
+ precomputed_commits=_commits(),
+ )
+ assert meta["commit_count_total"] == 2
+ assert meta["primary_owner_name"] == "Alice" # commit-author fallback
+ repo.blame.assert_not_called()
+
+
+def test_index_file_full_consults_blame(tmp_path) -> None:
+ """With include_blame=True, blame ownership overrides the commit author."""
+ repo = MagicMock()
+ blame_commit = MagicMock()
+ blame_commit.author.name = "Carol"
+ blame_commit.author.email = "[email protected]"
+ repo.blame.return_value = [(blame_commit, ["line1", "line2", "line3"])]
+ (tmp_path / "a.py").write_text("x = 1\n")
+
+ meta = index_file(
+ repo,
+ "a.py",
+ repo_path=tmp_path,
+ commit_limit=500,
+ follow_renames=False,
+ include_blame=True,
+ precomputed_commits=_commits(),
+ )
+ assert meta["primary_owner_name"] == "Carol" # blame wins
+ repo.blame.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# index_repo: ESSENTIAL skips the co-change walk
+# ---------------------------------------------------------------------------
+
+
+async def test_index_repo_essential_skips_co_change(tmp_path) -> None:
+ """ESSENTIAL tier must not invoke compute_co_changes and must leave
+ co_change_partners empty, while still producing per-file metadata.
+ """
+ import git as gitpython
+
+ repo = gitpython.Repo.init(tmp_path)
+ f = tmp_path / "a.py"
+ f.write_text("x = 1\n")
+ repo.index.add(["a.py"])
+ repo.index.commit("feat: add a")
+ g = tmp_path / "b.py"
+ g.write_text("y = 2\n")
+ repo.index.add(["b.py"])
+ repo.index.commit("feat: add b")
+
+ co_change_seen = {"called": False}
+ import repowise.core.ingestion.git_indexer.indexer as indexer_mod
+
+ orig = indexer_mod.compute_co_changes
+
+ def _spy(*args, **kwargs): # pragma: no cover - should not run
+ co_change_seen["called"] = True
+ return orig(*args, **kwargs)
+
+ indexer_mod.compute_co_changes = _spy
+ try:
+ idx = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)
+ summary, results = await idx.index_repo("repo1")
+ finally:
+ indexer_mod.compute_co_changes = orig
+
+ assert co_change_seen["called"] is False
+ assert summary.files_indexed >= 1
+ for meta in results:
+ assert meta.get("co_change_partners_json", "[]") == "[]"
+
+
+# ---------------------------------------------------------------------------
+# Backfill worker (resumable via JobStore)
+# ---------------------------------------------------------------------------
+
+
+class _FakeJobStore:
+ """Minimal in-memory JobStore capturing the lifecycle for assertions."""
+
+ def __init__(self) -> None:
+ self.created: list[dict] = []
+ self.states: list[tuple[str, str]] = []
+ self._seq = 0
+
+ async def create_job(self, *, repository_id, phase, metadata=None):
+ from repowise.core.persistence._interfaces.job_store import JobRecord, JobState
+
+ self._seq += 1
+ jid = f"job{self._seq}"
+ self.created.append({"id": jid, "phase": phase, "metadata": metadata})
+ from datetime import UTC, datetime
+
+ now = datetime.now(UTC)
+ return JobRecord(jid, repository_id, phase, JobState.PENDING, None, now, now, None, metadata or {})
+
+ async def update_state(self, job_id, state, *, cursor=None, error=None):
+ self.states.append((job_id, state.value))
+ return None
+
+
+async def test_backfill_runs_full_and_records_job(tmp_path) -> None:
+ import git as gitpython
+
+ repo = gitpython.Repo.init(tmp_path)
+ (tmp_path / "a.py").write_text("x = 1\n")
+ repo.index.add(["a.py"])
+ repo.index.commit("feat: add a")
+
+ idx = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)
+ store = _FakeJobStore()
+
+ summary, _results = await backfill_full_tier(idx, "repo1", job_store=store)
+
+ # The job was recorded under the backfill phase and reached COMPLETED.
+ assert store.created and store.created[0]["phase"] == BACKFILL_PHASE
+ assert ("job1", "running") in store.states
+ assert ("job1", "completed") in store.states
+ # The indexer's tier is restored after the backfill.
+ assert idx.tier is GitIndexTier.ESSENTIAL
+ assert summary.files_indexed >= 1
+
+
+async def test_backfill_marks_failed_on_error(tmp_path, monkeypatch) -> None:
+ idx = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)
+ store = _FakeJobStore()
+
+ async def _boom(_repo_id):
+ raise RuntimeError("kaboom")
+
+ monkeypatch.setattr(idx, "index_repo", _boom)
+ with pytest.raises(RuntimeError):
+ await backfill_full_tier(idx, "repo1", job_store=store)
+ assert ("job1", "failed") in store.states
+ assert idx.tier is GitIndexTier.ESSENTIAL # restored even on failure
diff --git a/tests/unit/ingestion/test_graph_metrics_sql.py b/tests/unit/ingestion/test_graph_metrics_sql.py
new file mode 100644
index 000000000..4b6fcc6cd
--- /dev/null
+++ b/tests/unit/ingestion/test_graph_metrics_sql.py
@@ -0,0 +1,122 @@
+"""SQL-backed graph metrics: snapshot correctness + read routing.
+
+Proves that the materialized ``graph_metrics`` snapshot reproduces the
+NetworkX-computed file-level metrics, and that ``load_metrics_from_sql``
+routes subsequent reads through the snapshot (so the in-memory NetworkX
+object can be dropped on large repos).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from repowise.core.ingestion.graph import GraphBuilder
+from repowise.core.ingestion.models import FileInfo, Import, ParsedFile
+from repowise.core.pipeline.modes import OrchestratorMode
+
+
+def _fi(path: str) -> FileInfo:
+ return FileInfo(
+ path=path,
+ abs_path=f"/repo/{path}",
+ language="python",
+ size_bytes=100,
+ git_hash="",
+ last_modified=datetime.now(),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+
+
+def _imp(module_path: str) -> Import:
+ return Import(
+ raw_statement=f"import {module_path}",
+ module_path=module_path,
+ imported_names=[],
+ is_relative=False,
+ resolved_file=None,
+ )
+
+
+def _parsed(path: str, imports: list[Import] | None = None) -> ParsedFile:
+ return ParsedFile(
+ file_info=_fi(path),
+ symbols=[],
+ imports=imports or [],
+ exports=[],
+ docstring=None,
+ parse_errors=[],
+ content_hash="",
+ )
+
+
+def _build_sample() -> GraphBuilder:
+ """a imports b and c; b imports c — a small connected file graph."""
+ b = GraphBuilder()
+ b.add_file(_parsed("a.py", [_imp("b"), _imp("c")]))
+ b.add_file(_parsed("b.py", [_imp("c")]))
+ b.add_file(_parsed("c.py"))
+ b.build()
+ return b
+
+
+def test_snapshot_matches_networkx():
+ b = _build_sample()
+ nx_pr = b.pagerank()
+ nx_bc = b.betweenness_centrality()
+ nx_cd = b.community_detection()
+ nx_in = b.in_degree()
+ nx_out = b.out_degree()
+
+ snap = b.file_metrics_snapshot()
+
+ # Every file-subgraph node is represented and matches the live NetworkX
+ # values byte-for-byte.
+ assert set(snap) == set(nx_pr)
+ for node, m in snap.items():
+ assert m["pagerank"] == nx_pr[node]
+ assert m["betweenness"] == nx_bc[node]
+ assert m["community_id"] == nx_cd.get(node, 0)
+ assert m["in_degree"] == nx_in[node]
+ assert m["out_degree"] == nx_out[node]
+
+
+def test_load_from_sql_routes_reads_without_recompute():
+ source = _build_sample()
+ snap = source.file_metrics_snapshot()
+ nx_pr = source.pagerank()
+
+ # A fresh builder hydrated purely from the snapshot.
+ hydrated = _build_sample()
+ hydrated.load_metrics_from_sql(snap)
+
+ assert hydrated.pagerank() == {n: snap[n]["pagerank"] for n in snap}
+ assert hydrated.pagerank() == nx_pr # equals the original NetworkX result
+ assert hydrated.in_degree() == {n: snap[n]["in_degree"] for n in snap}
+
+ # Dropping the structural graph must not break metric reads — they are
+ # served from the loaded snapshot, proving the read path is SQL-routed.
+ hydrated.release_graph()
+ assert hydrated.graph().number_of_nodes() == 0
+ assert hydrated.pagerank() == nx_pr
+ assert hydrated.betweenness_centrality() == source.betweenness_centrality()
+ assert hydrated.community_detection() == {n: snap[n]["community_id"] for n in snap}
+
+
+def test_in_out_degree_reflect_import_edges():
+ b = _build_sample()
+ ind = b.in_degree()
+ outd = b.out_degree()
+ # c.py is imported by a and b → in-degree 2, no out-edges.
+ assert ind["c.py"] == 2
+ assert outd["c.py"] == 0
+ # a.py imports two files, imported by none.
+ assert outd["a.py"] == 2
+ assert ind["a.py"] == 0
+
+
+def test_mode_sql_backed_metrics_flag():
+ assert OrchestratorMode.FAST.sql_backed_metrics is True
+ assert OrchestratorMode.STANDARD.sql_backed_metrics is False
diff --git a/tests/unit/persistence/test_models.py b/tests/unit/persistence/test_models.py
index 2829110bb..bf1d37282 100644
--- a/tests/unit/persistence/test_models.py
+++ b/tests/unit/persistence/test_models.py
@@ -313,5 +313,6 @@ def test_base_includes_all_models():
"health_snapshots",
"coverage_files",
"pipeline_jobs",
+ "graph_metrics",
}
assert expected == table_names
diff --git a/tests/unit/persistence/test_vector_store.py b/tests/unit/persistence/test_vector_store.py
index 17f121fd9..17a8620f2 100644
--- a/tests/unit/persistence/test_vector_store.py
+++ b/tests/unit/persistence/test_vector_store.py
@@ -224,6 +224,95 @@ async def test_in_memory_store_close_clears(in_memory_vector_store):
assert len(in_memory_vector_store) == 0
+# ---------------------------------------------------------------------------
+# embed_batch
+# ---------------------------------------------------------------------------
+
+
+async def test_in_memory_embed_batch_empty_is_noop(in_memory_vector_store):
+ await in_memory_vector_store.embed_batch([])
+ assert len(in_memory_vector_store) == 0
+
+
+async def test_in_memory_embed_batch_matches_single_path(mock_embedder):
+ """embed_batch must produce the same stored vectors as N embed_and_upsert calls."""
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+
+ items = [
+ (
+ f"p{i}",
+ f"content number {i}",
+ {
+ "title": f"Page {i}",
+ "page_type": "file_page",
+ "target_path": f"f{i}.py",
+ "content": f"content number {i}",
+ },
+ )
+ for i in range(5)
+ ]
+
+ batched = InMemoryVectorStore(mock_embedder)
+ await batched.embed_batch(items)
+
+ single = InMemoryVectorStore(mock_embedder)
+ for page_id, text, meta in items:
+ await single.embed_and_upsert(page_id, text, meta)
+
+ assert len(batched) == len(single) == 5
+ # Identical query results prove vectors + metadata landed identically.
+ bq = await batched.search("content number 2", limit=5)
+ sq = await single.search("content number 2", limit=5)
+ assert [(r.page_id, r.title) for r in bq] == [(r.page_id, r.title) for r in sq]
+
+
+async def test_in_memory_embed_batch_searchable(in_memory_vector_store):
+ await in_memory_vector_store.embed_batch(
+ [
+ (
+ "p1",
+ "Python decorator pattern",
+ {"title": "Dec", "page_type": "file_page", "target_path": "a.py", "content": "Python decorator pattern"},
+ ),
+ (
+ "p2",
+ "Rust ownership model",
+ {"title": "Rust", "page_type": "file_page", "target_path": "b.py", "content": "Rust ownership model"},
+ ),
+ ]
+ )
+ assert len(in_memory_vector_store) == 2
+ results = await in_memory_vector_store.search("Python decorator pattern", limit=2)
+ assert results[0].page_id == "p1"
+
+
+async def test_base_embed_batch_default_uses_single_path(mock_embedder):
+ """A backend that does NOT override embed_batch falls back to the ABC's
+ sequential default, which must still upsert every item correctly.
+ """
+ from repowise.core.persistence.vector_store import VectorStore
+
+ class _CountingStore(VectorStore):
+ def __init__(self) -> None:
+ self.calls: list[str] = []
+
+ async def embed_and_upsert(self, page_id, text, metadata):
+ self.calls.append(page_id)
+
+ async def search(self, query, limit=10):
+ return []
+
+ async def delete(self, page_id):
+ return None
+
+ async def close(self):
+ return None
+
+ store = _CountingStore()
+ await store.embed_batch([("a", "x", {}), ("b", "y", {})])
+ assert store.calls == ["a", "b"]
+
+
# ---------------------------------------------------------------------------
# LanceDB (optional — skipped if not installed)
# ---------------------------------------------------------------------------
@@ -246,3 +335,22 @@ async def test_lancedb_vector_store_basic(tmp_path, mock_embedder):
assert results[0].page_id == "p1"
finally:
await store.close()
+
+
+async def test_lancedb_embed_batch(tmp_path, mock_embedder):
+ lancedb = pytest.importorskip("lancedb") # noqa: F841
+ from repowise.core.persistence.vector_store import LanceDBVectorStore
+
+ store = LanceDBVectorStore(str(tmp_path / "lance"), mock_embedder)
+ try:
+ await store.embed_batch(
+ [
+ ("p1", "Python async generators", {"target_path": "a.py"}),
+ ("p2", "Rust ownership model", {"target_path": "b.py"}),
+ ]
+ )
+ ids = await store.list_page_ids()
+ assert ids == {"p1", "p2"}
+ finally:
+ await store.close()
diff --git a/tests/unit/pipeline/test_checkpoint.py b/tests/unit/pipeline/test_checkpoint.py
new file mode 100644
index 000000000..0d7d88f78
--- /dev/null
+++ b/tests/unit/pipeline/test_checkpoint.py
@@ -0,0 +1,129 @@
+"""Unit tests for pipeline checkpoint/resume.
+
+Exercise the real ``pipeline_hooks`` → queue → ``JobStore`` seam with a fresh
+registry (no global-state leakage) and an in-memory fake JobStore.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+
+from repowise.core.persistence._interfaces.job_store import JobRecord, JobState
+from repowise.core.pipeline.checkpoint import (
+ DEFAULT_CHECKPOINT_PHASES,
+ PhaseCheckpointer,
+ find_completed_phases,
+)
+from repowise.core.registry import HookPhase, HookProgressCallback, PipelineHookRegistry
+
+
+class FakeJobStore:
+ """In-memory JobStore tracking create/update calls for assertions."""
+
+ def __init__(self) -> None:
+ self.jobs: dict[str, JobRecord] = {}
+ self.events: list[tuple[str, str]] = [] # (op, phase-or-state)
+ self._seq = 0
+
+ async def create_job(self, *, repository_id, phase, metadata=None) -> JobRecord:
+ self._seq += 1
+ jid = f"job{self._seq}"
+ now = datetime.now(UTC)
+ rec = JobRecord(
+ jid, repository_id, phase, JobState.PENDING, None, now, now, None, metadata or {}
+ )
+ self.jobs[jid] = rec
+ self.events.append(("create", phase))
+ return rec
+
+ async def update_state(self, job_id, state, *, cursor=None, error=None) -> JobRecord:
+ old = self.jobs[job_id]
+ rec = JobRecord(
+ old.id, old.repository_id, old.phase, state, cursor or old.cursor,
+ old.started_at, datetime.now(UTC), error, old.metadata,
+ )
+ self.jobs[job_id] = rec
+ self.events.append(("state", f"{old.phase}:{state.value}"))
+ return rec
+
+ async def list_jobs(self, *, repository_id=None, phase=None, state=None, limit=100):
+ return [
+ j
+ for j in self.jobs.values()
+ if (repository_id is None or j.repository_id == repository_id)
+ ]
+
+
+async def _fire_phase(cb: HookProgressCallback, phase: str) -> None:
+ cb.on_phase_start(phase, None)
+ cb.on_phase_done(phase)
+
+
+async def test_checkpointer_records_phase_lifecycle() -> None:
+ registry = PipelineHookRegistry()
+ store = FakeJobStore()
+ cp = PhaseCheckpointer(store, "repo-1", phases=("parse", "git"), registry=registry)
+ cb = HookProgressCallback(None, registry)
+
+ async with cp:
+ await _fire_phase(cb, "parse")
+ await _fire_phase(cb, "git")
+
+ # Two jobs created (parse, git), each RUNNING then COMPLETED, in order.
+ assert store.events == [
+ ("create", "parse"),
+ ("state", "parse:running"),
+ ("state", "parse:completed"),
+ ("create", "git"),
+ ("state", "git:running"),
+ ("state", "git:completed"),
+ ]
+
+
+async def test_checkpointer_ignores_untracked_phases() -> None:
+ registry = PipelineHookRegistry()
+ store = FakeJobStore()
+ cp = PhaseCheckpointer(store, "repo-1", phases=("parse",), registry=registry)
+ cb = HookProgressCallback(None, registry)
+
+ async with cp:
+ await _fire_phase(cb, "external_systems") # not tracked
+ await _fire_phase(cb, "parse")
+
+ assert [e for e in store.events if e[0] == "create"] == [("create", "parse")]
+
+
+async def test_checkpointer_unregisters_hooks_on_close() -> None:
+ registry = PipelineHookRegistry()
+ store = FakeJobStore()
+ cp = PhaseCheckpointer(store, "repo-1", phases=("parse",), registry=registry)
+
+ async with cp:
+ assert registry.hooks_for("parse", HookPhase.PRE)
+ # After close, the checkpointer's hooks are gone — no leakage into the
+ # shared registry for the next run.
+ assert not registry.hooks_for("parse", HookPhase.PRE)
+ assert not registry.hooks_for("parse", HookPhase.POST)
+
+
+async def test_find_completed_phases() -> None:
+ store = FakeJobStore()
+ j1 = await store.create_job(repository_id="r", phase="parse")
+ await store.update_state(j1.id, JobState.COMPLETED)
+ j2 = await store.create_job(repository_id="r", phase="git")
+ await store.update_state(j2.id, JobState.RUNNING)
+
+ done = await find_completed_phases(store, "r")
+ assert done == {"parse"}
+
+
+def test_default_phases_are_sane() -> None:
+ assert "parse" in DEFAULT_CHECKPOINT_PHASES
+ assert "generation" in DEFAULT_CHECKPOINT_PHASES
+
+
[email protected]("phase", DEFAULT_CHECKPOINT_PHASES)
+def test_default_phase_names_are_strings(phase: str) -> None:
+ assert isinstance(phase, str) and phase
diff --git a/tests/unit/pipeline/test_modes.py b/tests/unit/pipeline/test_modes.py
new file mode 100644
index 000000000..b08aa32fe
--- /dev/null
+++ b/tests/unit/pipeline/test_modes.py
@@ -0,0 +1,22 @@
+"""Unit tests for OrchestratorMode policy."""
+
+from __future__ import annotations
+
+from repowise.core.ingestion.git_indexer import GitIndexTier
+from repowise.core.pipeline.modes import OrchestratorMode
+
+
+def test_standard_mode_full_git_and_docs() -> None:
+ assert OrchestratorMode.STANDARD.git_tier is GitIndexTier.FULL
+ assert OrchestratorMode.STANDARD.allows_doc_generation is True
+
+
+def test_fast_mode_essential_git_no_docs() -> None:
+ assert OrchestratorMode.FAST.git_tier is GitIndexTier.ESSENTIAL
+ assert OrchestratorMode.FAST.allows_doc_generation is False
+
+
+def test_mode_is_str_enum() -> None:
+ # StrEnum members compare equal to their string value — handy for CLI glue.
+ assert OrchestratorMode.FAST == "fast"
+ assert OrchestratorMode("standard") is OrchestratorMode.STANDARD