Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(core): tiered doc generation + graph metrics in SQL for large repos (#220)
Raghav Chamadiya099d2a81d70a
diff --git a/.gitignore b/.gitignoreindex 02eef4b0f..a43db9727 100644--- a/.gitignore+++ b/.gitignore@@ -83,6 +83,9 @@ next-env.d.ts.repowise-workspace/.repowise-workspace.yaml+# Benchmark output (large-repo scale runs) — keep the dir, ignore results+bench/results/+# Database*.db*.db-shmdiff --git a/packages/cli/src/repowise/cli/commands/init_cmd.py b/packages/cli/src/repowise/cli/commands/init_cmd.pyindex dc166c4f2..043e9d17c 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd.py@@ -948,6 +948,17 @@ def _workspace_init(default=False,help="Index files, git history, graph, and dead code — skip LLM page generation.",)+ "--mode",+ "run_mode",+ type=click.Choice(["standard", "fast"]),+ default="standard",+ help=(+ "Pipeline depth. 'fast' indexes graph + essential git only (no per-file "+ "blame, no co-change, no LLM docs) for a quick first pass on very large "+ "repos; backfill the rest later. Default: standard."+ ),+)@click.option("--exclude","-x",@@ -1025,6 +1036,7 @@ def init_command(reasoning: str | None,test_run: bool,index_only: bool,+ run_mode: str,exclude: tuple[str, ...],commit_limit: int | None,follow_renames: bool,@@ -1038,7 +1050,13 @@ def init_command(PATH defaults to the current directory.Use --index-only to run ingestion (AST, graph, git, dead code) without LLM generation.+ Use --mode fast for a quick graph + essential-git index of a very large repo."""+ # --mode fast is a graph + essential-git index with no LLM work, so it+ # implies index-only on the CLI side; the orchestrator mode below switches+ # the git tier to ESSENTIAL.+ if run_mode == "fast":+ index_only = Truefrom repowise.cli.ui import (BRAND,RichProgressCallback,@@ -1047,6 +1065,7 @@ def init_command(build_contextual_next_steps,format_elapsed,interactive_advanced_config,+ interactive_fast_mode_offer,interactive_mode_select,interactive_provider_select,load_dotenv,@@ -1055,6 +1074,7 @@ def init_command(print_phase_header,print_scan_summary,quick_repo_scan,+ should_offer_fast_mode,)start = time.monotonic()@@ -1130,6 +1150,10 @@ def init_command(# ---- Interactive mode (TTY, no explicit flags) ----is_interactive = sys.stdin.isatty() and provider_name is None and not index_only+ # Tiered doc generation cap (set in advanced mode); None = every selected+ # file page is a full-LLM tier-1 page (unchanged behaviour).+ tier1_top_n: int | None = None+# Pre-scan for interactive mode — fast stats to inform choicesscan_info = Noneif is_interactive:@@ -1141,9 +1165,18 @@ def init_command(if mode == "index_only":index_only = True+ # On a large repo, an index-only run is exactly the case where the+ # fast tier (essential git, no blame/co-change) pays off — offer it,+ # defaulting to yes since docs are already opted out.+ if (+ run_mode != "fast"+ and should_offer_fast_mode(scan_info)+ and interactive_fast_mode_offer(console, scan_info, default_fast=True)+ ):+ run_mode = "fast"elif mode == "advanced":provider_name, model = interactive_provider_select(console, model, repo_path=repo_path)- adv = interactive_advanced_config(console, scan=scan_info)+ adv = interactive_advanced_config(console, scan=scan_info, allow_fast=True)commit_limit = adv["commit_limit"]follow_renames = adv["follow_renames"]skip_tests = adv["skip_tests"]@@ -1154,8 +1187,21 @@ def init_command(test_run = adv["test_run"]embedder_name = adv.get("embedder") or embedder_nameinclude_submodules = adv.get("include_submodules", include_submodules)+ run_mode = adv.get("run_mode", run_mode)+ tier1_top_n = adv.get("tier1_top_n")+ if run_mode == "fast":+ index_only = Trueelse:provider_name, model = interactive_provider_select(console, model, repo_path=repo_path)+ # Full mode picked, but on a large repo offer the quick path too.+ # Default no here — the user explicitly asked for docs.+ if (+ run_mode != "fast"+ and should_offer_fast_mode(scan_info)+ and interactive_fast_mode_offer(console, scan_info, default_fast=False)+ ):+ run_mode = "fast"+ index_only = Trueeditor_options = resolve_editor_setup_options(console,@@ -1267,6 +1313,11 @@ def init_command(llm_client = provider if not index_only else decision_providerfrom repowise.core.pipeline import PhaseTimingRecorder, run_pipeline+ from repowise.core.pipeline.modes import OrchestratorMode++ orchestrator_mode = (+ OrchestratorMode.FAST if run_mode == "fast" else OrchestratorMode.STANDARD+ )with Progress(SpinnerColumn(),@@ -1298,6 +1349,7 @@ def init_command(llm_client=llm_client,concurrency=concurrency,test_run=test_run,+ mode=orchestrator_mode,progress=callback,))@@ -1383,6 +1435,7 @@ def init_command(language=language,reasoning=resolved_reasoning,enable_onboarding=onboarding,+ tier1_top_n=tier1_top_n,)if sys.stdin.isatty() and coverage_pct is None and not yes:options = compute_coverage_options(@@ -1792,6 +1845,7 @@ def init_command(next_steps = build_contextual_next_steps(index_only=True,+ fast_mode=(run_mode == "fast"),dead_unreachable=_dc_unreachable,dead_unused=_dc_unused,hotspot_count=_hotspot_count_final,diff --git a/packages/cli/src/repowise/cli/ui.py b/packages/cli/src/repowise/cli/ui.pyindex 01f228294..f1505795a 100644--- a/packages/cli/src/repowise/cli/ui.py+++ b/packages/cli/src/repowise/cli/ui.py@@ -406,6 +406,52 @@ def _ensure_gitignored(repo_path: Path) -> None:# ---------------------------------------------------------------------------+# A repo at or above this many files is "large" — large enough that a quick+# fast-mode first index (graph + essential git, no LLM docs) is worth offering.+LARGE_REPO_FILE_THRESHOLD = 5000+++def should_offer_fast_mode(scan: RepoScanInfo | None) -> bool:+ """Whether to surface the fast-mode offer for this repo.++ Fast mode only makes sense on large repos; small repos run full in seconds+ so the offer would just be noise.+ """+ return scan is not None and scan.total_files > LARGE_REPO_FILE_THRESHOLD+++def interactive_fast_mode_offer(+ console: Console,+ scan: RepoScanInfo | None,+ *,+ default_fast: bool,+) -> bool:+ """Offer fast mode after a large repo is detected. Returns True to use it.++ Shown only when :func:`should_offer_fast_mode` is true. Fast mode is a quick+ first index (dependency graph + essential git history, metrics in SQL) with+ no per-file blame, no co-change walk, and no LLM docs — backfillable later.+ """+ n = scan.total_files if scan else 0+ body = Text()+ body.append(" Large repository detected — ", style="bold")+ body.append(f"{n:,} files.\n\n", style=BRAND_STYLE)+ body.append(" Fast mode runs a quick first index:\n", style="bold")+ body.append(" • dependency graph + essential git history\n")+ body.append(" • graph metrics materialized to SQL\n")+ body.append(" • no per-file blame, no co-change walk, no LLM docs\n\n")+ body.append(" You can backfill full git history and generate docs later.\n", style="dim")+ console.print(+ Panel(+ body,+ title="[bold]Fast first index?[/bold]",+ border_style=BRAND,+ padding=(1, 2),+ )+ )+ return click.confirm(" Use fast mode?", default=default_fast)++def interactive_mode_select(console: Console) -> str:"""Let the user choose full / index-only / advanced.@@ -616,6 +662,8 @@ def _prompt_api_key(def interactive_advanced_config(console: Console,scan: RepoScanInfo | None = None,+ *,+ allow_fast: bool = False,) -> dict[str, Any]:"""Prompt for advanced init options, grouped into logical sections.@@ -667,6 +715,26 @@ def interactive_advanced_config(else:result["include_submodules"] = False+ # ── Run mode (large-repo scale) ───────────────────────────────────────+ # Fast mode = quick graph + essential-git index, no LLM docs. Suggested+ # by default on large repos; off otherwise. Only offered for single-repo+ # init (allow_fast); the workspace path leaves this untouched.+ is_large = bool(scan and scan.total_files > LARGE_REPO_FILE_THRESHOLD)+ if allow_fast:+ console.print()+ console.print(f" [{BRAND}]Run mode[/]")+ console.print(+ " [dim]standard = full depth · fast = quick graph + essential git, "+ "no LLM docs[/dim]"+ )+ result["run_mode"] = click.prompt(+ " Run mode",+ default="fast" if is_large else "standard",+ type=click.Choice(["standard", "fast"]),+ )+ else:+ result["run_mode"] = "standard"+# ── Exclude Patterns ──────────────────────────────────────────────────console.print()console.print(f" [{BRAND}]Exclude Patterns[/]")@@ -766,6 +834,22 @@ def interactive_advanced_config(default=False,)+ # Tiered doc generation: cap the number of full-LLM (tier-1) file pages on+ # large repos. The long tail is rendered from a deterministic template ++ # embedded for search (no LLM). 0 = no cap (every selected page is tier-1).+ # Only meaningful when docs actually generate (standard mode).+ result["tier1_top_n"] = None+ if allow_fast and result["run_mode"] == "standard":+ tier_default = 300 if is_large else 0+ console.print()+ console.print(" [dim]Tiered docs: cap full-LLM file pages; rest are template-only.[/dim]")+ tier_val = click.prompt(+ " Full-LLM file-page cap (tier-1, 0 = no cap)",+ default=tier_default,+ type=int,+ )+ result["tier1_top_n"] = tier_val if tier_val > 0 else None+# ── Summary ───────────────────────────────────────────────────────────console.print()summary = Table(box=None, padding=(0, 2), show_header=False)@@ -780,6 +864,10 @@ def interactive_advanced_config(summary.add_row("Concurrency", str(result["concurrency"]))summary.add_row("Reasoning", result["reasoning"])summary.add_row("Embedder", result["embedder"])+ if allow_fast:+ summary.add_row("Run mode", result["run_mode"])+ if result.get("tier1_top_n"):+ summary.add_row("Full-LLM page cap", str(result["tier1_top_n"]))if patterns:if len(patterns) <= 5:summary.add_row("Exclude", ", ".join(patterns))@@ -939,16 +1027,26 @@ def build_completion_panel(def build_contextual_next_steps(*,index_only: bool,+ fast_mode: bool = False,dead_unreachable: int = 0,dead_unused: int = 0,hotspot_count: int = 0,decision_count: int = 0,top_hotspot: str = "",) -> list[tuple[str, str]]:- """Build next-step suggestions based on what the analysis actually found."""+ """Build next-step suggestions based on what the analysis actually found.++ When *fast_mode* is set, the index used the essential git tier and skipped+ LLM docs, so the suggestions lead with how to upgrade to the full result.+ """steps: list[tuple[str, str]] = []- if index_only:+ if fast_mode:+ # Fast index: graph + essential git, no docs. Tell them how to get the+ # full thing — complete git history + generated wiki pages.+ steps.append(("repowise init", "run full mode: complete git history + generate docs"))+ steps.append(("repowise mcp .", "start MCP server for AI assistants now"))+ elif index_only:steps.append(("repowise mcp .", "start MCP server for AI assistants"))steps.append(("repowise init --provider gemini", "generate full documentation"))else:diff --git a/packages/core/alembic/versions/0021_graph_metrics.py b/packages/core/alembic/versions/0021_graph_metrics.pynew file mode 100644index 000000000..52b650118--- /dev/null+++ b/packages/core/alembic/versions/0021_graph_metrics.py@@ -0,0 +1,57 @@+"""Add graph_metrics table — materialized file-level graph metrics snapshot.++Lets large-repo metric reads (PageRank / betweenness / community / in-out+degree) be served from SQL instead of recomputing the NetworkX kernels.++Revision ID: 0021+Revises: 0020+Create Date: 2026-05-23+"""++from __future__ import annotations++from collections.abc import Sequence++import sqlalchemy as sa+from alembic import op++revision: str = "0021"+down_revision: str | None = "0020"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None+++def upgrade() -> None:+ op.create_table(+ "graph_metrics",+ sa.Column("id", sa.String(32), primary_key=True),+ sa.Column(+ "repository_id",+ sa.String(32),+ sa.ForeignKey("repositories.id", ondelete="CASCADE"),+ nullable=False,+ ),+ sa.Column("node_id", sa.Text, nullable=False),+ sa.Column("pagerank", sa.Float, nullable=False, server_default="0"),+ sa.Column("betweenness", sa.Float, nullable=False, server_default="0"),+ sa.Column("community_id", sa.Integer, nullable=False, server_default="0"),+ sa.Column("in_degree", sa.Integer, nullable=False, server_default="0"),+ sa.Column("out_degree", sa.Integer, nullable=False, server_default="0"),+ sa.Column(+ "created_at",+ sa.DateTime(timezone=True),+ nullable=False,+ server_default=sa.func.now(),+ ),+ sa.UniqueConstraint("repository_id", "node_id", name="uq_graph_metric"),+ )+ op.create_index(+ "ix_graph_metrics_repository_id",+ "graph_metrics",+ ["repository_id"],+ )+++def downgrade() -> None:+ op.drop_index("ix_graph_metrics_repository_id", table_name="graph_metrics")+ op.drop_table("graph_metrics")diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.pyindex 7372f7964..f5a7a06e2 100644--- a/packages/core/src/repowise/core/generation/models.py+++ b/packages/core/src/repowise/core/generation/models.py@@ -133,6 +133,15 @@ class GenerationConfig:jobs_dir: str = ".repowise/jobs"large_file_source_pct: float = 0.4 # use structural summary when source tokens > budget * thislanguage: str = "en"+ # ---- Tiered doc generation (large-repo scale) ---------------------+ # Caps the number of file pages that receive full LLM generation.+ # The top ``tier1_top_n`` selected file pages by PageRank are+ # generated by the LLM (tier-1, the existing path); the remaining+ # selected file pages are rendered from a deterministic Jinja+ # template and embedded for search — with no LLM call (tier-2).+ # ``None`` (default) preserves the prior behaviour exactly: every+ # selected file page is a full tier-1 LLM page.+ tier1_top_n: int | None = Nonedef __post_init__(self) -> None:if self.embed_concurrency is None:@@ -186,7 +195,7 @@ class GeneratedPage:confidence: float = 1.0freshness_status: str = "fresh" # FreshnessStatus literalmetadata: dict[str, object] = field(default_factory=dict)- # 1–3 sentence purpose blurb extracted from the rendered content. Used by+ # 1-3 sentence purpose blurb extracted from the rendered content. Used by# MCP get_context as the default narrative payload (content is gated behind# include=["full_doc"]).summary: str = ""diff --git a/packages/core/src/repowise/core/generation/page_generator.py b/packages/core/src/repowise/core/generation/page_generator.pydeleted file mode 100644index 7007282f2..000000000--- a/packages/core/src/repowise/core/generation/page_generator.py+++ /dev/null@@ -1,1685 +0,0 @@-"""Page generator — converts context dataclasses into GeneratedPage objects.--PageGenerator is the main orchestration layer. It:- 1. Calls ContextAssembler to build template context from ingestion data.- 2. Renders the Jinja2 user-prompt template.- 3. Calls the provider with the rendered prompt + system prompt constant.- 4. Wraps the response in a GeneratedPage.- 5. Manages concurrency (asyncio.Semaphore) and prompt caching (SHA256).--System prompts are module-level constants — the same string per page type on-every call. This enables Anthropic server-side prefix caching.-"""--from __future__ import annotations--import asyncio-import hashlib-import re-import uuid-from collections.abc import Callable-from dataclasses import dataclass-from datetime import UTC, datetime-from pathlib import Path-from typing import Any--import jinja2-import structlog--from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY-from repowise.core.ingestion.models import ParsedFile, RepoStructure-from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse--from . import onboarding as _onboarding-from .context_assembler import ContextAssembler, FilePageContext-from .models import (- GENERATION_LEVELS,- GeneratedPage,- GenerationConfig,- compute_page_id,- compute_source_hash,-)--log = structlog.get_logger(__name__)---@dataclass(frozen=True)-class PriorPage:- """Snapshot of a previously-generated page used for cross-run reuse.-- Lives in :class:`PageGenerator` keyed by ``page_id``. When the freshly- rendered prompt produces a matching ``source_hash`` under the same- ``model_name``, the LLM call is skipped and ``content`` is reused.- """-- source_hash: str- model_name: str- content: str- input_tokens: int = 0- output_tokens: int = 0- cached_tokens: int = 0--_LANGUAGE_NAMES = {- "en": "English",- "ru": "Russian",- "es": "Spanish",- "fr": "French",- "de": "German",- "zh": "Chinese",- "ja": "Japanese",- "ko": "Korean",- "it": "Italian",- "pt": "Portuguese",- "nl": "Dutch",- "pl": "Polish",- "tr": "Turkish",- "ar": "Arabic",- "hi": "Hindi",-}--# ----------------------------------------------------------------------------# System prompts — one per page type (constant strings for prefix caching)-# -----------------------------------------------------------------------------SYSTEM_PROMPTS: dict[str, str] = {- "file_page": (- "You are repowise, an expert technical documentation generator. "- "Your task is to produce comprehensive, accurate wiki pages from source code. "- "Output markdown only. Do not include preamble or apologies. "- "\n"- "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "- "state this file's job in one line. Use concrete architectural vocabulary "- "(orchestrator, entry point, parser, adapter, dispatcher, factory, resolver, "- "persistence layer, indexer, analyzer, etc.) and name what it produces or "- "consumes. This sentence is what semantic search and grep both anchor on, "- "so role keywords must appear at the very front — not buried in paragraph 3. "- "Bad: 'This file contains the X class which is used by Y.' "- "Good: 'X is the orchestrator for the indexing pipeline — it sequences "- "traversal, parsing, graph analysis, git enrichment, and persistence.' "- "If the user prompt lists Architectural signals (entry point, high PageRank, "- "bridge), weave them into that first sentence rather than restating them. "- "Required sections: ## Overview, ## Public API, ## Dependencies, ## Usage Notes."- ),- "symbol_spotlight": (- "You are repowise, an expert technical documentation generator. "- "Write a detailed spotlight page for a single code symbol. "- "Output markdown only. "- "Required sections: ## Purpose, ## Signature, ## Parameters, ## Returns, ## Example Usage."- ),- "module_page": (- "You are repowise, an expert technical documentation generator. "- "Write a module-level overview page summarising all files in the module. "- "Output markdown only. "- "\n"- "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "- "state the module's job in the larger system. Use architectural vocabulary "- "(ingestion subsystem, generation pipeline, persistence layer, transport "- "adapter, etc.) and name the inputs it consumes and the outputs it produces. "- "Bad: 'The X module contains 15 files responsible for...' "- "Good: 'The ingestion module is the entry stage of repowise's indexing "- "pipeline — it traverses a repository, parses files into ASTs, extracts "- "symbols, and yields ParsedFile objects for downstream analysis.' "- "Required sections: ## Overview, ## Public API Summary, ## Architecture Notes."- ),- "scc_page": (- "You are repowise, an expert technical documentation generator. "- "Document this circular dependency cycle and provide actionable refactoring advice. "- "Output markdown only. "- "Required sections: ## Cycle Description, ## Files Involved, ## Why This Exists, "- "## Refactoring Suggestions."- ),- "repo_overview": (- "You are repowise, an expert technical documentation generator. "- "Write a high-level repository overview suitable for onboarding new developers. "- "Output markdown only. "- "\n"- "CRITICAL — lead-with-purpose rule: the FIRST sentence under ## Project "- "Summary must answer 'what does this repository do, end-to-end?' in one "- "concrete sentence. Name the inputs, the pipeline, and the outputs in "- "architectural vocabulary. "- "Bad: 'This repository implements a documentation tool with 15 packages.' "- "Good: 'Repowise is a codebase documentation engine: it indexes a repository "- "by traversing files, parsing code into ASTs, analyzing dependencies, and "- "generating LLM-synthesised wiki pages served via MCP and a web UI.' "- "Required sections: ## Project Summary, ## Technology Stack, ## Entry Points, ## Architecture."- ),- "architecture_diagram": (- "You are repowise, an expert technical documentation generator. "- "Generate an architecture overview with a Mermaid diagram. "- "You MUST include a fenced mermaid block with graph TD showing key dependencies. "- "Output markdown only."- ),- "api_contract": (- "You are repowise, an expert technical documentation generator. "- "Document this API contract file for developers integrating with the service. "- "Output markdown only. "- "Required sections: ## Overview, ## Endpoints, ## Schemas, ## Authentication, ## Examples."- ),- "infra_page": (- "You are repowise, an expert technical documentation generator. "- "Document this infrastructure file for DevOps and platform engineers. "- "Output markdown only. "- "Required sections: ## Purpose, ## Key Targets/Stages, ## Configuration, ## Operational Notes."- ),- "onboarding": (- "You are repowise, an expert technical documentation generator producing "- "a single page in a curated Onboarding collection that a new contributor "- "or LLM agent reads first. "- "Write concise, navigable prose grounded in the structured signals supplied. "- "Do not invent file paths, symbol names, or rationale that is not in the context. "- "Output markdown only — follow the exact section structure the user prompt prescribes."- ),-}--_INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()-_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})-_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()---def _now_iso() -> str:- """Return current UTC time as ISO-8601 string."""- return datetime.now(UTC).isoformat()---class PageGenerator:- """Generate wiki pages by rendering prompts and calling an LLM provider.-- Args:- provider: Any BaseProvider implementation.- assembler: ContextAssembler instance.- config: GenerationConfig controlling budget, concurrency, caching.- jinja_env: Optional Jinja2 Environment (defaults to FileSystemLoader- pointing at the templates/ directory next to this file).- """-- def __init__(- self,- provider: BaseProvider,- assembler: ContextAssembler,- config: GenerationConfig,- jinja_env: jinja2.Environment | None = None,- vector_store: Any | None = None,- language: str = "en",- prior_pages: dict[str, "PriorPage"] | None = None,- ) -> None:- self._provider = provider- self._assembler = assembler- self._config = config- self._vector_store = vector_store- self._language = language- self._cache: dict[str, GeneratedResponse] = {}- # Map of page_id → PriorPage from previous generation runs. When the- # rendered prompt's source_hash matches the prior page's hash AND the- # model is the same, the LLM call is skipped and the prior content is- # reused. Wired by the orchestrator from the persisted wiki_pages- # table.- self._prior_pages: dict[str, PriorPage] = prior_pages or {}- self._reuse_count: int = 0-- if jinja_env is None:- templates_dir = Path(__file__).parent / "templates"- loader = jinja2.FileSystemLoader(str(templates_dir))- jinja_env = jinja2.Environment(- loader=loader,- undefined=jinja2.StrictUndefined,- autoescape=False,- )- self._jinja_env = jinja_env-- # ------------------------------------------------------------------- # Per-type generation methods- # -------------------------------------------------------------------- async def generate_file_page(- self,- parsed: ParsedFile,- graph: Any,- pagerank: dict[str, float],- betweenness: dict[str, float],- community: dict[str, int],- source_bytes: bytes,- ) -> GeneratedPage:- ctx = self._assembler.assemble_file_page(- parsed, graph, pagerank, betweenness, community, source_bytes- )- user_prompt = self._render("file_page.j2", ctx=ctx)- response = await self._call_provider(- "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path- )- return self._build_generated_page(- "file_page",- parsed.file_info.path,- f"File: {parsed.file_info.path}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["file_page"],- )-- async def generate_symbol_spotlight(- self,- symbol: Any,- parsed: ParsedFile,- pagerank: dict[str, float],- graph: Any,- source_map: dict[str, bytes] | None = None,- ) -> GeneratedPage:- ctx = self._assembler.assemble_symbol_spotlight(- symbol,- parsed,- pagerank,- graph,- source_bytes=(source_map or {}).get(parsed.file_info.path, b""),- )- user_prompt = self._render("symbol_spotlight.j2", ctx=ctx)- target = f"{parsed.file_info.path}::{symbol.name}"- response = await self._call_provider(- "symbol_spotlight", user_prompt, str(uuid.uuid4()), target_path=target- )- return self._build_generated_page(- "symbol_spotlight",- f"{parsed.file_info.path}::{symbol.name}",- f"Symbol: {symbol.qualified_name}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["symbol_spotlight"],- )-- async def generate_module_page(- self,- module_path: str,- language: str,- file_contexts: list[FilePageContext],- graph: Any,- git_meta_map: dict[str, dict] | None = None,- page_summaries: dict[str, str] | None = None,- decision_records: list[dict] | None = None,- dead_code_findings: list[dict] | None = None,- external_systems: list[dict] | None = None,- community_label: str | None = None,- community_cohesion: float | None = None,- target_path: str | None = None,- ) -> GeneratedPage:- ctx = self._assembler.assemble_module_page(- module_path,- language,- file_contexts,- graph,- page_summaries=page_summaries,- git_meta_map=git_meta_map,- decision_records=decision_records,- dead_code_findings=dead_code_findings,- external_systems=external_systems,- community_label=community_label,- community_cohesion=community_cohesion,- )- module_git_summary = None- if git_meta_map:- from collections import Counter-- file_paths = [fc.file_path for fc in file_contexts]- metas = [git_meta_map[f] for f in file_paths if f in git_meta_map]- if metas:- owner_counts = Counter(- m.get("primary_owner_name") for m in metas if m.get("primary_owner_name")- )- most_active = max(metas, key=lambda m: m.get("commit_count_90d", 0))- module_git_summary = {- "top_owners": [- {"name": n, "file_count": c} for n, c in owner_counts.most_common(3)- ],- "most_active_file": most_active.get("file_path", ""),- "most_active_commits_90d": most_active.get("commit_count_90d", 0),- }- user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)- page_target = target_path or module_path- response = await self._call_provider(- "module_page", user_prompt, str(uuid.uuid4()), target_path=page_target- )- return self._build_generated_page(- "module_page",- page_target,- f"Module: {module_path}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["module_page"],- )-- async def generate_scc_page(- self,- scc_id: str,- scc_files: list[str],- file_contexts: list[FilePageContext],- ) -> GeneratedPage:- ctx = self._assembler.assemble_scc_page(scc_id, scc_files, file_contexts)- user_prompt = self._render("scc_page.j2", ctx=ctx)- response = await self._call_provider(- "scc_page", user_prompt, str(uuid.uuid4()), target_path=scc_id- )- return self._build_generated_page(- "scc_page",- scc_id,- f"Circular Dependency: {scc_id}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["scc_page"],- )-- async def generate_repo_overview(- self,- repo_structure: RepoStructure,- pagerank: dict[str, float],- sccs: list[Any],- community: dict[str, int],- git_meta_map: dict[str, dict] | None = None,- graph_builder: Any | None = None,- repo_name: str | None = None,- external_systems: list[dict] | None = None,- decision_records: list[dict] | None = None,- ) -> GeneratedPage:- ctx = self._assembler.assemble_repo_overview(- repo_structure,- pagerank,- sccs,- community,- graph_builder=graph_builder,- external_systems=external_systems,- decision_records=decision_records,- )- repo_git_summary = None- if git_meta_map:- metas = list(git_meta_map.values())- top_churn = sorted(metas, key=lambda m: m.get("commit_count_90d", 0), reverse=True)[:3]- oldest = min(- (m for m in metas if m.get("first_commit_at")),- key=lambda m: m["first_commit_at"],- default=None,- )- repo_git_summary = {- "hotspot_count": sum(1 for m in metas if m.get("is_hotspot")),- "stable_count": sum(1 for m in metas if m.get("is_stable")),- "top_churn_files": [m.get("file_path", "") for m in top_churn],- "oldest_file": oldest.get("file_path", "") if oldest else "",- "oldest_file_age_days": oldest.get("age_days", 0) if oldest else 0,- }- if not repo_name:- repo_name = getattr(repo_structure, "name", None) or "repo"- user_prompt = self._render("repo_overview.j2", ctx=ctx, repo_git_summary=repo_git_summary)- response = await self._call_provider(- "repo_overview", user_prompt, str(uuid.uuid4()), target_path=repo_name- )- return self._build_generated_page(- "repo_overview",- repo_name,- f"Repository Overview: {repo_name}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["repo_overview"],- )-- async def generate_architecture_diagram(- self,- graph: Any,- pagerank: dict[str, float],- community: dict[str, int],- sccs: list[Any],- repo_name: str,- ) -> GeneratedPage:- ctx = self._assembler.assemble_architecture_diagram(- graph, pagerank, community, sccs, repo_name- )- user_prompt = self._render("architecture_diagram.j2", ctx=ctx)- response = await self._call_provider(- "architecture_diagram", user_prompt, str(uuid.uuid4()), target_path=repo_name- )- return self._build_generated_page(- "architecture_diagram",- repo_name,- f"Architecture Diagram: {repo_name}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["architecture_diagram"],- )-- async def generate_api_contract(- self,- parsed: ParsedFile,- source_bytes: bytes,- ) -> GeneratedPage:- ctx = self._assembler.assemble_api_contract(parsed, source_bytes)- user_prompt = self._render("api_contract.j2", ctx=ctx)- response = await self._call_provider(- "api_contract", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path- )- return self._build_generated_page(- "api_contract",- parsed.file_info.path,- f"API Contract: {parsed.file_info.path}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["api_contract"],- )-- async def generate_onboarding_page(- self,- spec: "_onboarding.SubkindSpec",- signals: "_onboarding.OnboardingSignals",- ) -> GeneratedPage | None:- """Generate one onboarding page from a registered subkind spec.-- Returns ``None`` when the subkind's gate fails (``build_context``- returned ``None``) — the slot is silently skipped for this repo.- """- ctx = spec.build_context(signals)- if ctx is None:- log.debug("onboarding.gate_skipped", slot=spec.slot)- return None-- template_name = f"onboarding/{spec.template}"- user_prompt = self._render(template_name, ctx=ctx, slot=spec.slot)- target = _onboarding.target_path(spec.slot)- response = await self._call_provider(- "onboarding", user_prompt, str(uuid.uuid4()), target_path=target- )- page = self._build_generated_page(- "onboarding",- target,- spec.title,- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["onboarding"],- )- # Subkind discriminator lives in metadata; page_type alone is shared- # across all six generated onboarding slots.- page.metadata["subkind"] = spec.slot- page.metadata["onboarding_slot"] = spec.slot- return page-- @staticmethod- def _tag_promoted_pages(pages: list[GeneratedPage]) -> None:- """Tag repo_overview / architecture_diagram pages with their slot.-- Mutates each matching page's ``metadata["onboarding_slot"]`` so the- UI groups them into the Onboarding folder without changing their- underlying ``page_type``. Idempotent and tolerant of missing pages.- """- for page in pages:- slot = _onboarding.PROMOTED_SLOTS.get(page.page_type)- if slot is not None:- page.metadata["onboarding_slot"] = slot-- async def generate_infra_page(- self,- parsed: ParsedFile,- source_bytes: bytes,- ) -> GeneratedPage:- ctx = self._assembler.assemble_infra_page(parsed, source_bytes)- user_prompt = self._render("infra_page.j2", ctx=ctx)- response = await self._call_provider(- "infra_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path- )- return self._build_generated_page(- "infra_page",- parsed.file_info.path,- f"Infrastructure: {parsed.file_info.path}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["infra_page"],- )-- # ------------------------------------------------------------------- # generate_all — orchestration- # -------------------------------------------------------------------- async def generate_all(- self,- parsed_files: list[ParsedFile],- source_map: dict[str, bytes],- graph_builder: Any, # GraphBuilder- repo_structure: RepoStructure,- repo_name: str,- job_system: Any | None = None, # JobSystem | None- on_page_done: Callable[[str], None] | None = None,- on_total_known: Callable[[int], None] | None = None,- on_subphase: Callable[[str, int | None], None] | None = None,- git_meta_map: dict[str, dict] | None = None,- resume: bool = False,- repo_path: Path | str | None = None,- dead_code_report: Any | None = None,- decision_report: Any | None = None,- external_systems: list[dict] | None = None,- ) -> list[GeneratedPage]:- """Generate all wiki pages for a repository.-- Runs generation in 8 ordered levels. Each level's pages are generated- concurrently (up to config.max_concurrency). Failures within a level- are logged but do not abort the remaining levels.-- Args:- parsed_files: All ParsedFile objects from the ingestion pipeline.- source_map: Raw file bytes keyed by relative path.- graph_builder: Finalized GraphBuilder (build() already called).- repo_structure: High-level repo metadata.- repo_name: Human-readable repository name.- job_system: Optional JobSystem for checkpoint persistence.- on_page_done: Optional callback per completed page.- git_meta_map: Optional dict of git metadata by file path.-- Returns:- List of GeneratedPage objects in level order.- """- graph = graph_builder.graph()- pagerank = graph_builder.pagerank()- betweenness = graph_builder.betweenness_centrality()- community = graph_builder.community_detection()- sccs = graph_builder.strongly_connected_components()-- # ---- Build per-file lookup maps from phase-2 signals ----- dead_code_by_file: dict[str, list[dict]] = {}- if dead_code_report is not None and getattr(dead_code_report, "findings", None):- for f in dead_code_report.findings:- dead_code_by_file.setdefault(f.file_path, []).append(- {- "symbol_name": f.symbol_name,- "symbol_kind": f.symbol_kind,- "kind": str(f.kind),- "reason": f.reason,- "confidence": f.confidence,- "safe_to_delete": f.safe_to_delete,- }- )-- decisions_by_file: dict[str, list[dict]] = {}- decisions_all: list[dict] = []- if decision_report is not None and getattr(decision_report, "decisions", None):- for d in decision_report.decisions:- payload = {- "title": d.title,- "decision": d.decision,- "rationale": d.rationale,- "source": d.source,- "confidence": d.confidence,- "evidence_file": d.evidence_file,- }- decisions_all.append(payload)- for fp in d.affected_files or []:- decisions_by_file.setdefault(fp, []).append(payload)-- external_systems = external_systems or []-- all_pages: list[GeneratedPage] = []- semaphore = asyncio.Semaphore(self._config.max_concurrency)- embed_semaphore = asyncio.Semaphore(self._config.embed_concurrency or 1)- # Summaries of completed pages: target_path → brief summary text (for dep context)- completed_page_summaries: dict[str, str] = {}-- def _extract_summary(content: str) -> str:- if "## Overview" in content:- start = content.index("## Overview") + len("## Overview")- end = content.find("\n##", start)- return content[start : end if end > 0 else start + 1600].strip()[:400]- return content[:400]-- # Determine already-completed pages (for resume support)- completed_ids: set[str] = set()- job_id: str | None = None- if job_system is not None:- repo_path_str = (- str(Path(repo_path).resolve())- if repo_path- else str(getattr(repo_structure, "root_path", "."))- )- # On resume, query the vector store directly — it is the ground truth- if resume and self._vector_store is not None:- completed_ids = await self._vector_store.list_page_ids()- if completed_ids:- log.info(- "Resuming generation from vector store",- already_completed=len(completed_ids),- )- job_id = job_system.create_job(- repo_path_str,- self._config,- self._provider.provider_name,- self._provider.model_name,- )-- async def _embed_async(page: GeneratedPage) -> None:- """Embed a finished page into the vector store. Safe to fire-and-forget.-- Errors are swallowed at debug level — embedding is a RAG- enhancement, not load-bearing for the page itself.- """- try:- summary = _extract_summary(page.content)- async with embed_semaphore:- await self._vector_store.embed_and_upsert(- page.page_id,- page.content,- {- "page_type": page.page_type,- "target_path": page.target_path,- "content": page.content[:600],- "summary": summary,- },- )- except Exception as e:- log.debug("rag.embed_failed", page_id=page.page_id, error=str(e))-- async def run_level(named_coros: list[tuple[str, Any]], level: int) -> list[GeneratedPage]:- if job_system is not None and job_id is not None:- job_system.update_level(job_id, level)-- # Embed tasks spawned during this level. Drained after the- # gather() so the next level's RAG search sees a fully-indexed- # store, but never blocks the per-page wave.- pending_embeds: list[asyncio.Task[None]] = []-- async def guarded_named(page_id: str, coro: Any) -> Any:- try:- async with semaphore:- result = await coro-- if isinstance(result, GeneratedPage):- # Summary capture is cheap (string ops) — keep- # inline so the next page's context assembly sees- # it immediately.- completed_page_summaries[result.target_path] = _extract_summary(- result.content- )- # Progress tick fires the moment the LLM call- # returns — the user sees the page as done before- # the embed completes.- if on_page_done is not None:- on_page_done(result.page_type)- # Fire-and-forget the embed/upsert. Removes ~1- # embedder round-trip from this task's critical- # path so the LLM slot frees up immediately.- if self._vector_store is not None:- pending_embeds.append(- asyncio.create_task(_embed_async(result))- )- return result- except Exception as exc:- if job_system is not None and job_id is not None:- job_system.fail_page(job_id, page_id, str(exc))- log.error(- "page_generation_failed",- page_id=page_id,- level=level,- error=str(exc),- )- return exc # return as value so gather works-- tasks = [guarded_named(pid, c) for pid, c in named_coros]- results = await asyncio.gather(*tasks)- # Drain pending embeds before declaring the level done — the- # next level's RAG search depends on these landing in the store.- if pending_embeds:- await asyncio.gather(*pending_embeds, return_exceptions=True)- pages = [r for r in results if isinstance(r, GeneratedPage)]- if job_system is not None and job_id is not None:- for r in pages:- job_system.complete_page(job_id, r.page_id)- return pages-- # ---- Page selection (single source of truth) ----- # The selection subsystem scores every candidate, allocates the- # global budget across page-type buckets, and returns an- # allow-set. Each level's emit loop iterates only over members- # of that allow-set — there are no bypass paths. The same- # function is called by the cost estimator so the pre-run- # estimate cannot drift from the actual run.- code_files = [- p- for p in parsed_files- if not p.file_info.is_api_contract- and not _is_infra_file(p)- and p.file_info.language in _CODE_LANGUAGES- ]-- # Near-clone dedupe runs before scoring so clone losers never- # consume scoring budget. Entry points are never dropped.- if getattr(self._config, "dedupe_near_clones", True):- drop_paths = _select_clone_representatives(code_files, pagerank)- if drop_paths:- log.info("page_selection.clone_dedupe", dropped=len(drop_paths))- code_files = [p for p in code_files if p.file_info.path not in drop_paths]- parsed_files_for_selection = [- p for p in parsed_files if p.file_info.path not in drop_paths- ]- else:- parsed_files_for_selection = parsed_files- else:- parsed_files_for_selection = parsed_files-- try:- community_info_map = graph_builder.community_info() or {}- except Exception:- community_info_map = {}-- from .selection import SelectionInputs, select_pages-- selection = select_pages(- SelectionInputs(- parsed_files=parsed_files_for_selection,- pagerank=pagerank,- betweenness=betweenness,- community=community,- community_info=community_info_map,- sccs=list(sccs),- git_meta_map=git_meta_map,- config=self._config,- )- )-- # Bucket allow-sets — O(1) membership checks in the level loops.- sel_file_paths: set[str] = set(selection.file_page_paths)- sel_symbol_keys: set[tuple[str, str]] = set(selection.symbol_spotlights)- sel_api_paths: set[str] = set(selection.api_contract_paths)- sel_infra_paths: set[str] = set(selection.infra_paths)- sel_module_groups = list(selection.module_groups)- sel_scc_groups = list(selection.scc_groups)-- # Sort code_files for stable level-2 ordering: selected files- # first (so dep summaries land in the store earliest), then by- # PageRank desc. The topo-sort below further refines this.- code_files = sorted(- code_files,- key=lambda p: (- p.file_info.path not in sel_file_paths,- not p.file_info.is_entry_point,- -pagerank.get(p.file_info.path, 0.0),- ),- )-- estimated_total = (- selection.counts()["api_contract"]- + selection.counts()["symbol_spotlight"]- + selection.counts()["file_page"]- + selection.counts()["scc_page"]- + selection.counts()["module_page"]- + int(selection.emit_repo_overview)- + int(selection.emit_arch_diagram)- + selection.counts()["infra_page"]- )- remaining_total = max(0, estimated_total - len(completed_ids))- if on_total_known is not None:- on_total_known(remaining_total)- if job_system is not None and job_id is not None:- job_system.start_job(job_id, estimated_total)-- # ---- Level 0: api_contract (allow-set filtered) ----- api_files = [- p- for p in parsed_files- if p.file_info.is_api_contract and p.file_info.path in sel_api_paths- ]- level0_coros = [- (- compute_page_id("api_contract", p.file_info.path),- self.generate_api_contract(p, source_map.get(p.file_info.path, b"")),- )- for p in api_files- if compute_page_id("api_contract", p.file_info.path) not in completed_ids- ]- # ---- Level 1: symbol_spotlight (allow-set filtered) ----- # The selection layer already picked the top symbols by score;- # here we just resolve them back to (Symbol, ParsedFile) pairs.- parsed_by_path: dict[str, ParsedFile] = {- p.file_info.path: p for p in parsed_files- }- top_symbols: list[tuple[Any, ParsedFile]] = []- for file_path, sym_name in selection.symbol_spotlights:- pf = parsed_by_path.get(file_path)- if pf is None:- continue- sym = next((s for s in pf.symbols if s.name == sym_name), None)- if sym is not None:- top_symbols.append((sym, pf))-- level1_coros = [- (- compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}"),- self.generate_symbol_spotlight(sym, pf, pagerank, graph, source_map=source_map),- )- for sym, pf in top_symbols- if compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}")- not in completed_ids- ]-- # Levels 0 (api_contract) and 1 (symbol_spotlight) share no- # data dependencies — both feed into nothing else upstream of- # Level 2 — so they run in one merged batch instead of two- # sequential barriers. ``run_level`` already bounds total- # concurrency via ``self._config.max_concurrency`` so the merge- # only removes idle slots, never over-saturates the provider.- level01_pages = await run_level(level0_coros + level1_coros, 1)- all_pages.extend(level01_pages)-- # ---- Level 2: file_page (significant code files only) ----- # Context is assembled for ALL code files (module pages need it).- # Pages are generated only for files that cross the significance bar.- # page_summaries from level 0+1 are available here (B2).- #- # Topo-sort: process leaves (no internal out-edges) before roots so that- # dependency summaries are available when assembling dependents' contexts.- # Falls back to existing priority order if networkx is unavailable or graph- # has cycles.- code_file_paths = [p.file_info.path for p in code_files]- try:- import networkx as nx # type: ignore[import]-- # Build a subgraph of just the code files we are about to generate- code_file_set = set(code_file_paths)- dag = nx.DiGraph()- dag.add_nodes_from(code_file_paths)- for path_ in code_file_paths:- if path_ in graph:- for succ in graph.successors(path_):- if succ in code_file_set:- dag.add_edge(path_, succ) # path_ depends on succ-- if nx.is_directed_acyclic_graph(dag):- # topological_sort yields nodes in an order where for each edge u→v,- # u comes before v — i.e. dependents before dependencies.- # We want leaves (dependencies) first, so reverse the order.- topo_order = list(reversed(list(nx.topological_sort(dag))))- else:- # Cycle present: condense SCCs, topo-sort condensation, then expand.- condensation = nx.condensation(dag)- topo_order_scc = list(reversed(list(nx.topological_sort(condensation))))- scc_members: dict[int, list[str]] = {- n: list(condensation.nodes[n]["members"]) for n in condensation.nodes- }- topo_order = [node for scc_id in topo_order_scc for node in scc_members[scc_id]]-- # Preserve priority ordering within the topo-sort by mapping paths to- # their original priority index.- priority_index = {p: i for i, p in enumerate(code_file_paths)}- topo_order = [p for p in topo_order if p in priority_index]- # Re-sort code_files to match topo_order- path_to_parsed = {p.file_info.path: p for p in code_files}- code_files = [path_to_parsed[p] for p in topo_order if p in path_to_parsed]- except Exception:- pass # Keep existing priority order on any failure-- file_page_contexts: dict[str, FilePageContext] = {}-- # Batch-prefetch dependency summaries from the vector store in a- # SINGLE call covering every code file's dependencies — replaces- # the prior per-file serial loop that turned N×M awaits into a- # measurable bottleneck on the level-2 critical path.- if self._vector_store is not None:- needed_deps: set[str] = set()- for p in code_files:- path_ = p.file_info.path- if path_ not in graph:- continue- for dep in graph.successors(path_):- if dep.startswith("external:"):- continue- if dep in completed_page_summaries:- continue- needed_deps.add(dep)- if needed_deps:- try:- batch = await self._vector_store.get_page_summaries_by_paths(- list(needed_deps)- )- for dep_path, payload in batch.items():- summary = payload.get("summary") if payload else None- if summary:- completed_page_summaries[dep_path] = summary- except Exception as exc:- log.debug("rag.batch_dep_prefetch_failed", error=str(exc))-- level2_coros: list[tuple[str, Any]] = []- for p in code_files:- ctx = self._assembler.assemble_file_page(- p,- graph,- pagerank,- betweenness,- community,- source_map.get(p.file_info.path, b""),- git_meta=git_meta_map.get(p.file_info.path) if git_meta_map else None,- page_summaries=completed_page_summaries,- dead_code_findings=dead_code_by_file.get(p.file_info.path),- decision_records=decisions_by_file.get(p.file_info.path),- )- file_page_contexts[p.file_info.path] = ctx- pid = compute_page_id("file_page", p.file_info.path)- if p.file_info.path in sel_file_paths and pid not in completed_ids:- level2_coros.append((pid, self._generate_file_page_from_ctx(p, ctx)))-- level2_pages = await run_level(level2_coros, 2)- all_pages.extend(level2_pages)-- # ---- Level 3: scc_page (allow-set filtered) ----- scc_coros: list[tuple[str, Any]] = []- for scc_id, scc_files in sel_scc_groups:- fc_list = [file_page_contexts[f] for f in scc_files if f in file_page_contexts]- pid = compute_page_id("scc_page", scc_id)- if pid not in completed_ids:- scc_coros.append((pid, self.generate_scc_page(scc_id, scc_files, fc_list)))- level3_pages = await run_level(scc_coros, 3)- all_pages.extend(level3_pages)-- # ---- Level 4: module_page (allow-set filtered) ----- # Module groups come from the selection layer; we only need to- # resolve FilePageContext objects for the files we already- # built contexts for in Level 2.- level4_coros: list[tuple[str, Any]] = []- for mg in sel_module_groups:- fcs = [- file_page_contexts[fp]- for fp in mg.file_paths- if fp in file_page_contexts- ]- if not fcs:- continue- page_id = compute_page_id("module_page", mg.key)- if page_id in completed_ids:- continue- level4_coros.append(- (- page_id,- self.generate_module_page(- mg.display,- mg.language,- fcs,- graph,- git_meta_map=git_meta_map,- page_summaries=completed_page_summaries,- decision_records=decisions_all,- dead_code_findings=[- d for fc in fcs for d in dead_code_by_file.get(fc.file_path, [])- ],- external_systems=external_systems,- community_label=mg.label,- community_cohesion=mg.cohesion,- target_path=mg.key,- ),- )- )- level4_pages = await run_level(level4_coros, 4)- all_pages.extend(level4_pages)-- # ---- Level 6: repo_overview + architecture_diagram ----- level6_coros: list[tuple[str, Any]] = []- if compute_page_id("repo_overview", repo_name) not in completed_ids:- level6_coros.append(- (- compute_page_id("repo_overview", repo_name),- self.generate_repo_overview(- repo_structure,- pagerank,- sccs,- community,- git_meta_map=git_meta_map,- graph_builder=graph_builder,- repo_name=repo_name,- external_systems=external_systems,- decision_records=decisions_all[:10],- ),- )- )- if compute_page_id("architecture_diagram", repo_name) not in completed_ids:- level6_coros.append(- (- compute_page_id("architecture_diagram", repo_name),- self.generate_architecture_diagram(graph, pagerank, community, sccs, repo_name),- )- )- # ---- Level 7: infra_page (allow-set filtered) ----- infra_files = [- p- for p in parsed_files- if _is_infra_file(p) and p.file_info.path in sel_infra_paths- ]- level7_coros: list[tuple[str, Any]] = [- (- compute_page_id("infra_page", p.file_info.path),- self.generate_infra_page(p, source_map.get(p.file_info.path, b"")),- )- for p in infra_files- if compute_page_id("infra_page", p.file_info.path) not in completed_ids- ]-- # ---- Level 8: onboarding (curated collection) ----- # Each subkind defines its own gate inside build_context — slots- # whose gates fail return None and are skipped entirely. Promoted- # slots (project_overview / architecture_guide) are tagged onto the- # level-6 pages and don't appear here.- level8_coros: list[tuple[str, Any]] = []- if getattr(self._config, "enable_onboarding", True):- specs = _onboarding.iter_specs()- if specs:- if on_subphase is not None:- try:- on_subphase("onboarding", len(specs))- except Exception:- pass- signals = _onboarding.OnboardingSignals(- repo_name=repo_name,- repo_structure=repo_structure,- parsed_files=tuple(parsed_files),- source_map=source_map,- graph_builder=graph_builder,- pagerank=pagerank,- betweenness=betweenness,- community=community,- sccs=tuple(sccs),- git_meta_map=git_meta_map,- dead_code_by_file=dead_code_by_file,- decisions_all=tuple(decisions_all),- external_systems=tuple(external_systems),- completed_page_summaries=dict(completed_page_summaries),- )- for spec in specs:- page_id = compute_page_id(- "onboarding", _onboarding.target_path(spec.slot)- )- if page_id in completed_ids:- continue- level8_coros.append(- (page_id, self.generate_onboarding_page(spec, signals))- )-- # Levels 6, 7, and 8 share no data dependencies with each other- # — Level 6 needs only the graph + repo metadata; Level 7- # documents standalone infra files; Level 8 consumes a frozen- # ``OnboardingSignals`` snapshot. Run them in a single merged- # batch instead of three sequential barriers.- final_pages = await run_level(level6_coros + level7_coros + level8_coros, 8)- # Tag promoted onboarding slots (repo_overview / architecture_diagram)- # so the UI groups them into the Onboarding folder. No content change.- self._tag_promoted_pages(final_pages)- all_pages.extend(final_pages)-- # Post-generation: resolve backtick-quoted refs in every page's- # markdown to other pages' ``page_id``s and stash the result in- # ``metadata["wiki_links"]``. The reverse index lands in- # ``metadata["backlinks"]``. Pure regex + dict lookup — no LLM- # call, safe to run on every generation.- try:- from .interlinking import attach_wiki_links_and_backlinks-- attach_wiki_links_and_backlinks(all_pages, parsed_files)- except Exception as exc:- log.debug("interlinking.failed", error=str(exc))-- # Finalize job- if job_system is not None and job_id is not None:- job_system.complete_job(job_id)-- log.info(- "Generation complete",- total_pages=len(all_pages),- provider=self._provider.provider_name,- model=self._provider.model_name,- )- return all_pages-- # ------------------------------------------------------------------- # Internal helpers- # -------------------------------------------------------------------- async def _generate_file_page_from_ctx(- self,- parsed: ParsedFile,- ctx: FilePageContext,- ) -> GeneratedPage:- """Generate a file_page from a pre-assembled context (avoids double-assembly)."""- # RAG context: query vector store for related pages (B1).- # Gated by two short-circuits so we don't burn an embedder- # round-trip on every page when the result wouldn't help:- # 1. ``enable_rag_context`` config flag (off → fully skip).- # 2. ``rag_min_store_size`` — early pages run against an empty- # or near-empty store and the search returns nothing useful.- if self._vector_store is not None and getattr(- self._config, "enable_rag_context", True- ):- min_store_size = max(0, int(getattr(self._config, "rag_min_store_size", 10) or 0))- store_ok = True- if min_store_size > 0:- try:- current_ids = await self._vector_store.list_page_ids()- store_ok = len(current_ids) >= min_store_size- except Exception:- # If the store can't be sized cheaply, fall through to- # the search — it'll either succeed or hit the- # existing exception path.- store_ok = True- if store_ok:- query_terms = parsed.exports or [- s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"- ]- if query_terms:- try:- results = await self._vector_store.search(- ", ".join(query_terms[:5]), limit=3- )- self_id = f"file_page:{parsed.file_info.path}"- ctx.rag_context = [- f"[{r.page_id}]\n{r.snippet}"- for r in results- if r.page_id != self_id- ]- except Exception as e:- log.debug(- "rag.search_failed", path=parsed.file_info.path, error=str(e)- )- user_prompt = self._render("file_page.j2", ctx=ctx)- response = await self._call_provider(- "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path- )- page = self._build_generated_page(- "file_page",- parsed.file_info.path,- f"File: {parsed.file_info.path}",- response,- compute_source_hash(user_prompt),- GENERATION_LEVELS["file_page"],- )- # Cross-check LLM output against actual symbols- hal_warnings = _validate_symbol_references(response.content, parsed)- if hal_warnings:- log.warning(- "hallucination_check",- path=parsed.file_info.path,- count=len(hal_warnings),- refs=hal_warnings[:5],- )- page.metadata["hallucination_warnings"] = hal_warnings- return page-- async def _call_provider(- self,- page_type: str,- user_prompt: str,- request_id: str,- target_path: str | None = None,- ) -> GeneratedResponse:- """Call the provider with caching, optionally prefixing a language instruction."""- # Persistent cross-run cache: if the page exists from a prior run, was- # produced by the same model, and the prompt's source_hash matches,- # reuse the stored content without an LLM call.- if self._config.cache_enabled and target_path is not None:- page_id = compute_page_id(page_type, target_path)- prior = self._prior_pages.get(page_id)- if prior is not None and prior.model_name == self._provider.model_name:- current_hash = compute_source_hash(user_prompt)- if prior.source_hash == current_hash:- self._reuse_count += 1- log.debug(- "page_cache.persistent_hit",- page_type=page_type,- target_path=target_path,- )- return GeneratedResponse(- content=prior.content,- input_tokens=0,- output_tokens=0,- cached_tokens=0,- usage={"reused_from_prior_run": True},- )-- key = self._compute_cache_key(page_type, user_prompt)- if self._config.cache_enabled and key in self._cache:- log.debug("Cache hit", page_type=page_type, key=key[:8])- return self._cache[key]-- system_prompt = self._build_system_prompt(page_type)-- # The same system prompt is reused for every page of a given type, so- # mark it as cacheable. Providers without server-side prompt caching- # ignore the hint safely.- cache_hints: tuple[CacheHint, ...] = (- (CacheHint(segment="system"),) if self._config.cache_enabled else ()- )-- response = await self._provider.generate(- system_prompt,- user_prompt,- max_tokens=self._config.max_tokens,- temperature=self._config.temperature,- request_id=request_id,- reasoning=self._config.reasoning,- cache_hints=cache_hints,- )-- if self._config.cache_enabled:- self._cache[key] = response-- return response-- def _build_system_prompt(self, page_type: str) -> str:- base_system = SYSTEM_PROMPTS[page_type]- # Sanitize the configured language code: lower, strip, drop anything that isn't- # alphanumeric or underscore. Prevents user-supplied config from injecting- # newlines or extra instructions into the system prompt.- raw = (self._language or "en").lower().strip()- lang_code = "".join(ch for ch in raw if ch.isalnum() or ch == "_")- if lang_code not in _LANGUAGE_NAMES:- if lang_code != "en":- log.warning("unknown_language_code", code=lang_code, fallback="en")- lang_code = "en"- if lang_code == "en":- return base_system- lang_name = _LANGUAGE_NAMES[lang_code]- instruction = (- f"Generate all documentation content in {lang_name}. "- "Keep all code, file paths, and symbol names in their original form. "- "Do not translate them.\n\n"- )- return instruction + base_system-- def _compute_cache_key(self, page_type: str, user_prompt: str) -> str:- """Return SHA256(model + language + page_type + user_prompt) as cache key."""- raw = f"{self._provider.model_name}:{self._language}:{page_type}:{user_prompt}"- return hashlib.sha256(raw.encode()).hexdigest()-- def _build_generated_page(- self,- page_type: str,- target_path: str,- title: str,- response: GeneratedResponse,- source_hash: str,- level: int,- ) -> GeneratedPage:- """Wrap a GeneratedResponse in a GeneratedPage."""- now = _now_iso()- return GeneratedPage(- page_id=compute_page_id(page_type, target_path),- page_type=page_type,- title=title,- content=response.content,- summary=_extract_summary(response.content),- source_hash=source_hash,- model_name=self._provider.model_name,- provider_name=self._provider.provider_name,- input_tokens=response.input_tokens,- output_tokens=response.output_tokens,- cached_tokens=response.cached_tokens,- generation_level=level,- target_path=target_path,- created_at=now,- updated_at=now,- )-- def _render(self, template_name: str, **kwargs: Any) -> str:- """Render a Jinja2 template with the given kwargs."""- template = self._jinja_env.get_template(template_name)- return template.render(**kwargs)---# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _extract_summary(content: str, max_chars: int = 320) -> str:- """Extract a 1–3 sentence purpose blurb from rendered wiki markdown.-- Strategy: walk lines top-to-bottom, skip blanks/headings/list-markers/HTML- comments, and take the first prose paragraph. Truncate at sentence boundary- near max_chars. Fully deterministic — no extra LLM call.- """- if not content:- return ""- para_lines: list[str] = []- for raw in content.splitlines():- line = raw.strip()- if not line:- if para_lines:- break- continue- if line.startswith(("#", ">", "```", "---", "<!--", "|", "- ", "* ", "1.")):- if para_lines:- break- continue- para_lines.append(line)- if not para_lines:- return ""- text = " ".join(para_lines)- if len(text) <= max_chars:- return text- # Truncate at the last sentence boundary before max_chars- cut = text[:max_chars]- last_period = max(cut.rfind(". "), cut.rfind("? "), cut.rfind("! "))- if last_period > max_chars // 2:- return cut[: last_period + 1]- return cut.rstrip() + "…"---def _is_infra_file(parsed: ParsedFile) -> bool:- """Return True if the file is an infrastructure file."""- lang = parsed.file_info.language- if lang in _INFRA_LANGUAGES:- return True- name = Path(parsed.file_info.path).name- return name in _INFRA_FILENAMES---def _is_significant_file(- parsed: ParsedFile,- pagerank: dict[str, float],- betweenness: dict[str, float],- config: Any, # GenerationConfig- pr_threshold: float,-) -> bool:- """Return True if this code file deserves its own file_page.-- A file is significant if it is connected/important in the dependency graph- (entry point, top PageRank percentile, or bridge file) AND has enough- content to document.-- The symbol requirement is waived for files with no original definitions- (state modules, __init__ re-exporters, config files) that are still heavily- imported — these are architecturally important even without function bodies.- Package __init__.py files with any symbols are always included since they- are the public interface of their module.- """- path = parsed.file_info.path- pr = pagerank.get(path, 0.0)- bet = betweenness.get(path, 0.0)- is_entry = parsed.file_info.is_entry_point-- # Package __init__.py files are module interfaces — always include them- # if they have any symbols (re-exports, __getattr__, etc.)- if path.endswith("__init__.py") and len(parsed.symbols) > 0:- return True-- # Test files are always significant when present. They have near-zero- # PageRank because nothing imports them back, but they answer "what- # tests exercise X" / "where is Y verified" questions that the doc layer- # is the right place to surface. Users who want to exclude tests- # entirely can do so via skip_tests in the orchestrator upstream.- if parsed.file_info.is_test and len(parsed.symbols) > 0:- return True-- # Must appear significant in the graph- if not (is_entry or pr >= pr_threshold or bet > 0.0):- return False-- # Trivial-file gate: small files with almost no symbols (data classes,- # marker classes, single-message wrappers like Messages/*.cs) produce- # low-value pages. Entry points and graph hubs (high PageRank) bypass.- if (- getattr(config, "skip_trivial_files", True)- and not is_entry- and pr < pr_threshold * 2- and len(parsed.symbols) <= 2- and parsed.file_info.size_bytes < 1500- ):- return False-- # Waive the symbol-count requirement for graph-connected files that have- # no original definitions of their own (e.g. state/config modules that- # are imported by many files but mostly re-export or assemble values).- if len(parsed.symbols) < config.file_page_min_symbols:- return is_entry or pr >= pr_threshold-- return True---def _select_clone_representatives(- code_files: list[ParsedFile],- pagerank: dict[str, float],- *,- min_cluster_size: int = 3,-) -> set[str]:- """Return paths of files to *drop* because they are near-clones.-- Groups files by (parent_directory, signature shape), where the shape is the- sorted tuple of ``(symbol_kind, symbol_name)`` pairs from the parser. When- a cluster has at least ``min_cluster_size`` members, the highest-PageRank- member is kept and the rest are dropped. Entry points are never dropped.-- Language-agnostic: works for any language whose symbols carry a kind+name,- which the parser guarantees.- """- from collections import defaultdict-- clusters: dict[tuple[str, tuple[tuple[str, str], ...]], list[ParsedFile]] = defaultdict(list)- for p in code_files:- if p.file_info.is_entry_point or not p.symbols:- continue- parent = str(Path(p.file_info.path).parent.as_posix())- shape = tuple(sorted((str(s.kind), s.name) for s in p.symbols))- clusters[(parent, shape)].append(p)-- drop: set[str] = set()- for members in clusters.values():- if len(members) < min_cluster_size:- continue- members.sort(key=lambda p: pagerank.get(p.file_info.path, 0.0), reverse=True)- for loser in members[1:]:- drop.add(loser.file_info.path)- return drop---# ----------------------------------------------------------------------------# LLM output validation-# -----------------------------------------------------------------------------# Common words that appear in backticks but are not code symbols.-_BACKTICK_SKIP = frozenset(- {- # Python builtins & keywords- "True",- "False",- "None",- "self",- "cls",- "super",- "str",- "int",- "float",- "bool",- "list",- "dict",- "set",- "tuple",- "bytes",- "object",- "type",- "Any",- "Optional",- "Union",- "async",- "await",- "return",- "yield",- "import",- "from",- "class",- "def",- "if",- "else",- "for",- "while",- "try",- "except",- "raise",- "with",- "pass",- "break",- "continue",- "lambda",- "in",- "not",- "and",- "or",- "is",- "del",- "assert",- "finally",- "elif",- "as",- "global",- "nonlocal",- # JS/TS keywords- "null",- "undefined",- "this",- "const",- "let",- "var",- "function",- "export",- "default",- "extends",- "implements",- "interface",- "enum",- "new",- "typeof",- "instanceof",- "void",- "never",- "string",- "number",- "boolean",- "symbol",- "bigint",- "unknown",- "readonly",- "abstract",- "static",- "private",- "protected",- "public",- "require",- "module",- "exports",- "Promise",- "Map",- "Set",- "Array",- "Object",- "Error",- "Date",- "RegExp",- "JSON",- "Math",- "console",- # Common tool/ecosystem names- "pip",- "npm",- "npx",- "yarn",- "pnpm",- "go",- "rust",- "python",- "node",- "cargo",- "uv",- "git",- "docker",- "make",- # Common framework/lib names the LLM mentions in prose- "FastAPI",- "React",- "Next",- "Express",- "Django",- "Flask",- "SQLAlchemy",- "Pydantic",- "Click",- "Typer",- "pytest",- "asyncio",- "pathlib",- "dataclass",- "dataclasses",- }-)--# Regex: single-backtick references that look like identifiers.-_BACKTICK_REF_RE = re.compile(r"(?<!`)` *([A-Za-z_]\w*(?:\.\w+)*) *`(?!`)")--# Patterns that indicate the backtick content is a path, command, or-# value rather than a symbol reference — these should never be flagged.-_PATH_OR_CMD_RE = re.compile(- r"[/\\]" # contains path separator- r"|\.(?:py|ts|js|json|yaml|yml|toml|md|sh|sql|css|html)$" # file extension- r"|^[a-z][\w-]*$" # all-lowercase with hyphens = CLI command/flag-)---def _validate_symbol_references(- content: str,- parsed: ParsedFile,-) -> list[str]:- """Cross-check backtick-quoted names in LLM output against actual symbols.-- Returns a list of warning strings for references that don't match any- known symbol, export, or import in the ParsedFile. Designed to have low- false-positive rates — only flags references that look like symbol names- but can't be found anywhere in the file's AST, imports, or source text.- """- refs = set(_BACKTICK_REF_RE.findall(content))- if not refs:- return []-- # Build the known-names set from AST data- known: set[str] = set()- for s in parsed.symbols:- known.add(s.name)- known.add(s.qualified_name)- # Decorator names are valid references (e.g. @app.command("init"))- for dec in s.decorators:- # Extract the decorator function name: "@app.command" → "command"- dec_name = dec.lstrip("@").split("(")[0]- known.add(dec_name)- known.add(dec_name.split(".")[-1])- known.update(parsed.exports)- for imp in parsed.imports:- if imp.module_path:- # Add both the final component and intermediate segments- parts = imp.module_path.split(".")- known.update(parts)- known.update(imp.imported_names)- # Named bindings from import resolution- for binding in getattr(imp, "bindings", []):- known.add(binding.local_name)- if binding.exported_name:- known.add(binding.exported_name)-- # Also add all string literals from the source that look like identifiers- # (catches Click command names, decorator arguments, dict keys, etc.)- source_text = ""- if hasattr(parsed, "file_info") and hasattr(parsed.file_info, "path"):- # The source is in the context, but we only have the parsed file here.- # Use docstring and symbol names as a cheap approximation.- if parsed.docstring:- known.update(w for w in parsed.docstring.split() if w.isidentifier())-- warnings: list[str] = []- for ref in refs:- if ref in _BACKTICK_SKIP:- continue- # Skip short refs (1-2 chars are usually variables like `x`, `i`, `db`)- if len(ref) <= 2:- continue- # Skip anything that looks like a path, file, or CLI command- if _PATH_OR_CMD_RE.search(ref):- continue- # Skip all-uppercase (likely constants from other files: `MAX_RETRIES`)- if ref.isupper():- continue- # Check against known names- base = ref.split(".")[-1]- if ref in known or base in known:- continue- # Skip if the ref is a substring of any known symbol (covers partial- # references like `parse` when `parse_file` exists)- if any(ref in k for k in known if len(k) > len(ref)):- continue- warnings.append(ref)- return warningsdiff --git a/packages/core/src/repowise/core/generation/page_generator/README.md b/packages/core/src/repowise/core/generation/page_generator/README.mdnew file mode 100644index 000000000..ee28050a1--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/README.md@@ -0,0 +1,64 @@+# `page_generator`++Converts assembled context dataclasses into `GeneratedPage` objects by+rendering Jinja prompts, calling an LLM provider, and (for the long tail of+large repos) rendering deterministic template-only pages.++## Purpose++`PageGenerator.generate_all()` is the top of the documentation-generation+stage. It selects which pages to produce (via `generation.selection`), then+generates them in ordered, concurrency-bounded levels:++| Level | Page type | Notes |+| --- | --- | --- |+| 0/1 | `api_contract`, `symbol_spotlight` | merged batch |+| 2 | `file_page` | topo-ordered; tier-1 LLM or tier-2 template |+| 3 | `scc_page` | circular-dependency cycles |+| 4 | `module_page` | community / directory groups |+| 6/7/8 | `repo_overview` + `architecture_diagram`, `infra_page`, `onboarding` | merged batch |++### Tiered doc generation++`GenerationConfig.tier1_top_n` caps how many file pages get full LLM+generation. The top *N* selected files by PageRank are **tier-1** (the+existing LLM path); the rest are **tier-2** — rendered from+`templates/file_page_tier2.j2` with no LLM call and embedded for search.+`tier1_top_n = None` (default) puts every selected page in tier-1, so+behaviour is unchanged unless the knob is set.++## Public API++```python+from repowise.core.generation.page_generator import PageGenerator, SYSTEM_PROMPTS, PriorPage+```++- `PageGenerator(provider, assembler, config, ...)` — `await .generate_all(...)`+ returns `list[GeneratedPage]`. Per-type `generate_file_page` / `generate_*`+ methods are also public.+- `SYSTEM_PROMPTS` — constant system prompt per page type (prefix-cacheable).+- `PriorPage` — cross-run reuse snapshot keyed by `page_id`.++## Internal layout++- `core.py` — `PageGenerator` (provider call, two-tier caching, file-page LLM+ path + deterministic tier-2 renderer).+- `pertype.py` — `PerTypeGenerationMixin` with the per-page-type methods.+- `orchestrate.py` — `_GenerationRun` (per-call state, level runner) and+ `run_generate_all` (the entry point `generate_all` delegates to).+- `levels.py` — pure per-level coroutine builders reading the run state.+- `tiering.py` — `partition_file_tiers` (tier-1/tier-2 split by PageRank).+- `prompts.py`, `validation.py`, `helpers.py` — constants and pure helpers.++## Extension points++- New page type: add a `generate_*` method to `pertype.py`, a level builder to+ `levels.py`, and wire it into `_GenerationRun.execute()`.+- New tier policy: extend `partition_file_tiers` / add a renderer template.++## Tests++- `tests/unit/generation/test_page_generator.py`+- `tests/unit/generation/test_page_generator_tiering.py`+- `tests/integration/test_generation_pipeline.py`+"""diff --git a/packages/core/src/repowise/core/generation/page_generator/__init__.py b/packages/core/src/repowise/core/generation/page_generator/__init__.pynew file mode 100644index 000000000..7f90e6b31--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/__init__.py@@ -0,0 +1,41 @@+"""Page generator package — converts context dataclasses into GeneratedPage objects.++Public surface (import path preserved from the former single-module form):++ from repowise.core.generation.page_generator import PageGenerator, SYSTEM_PROMPTS, PriorPage++Internal layout:+ core.py — the PageGenerator class (provider call, caching, file pages)+ pertype.py — per-page-type generate_* methods (mixin)+ orchestrate.py — level-by-level generate_all orchestration+ levels.py — per-level coroutine builders+ tiering.py — tier-1/tier-2 partition (large-repo doc generation)+ prompts.py — SYSTEM_PROMPTS constants + language names+ validation.py — hallucination detection for LLM output+ helpers.py — pure helpers (summaries, significance, clone dedupe)+"""++from __future__ import annotations++from .core import PageGenerator, PriorPage+from .helpers import (+ _extract_summary,+ _is_infra_file,+ _is_significant_file,+ _now_iso,+ _select_clone_representatives,+)+from .prompts import SYSTEM_PROMPTS+from .validation import _validate_symbol_references++__all__ = [+ "SYSTEM_PROMPTS",+ "PageGenerator",+ "PriorPage",+ "_extract_summary",+ "_is_infra_file",+ "_is_significant_file",+ "_now_iso",+ "_select_clone_representatives",+ "_validate_symbol_references",+]diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.pynew file mode 100644index 000000000..2d57e8db3--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/core.py@@ -0,0 +1,389 @@+"""PageGenerator — renders prompts, calls the provider, wraps responses.++PageGenerator is the main orchestration layer. It:+ 1. Calls ContextAssembler to build template context from ingestion data.+ 2. Renders the Jinja2 user-prompt template.+ 3. Calls the provider with the rendered prompt + system prompt constant.+ 4. Wraps the response in a GeneratedPage.+ 5. Manages concurrency (asyncio.Semaphore) and prompt caching (SHA256).++The level-by-level orchestration of ``generate_all`` lives in+``orchestrate.py``; the per-type ``generate_*`` methods live in ``pertype.py``.+"""++from __future__ import annotations++import hashlib+import uuid+from collections.abc import Callable+from dataclasses import dataclass+from pathlib import Path+from typing import TYPE_CHECKING, Any++import jinja2+import structlog++from repowise.core.ingestion.models import ParsedFile, RepoStructure+from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse++from ..context_assembler import ContextAssembler, FilePageContext+from ..models import (+ GENERATION_LEVELS,+ GeneratedPage,+ GenerationConfig,+ compute_page_id,+ compute_source_hash,+)+from .helpers import _extract_summary, _now_iso+from .pertype import PerTypeGenerationMixin+from .prompts import _LANGUAGE_NAMES, SYSTEM_PROMPTS+from .validation import _validate_symbol_references++if TYPE_CHECKING:+ from pathlib import Path as _Path # noqa: F401++log = structlog.get_logger(__name__)+++@dataclass(frozen=True)+class PriorPage:+ """Snapshot of a previously-generated page used for cross-run reuse.++ Lives in :class:`PageGenerator` keyed by ``page_id``. When the freshly+ rendered prompt produces a matching ``source_hash`` under the same+ ``model_name``, the LLM call is skipped and ``content`` is reused.+ """++ source_hash: str+ model_name: str+ content: str+ input_tokens: int = 0+ output_tokens: int = 0+ cached_tokens: int = 0+++class PageGenerator(PerTypeGenerationMixin):+ """Generate wiki pages by rendering prompts and calling an LLM provider.++ Args:+ provider: Any BaseProvider implementation.+ assembler: ContextAssembler instance.+ config: GenerationConfig controlling budget, concurrency, caching.+ jinja_env: Optional Jinja2 Environment (defaults to FileSystemLoader+ pointing at the templates/ directory next to this package).+ """++ def __init__(+ self,+ provider: BaseProvider,+ assembler: ContextAssembler,+ config: GenerationConfig,+ jinja_env: jinja2.Environment | None = None,+ vector_store: Any | None = None,+ language: str = "en",+ prior_pages: dict[str, PriorPage] | None = None,+ ) -> None:+ self._provider = provider+ self._assembler = assembler+ self._config = config+ self._vector_store = vector_store+ self._language = language+ self._cache: dict[str, GeneratedResponse] = {}+ # Map of page_id → PriorPage from previous generation runs. When the+ # rendered prompt's source_hash matches the prior page's hash AND the+ # model is the same, the LLM call is skipped and the prior content is+ # reused. Wired by the orchestrator from the persisted wiki_pages+ # table.+ self._prior_pages: dict[str, PriorPage] = prior_pages or {}+ self._reuse_count: int = 0++ if jinja_env is None:+ templates_dir = Path(__file__).parent.parent / "templates"+ loader = jinja2.FileSystemLoader(str(templates_dir))+ jinja_env = jinja2.Environment(+ loader=loader,+ undefined=jinja2.StrictUndefined,+ autoescape=False,+ )+ self._jinja_env = jinja_env++ # ------------------------------------------------------------------+ # generate_all — orchestration (delegates to orchestrate.py)+ # ------------------------------------------------------------------++ async def generate_all(+ self,+ parsed_files: list[ParsedFile],+ source_map: dict[str, bytes],+ graph_builder: Any, # GraphBuilder+ repo_structure: RepoStructure,+ repo_name: str,+ job_system: Any | None = None, # JobSystem | None+ on_page_done: Callable[[str], None] | None = None,+ on_total_known: Callable[[int], None] | None = None,+ on_subphase: Callable[[str, int | None], None] | None = None,+ git_meta_map: dict[str, dict] | None = None,+ resume: bool = False,+ repo_path: Path | str | None = None,+ dead_code_report: Any | None = None,+ decision_report: Any | None = None,+ external_systems: list[dict] | None = None,+ ) -> list[GeneratedPage]:+ """Generate all wiki pages for a repository.++ Runs generation in ordered levels. Each level's pages are generated+ concurrently (up to config.max_concurrency). Failures within a level+ are logged but do not abort the remaining levels.+ """+ from .orchestrate import run_generate_all++ return await run_generate_all(+ self,+ parsed_files=parsed_files,+ source_map=source_map,+ graph_builder=graph_builder,+ repo_structure=repo_structure,+ repo_name=repo_name,+ job_system=job_system,+ on_page_done=on_page_done,+ on_total_known=on_total_known,+ on_subphase=on_subphase,+ git_meta_map=git_meta_map,+ resume=resume,+ repo_path=repo_path,+ dead_code_report=dead_code_report,+ decision_report=decision_report,+ external_systems=external_systems,+ )++ # ------------------------------------------------------------------+ # File-page generation (LLM + deterministic tier-2)+ # ------------------------------------------------------------------++ async def _generate_file_page_from_ctx(+ self,+ parsed: ParsedFile,+ ctx: FilePageContext,+ ) -> GeneratedPage:+ """Generate a file_page from a pre-assembled context (avoids double-assembly)."""+ # RAG context: query vector store for related pages (B1).+ # Gated by two short-circuits so we don't burn an embedder+ # round-trip on every page when the result wouldn't help:+ # 1. ``enable_rag_context`` config flag (off → fully skip).+ # 2. ``rag_min_store_size`` — early pages run against an empty+ # or near-empty store and the search returns nothing useful.+ if self._vector_store is not None and getattr(+ self._config, "enable_rag_context", True+ ):+ min_store_size = max(0, int(getattr(self._config, "rag_min_store_size", 10) or 0))+ store_ok = True+ if min_store_size > 0:+ try:+ current_ids = await self._vector_store.list_page_ids()+ store_ok = len(current_ids) >= min_store_size+ except Exception:+ # If the store can't be sized cheaply, fall through to+ # the search — it'll either succeed or hit the+ # existing exception path.+ store_ok = True+ if store_ok:+ query_terms = parsed.exports or [+ s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"+ ]+ if query_terms:+ try:+ results = await self._vector_store.search(+ ", ".join(query_terms[:5]), limit=3+ )+ self_id = f"file_page:{parsed.file_info.path}"+ ctx.rag_context = [+ f"[{r.page_id}]\n{r.snippet}"+ for r in results+ if r.page_id != self_id+ ]+ except Exception as e:+ log.debug(+ "rag.search_failed", path=parsed.file_info.path, error=str(e)+ )+ user_prompt = self._render("file_page.j2", ctx=ctx)+ response = await self._call_provider(+ "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path+ )+ page = self._build_generated_page(+ "file_page",+ parsed.file_info.path,+ f"File: {parsed.file_info.path}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["file_page"],+ )+ # Cross-check LLM output against actual symbols+ hal_warnings = _validate_symbol_references(response.content, parsed)+ if hal_warnings:+ log.warning(+ "hallucination_check",+ path=parsed.file_info.path,+ count=len(hal_warnings),+ refs=hal_warnings[:5],+ )+ page.metadata["hallucination_warnings"] = hal_warnings+ return page++ async def _generate_file_page_tier2(+ self,+ parsed: ParsedFile,+ ctx: FilePageContext,+ ) -> GeneratedPage:+ """Render a deterministic (no-LLM) tier-2 file page.++ Used for the long tail of selected files on large repos. The page is+ built straight from the assembled context via a Jinja template, marked+ as template-generated, and carries zero token cost. It is embedded for+ search by the level runner like any other page. No provider call and+ no hallucination check (the content is factual by construction).+ """+ content = self._render("file_page_tier2.j2", ctx=ctx)+ now = _now_iso()+ page = GeneratedPage(+ page_id=compute_page_id("file_page", parsed.file_info.path),+ page_type="file_page",+ title=f"File: {parsed.file_info.path}",+ content=content,+ summary=_extract_summary(content),+ source_hash=compute_source_hash(content),+ model_name=self._provider.model_name,+ provider_name="template",+ input_tokens=0,+ output_tokens=0,+ cached_tokens=0,+ generation_level=GENERATION_LEVELS["file_page"],+ target_path=parsed.file_info.path,+ created_at=now,+ updated_at=now,+ )+ page.metadata["doc_tier"] = 2+ return page++ # ------------------------------------------------------------------+ # Provider call + page assembly+ # ------------------------------------------------------------------++ async def _call_provider(+ self,+ page_type: str,+ user_prompt: str,+ request_id: str,+ target_path: str | None = None,+ ) -> GeneratedResponse:+ """Call the provider with caching, optionally prefixing a language instruction."""+ # Persistent cross-run cache: if the page exists from a prior run, was+ # produced by the same model, and the prompt's source_hash matches,+ # reuse the stored content without an LLM call.+ if self._config.cache_enabled and target_path is not None:+ page_id = compute_page_id(page_type, target_path)+ prior = self._prior_pages.get(page_id)+ if prior is not None and prior.model_name == self._provider.model_name:+ current_hash = compute_source_hash(user_prompt)+ if prior.source_hash == current_hash:+ self._reuse_count += 1+ log.debug(+ "page_cache.persistent_hit",+ page_type=page_type,+ target_path=target_path,+ )+ return GeneratedResponse(+ content=prior.content,+ input_tokens=0,+ output_tokens=0,+ cached_tokens=0,+ usage={"reused_from_prior_run": True},+ )++ key = self._compute_cache_key(page_type, user_prompt)+ if self._config.cache_enabled and key in self._cache:+ log.debug("Cache hit", page_type=page_type, key=key[:8])+ return self._cache[key]++ system_prompt = self._build_system_prompt(page_type)++ # The same system prompt is reused for every page of a given type, so+ # mark it as cacheable. Providers without server-side prompt caching+ # ignore the hint safely.+ cache_hints: tuple[CacheHint, ...] = (+ (CacheHint(segment="system"),) if self._config.cache_enabled else ()+ )++ response = await self._provider.generate(+ system_prompt,+ user_prompt,+ max_tokens=self._config.max_tokens,+ temperature=self._config.temperature,+ request_id=request_id,+ reasoning=self._config.reasoning,+ cache_hints=cache_hints,+ )++ if self._config.cache_enabled:+ self._cache[key] = response++ return response++ def _build_system_prompt(self, page_type: str) -> str:+ base_system = SYSTEM_PROMPTS[page_type]+ # Sanitize the configured language code: lower, strip, drop anything that isn't+ # alphanumeric or underscore. Prevents user-supplied config from injecting+ # newlines or extra instructions into the system prompt.+ raw = (self._language or "en").lower().strip()+ lang_code = "".join(ch for ch in raw if ch.isalnum() or ch == "_")+ if lang_code not in _LANGUAGE_NAMES:+ if lang_code != "en":+ log.warning("unknown_language_code", code=lang_code, fallback="en")+ lang_code = "en"+ if lang_code == "en":+ return base_system+ lang_name = _LANGUAGE_NAMES[lang_code]+ instruction = (+ f"Generate all documentation content in {lang_name}. "+ "Keep all code, file paths, and symbol names in their original form. "+ "Do not translate them.\n\n"+ )+ return instruction + base_system++ def _compute_cache_key(self, page_type: str, user_prompt: str) -> str:+ """Return SHA256(model + language + page_type + user_prompt) as cache key."""+ raw = f"{self._provider.model_name}:{self._language}:{page_type}:{user_prompt}"+ return hashlib.sha256(raw.encode()).hexdigest()++ def _build_generated_page(+ self,+ page_type: str,+ target_path: str,+ title: str,+ response: GeneratedResponse,+ source_hash: str,+ level: int,+ ) -> GeneratedPage:+ """Wrap a GeneratedResponse in a GeneratedPage."""+ now = _now_iso()+ return GeneratedPage(+ page_id=compute_page_id(page_type, target_path),+ page_type=page_type,+ title=title,+ content=response.content,+ summary=_extract_summary(response.content),+ source_hash=source_hash,+ model_name=self._provider.model_name,+ provider_name=self._provider.provider_name,+ input_tokens=response.input_tokens,+ output_tokens=response.output_tokens,+ cached_tokens=response.cached_tokens,+ generation_level=level,+ target_path=target_path,+ created_at=now,+ updated_at=now,+ )++ def _render(self, template_name: str, **kwargs: Any) -> str:+ """Render a Jinja2 template with the given kwargs."""+ template = self._jinja_env.get_template(template_name)+ return template.render(**kwargs)diff --git a/packages/core/src/repowise/core/generation/page_generator/helpers.py b/packages/core/src/repowise/core/generation/page_generator/helpers.pynew file mode 100644index 000000000..1a0b65f74--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/helpers.py@@ -0,0 +1,219 @@+"""Pure helper functions for the page generator.++These have no dependency on :class:`PageGenerator` state and are unit-tested+in isolation. ``_extract_summary`` / ``_is_significant_file`` /+``_select_clone_representatives`` keep their original names because external+docs and call sites reference them.+"""++from __future__ import annotations++from collections import defaultdict+from datetime import UTC, datetime+from pathlib import Path+from typing import Any++from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from repowise.core.ingestion.models import ParsedFile++_INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()+_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})+_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()+++def _now_iso() -> str:+ """Return current UTC time as ISO-8601 string."""+ return datetime.now(UTC).isoformat()+++def _extract_summary(content: str, max_chars: int = 320) -> str:+ """Extract a 1-3 sentence purpose blurb from rendered wiki markdown.++ Strategy: walk lines top-to-bottom, skip blanks/headings/list-markers/HTML+ comments, and take the first prose paragraph. Truncate at sentence boundary+ near max_chars. Fully deterministic — no extra LLM call.+ """+ if not content:+ return ""+ para_lines: list[str] = []+ for raw in content.splitlines():+ line = raw.strip()+ if not line:+ if para_lines:+ break+ continue+ if line.startswith(("#", ">", "```", "---", "<!--", "|", "- ", "* ", "1.")):+ if para_lines:+ break+ continue+ para_lines.append(line)+ if not para_lines:+ return ""+ text = " ".join(para_lines)+ if len(text) <= max_chars:+ return text+ # Truncate at the last sentence boundary before max_chars+ cut = text[:max_chars]+ last_period = max(cut.rfind(". "), cut.rfind("? "), cut.rfind("! "))+ if last_period > max_chars // 2:+ return cut[: last_period + 1]+ return cut.rstrip() + "…"+++def overview_summary(content: str) -> str:+ """Return a dependency-context blurb favouring the ## Overview section.++ Distinct from :func:`_extract_summary`: this is the summary fed into+ downstream pages' dependency context and into the vector-store embed+ payload, where the ## Overview lead sentence is the most useful anchor.+ """+ if "## Overview" in content:+ start = content.index("## Overview") + len("## Overview")+ end = content.find("\n##", start)+ return content[start : end if end > 0 else start + 1600].strip()[:400]+ return content[:400]+++def _is_infra_file(parsed: ParsedFile) -> bool:+ """Return True if the file is an infrastructure file."""+ lang = parsed.file_info.language+ if lang in _INFRA_LANGUAGES:+ return True+ name = Path(parsed.file_info.path).name+ return name in _INFRA_FILENAMES+++def _is_significant_file(+ parsed: ParsedFile,+ pagerank: dict[str, float],+ betweenness: dict[str, float],+ config: Any, # GenerationConfig+ pr_threshold: float,+) -> bool:+ """Return True if this code file deserves its own file_page.++ A file is significant if it is connected/important in the dependency graph+ (entry point, top PageRank percentile, or bridge file) AND has enough+ content to document.++ The symbol requirement is waived for files with no original definitions+ (state modules, __init__ re-exporters, config files) that are still heavily+ imported — these are architecturally important even without function bodies.+ Package __init__.py files with any symbols are always included since they+ are the public interface of their module.+ """+ path = parsed.file_info.path+ pr = pagerank.get(path, 0.0)+ bet = betweenness.get(path, 0.0)+ is_entry = parsed.file_info.is_entry_point++ # Package __init__.py files are module interfaces — always include them+ # if they have any symbols (re-exports, __getattr__, etc.)+ if path.endswith("__init__.py") and len(parsed.symbols) > 0:+ return True++ # Test files are always significant when present. They have near-zero+ # PageRank because nothing imports them back, but they answer "what+ # tests exercise X" / "where is Y verified" questions that the doc layer+ # is the right place to surface. Users who want to exclude tests+ # entirely can do so via skip_tests in the orchestrator upstream.+ if parsed.file_info.is_test and len(parsed.symbols) > 0:+ return True++ # Must appear significant in the graph+ if not (is_entry or pr >= pr_threshold or bet > 0.0):+ return False++ # Trivial-file gate: small files with almost no symbols (data classes,+ # marker classes, single-message wrappers like Messages/*.cs) produce+ # low-value pages. Entry points and graph hubs (high PageRank) bypass.+ if (+ getattr(config, "skip_trivial_files", True)+ and not is_entry+ and pr < pr_threshold * 2+ and len(parsed.symbols) <= 2+ and parsed.file_info.size_bytes < 1500+ ):+ return False++ # Waive the symbol-count requirement for graph-connected files that have+ # no original definitions of their own (e.g. state/config modules that+ # are imported by many files but mostly re-export or assemble values).+ if len(parsed.symbols) < config.file_page_min_symbols:+ return is_entry or pr >= pr_threshold++ return True+++def _select_clone_representatives(+ code_files: list[ParsedFile],+ pagerank: dict[str, float],+ *,+ min_cluster_size: int = 3,+) -> set[str]:+ """Return paths of files to *drop* because they are near-clones.++ Groups files by (parent_directory, signature shape), where the shape is the+ sorted tuple of ``(symbol_kind, symbol_name)`` pairs from the parser. When+ a cluster has at least ``min_cluster_size`` members, the highest-PageRank+ member is kept and the rest are dropped. Entry points are never dropped.++ Language-agnostic: works for any language whose symbols carry a kind+name,+ which the parser guarantees.+ """+ clusters: dict[tuple[str, tuple[tuple[str, str], ...]], list[ParsedFile]] = defaultdict(list)+ for p in code_files:+ if p.file_info.is_entry_point or not p.symbols:+ continue+ parent = str(Path(p.file_info.path).parent.as_posix())+ shape = tuple(sorted((str(s.kind), s.name) for s in p.symbols))+ clusters[(parent, shape)].append(p)++ drop: set[str] = set()+ for members in clusters.values():+ if len(members) < min_cluster_size:+ continue+ members.sort(key=lambda p: pagerank.get(p.file_info.path, 0.0), reverse=True)+ for loser in members[1:]:+ drop.add(loser.file_info.path)+ return drop+++def build_dead_code_map(dead_code_report: Any | None) -> dict[str, list[dict]]:+ """Index dead-code findings by file path for per-page lookup."""+ dead_code_by_file: dict[str, list[dict]] = {}+ if dead_code_report is not None and getattr(dead_code_report, "findings", None):+ for f in dead_code_report.findings:+ dead_code_by_file.setdefault(f.file_path, []).append(+ {+ "symbol_name": f.symbol_name,+ "symbol_kind": f.symbol_kind,+ "kind": str(f.kind),+ "reason": f.reason,+ "confidence": f.confidence,+ "safe_to_delete": f.safe_to_delete,+ }+ )+ return dead_code_by_file+++def build_decision_maps(+ decision_report: Any | None,+) -> tuple[dict[str, list[dict]], list[dict]]:+ """Index decision records by file path and as a flat list."""+ decisions_by_file: dict[str, list[dict]] = {}+ decisions_all: list[dict] = []+ if decision_report is not None and getattr(decision_report, "decisions", None):+ for d in decision_report.decisions:+ payload = {+ "title": d.title,+ "decision": d.decision,+ "rationale": d.rationale,+ "source": d.source,+ "confidence": d.confidence,+ "evidence_file": d.evidence_file,+ }+ decisions_all.append(payload)+ for fp in d.affected_files or []:+ decisions_by_file.setdefault(fp, []).append(payload)+ return decisions_by_file, decisions_alldiff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.pynew file mode 100644index 000000000..271157488--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py@@ -0,0 +1,306 @@+"""Per-level coroutine builders for the generation orchestrator.++Each function takes the live :class:`_GenerationRun` and returns a list of+``(page_id, coroutine)`` tuples for one generation level. They read graph+metrics, selection allow-sets, and the shared context cache off the run+object. Behaviour mirrors the original inline ``generate_all`` exactly.+"""++from __future__ import annotations++import contextlib+from typing import TYPE_CHECKING, Any++import structlog++from .. import onboarding as _onboarding+from ..context_assembler import FilePageContext+from ..models import compute_page_id+from .helpers import _is_infra_file++if TYPE_CHECKING:+ from .orchestrate import _GenerationRun++log = structlog.get_logger(__name__)+++def build_level01_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 0 (api_contract) + level 1 (symbol_spotlight), merged."""+ gen = run.gen+ # ---- Level 0: api_contract (allow-set filtered) ----+ api_files = [+ p+ for p in run.parsed_files+ if p.file_info.is_api_contract and p.file_info.path in run.sel_api_paths+ ]+ level0 = [+ (+ compute_page_id("api_contract", p.file_info.path),+ gen.generate_api_contract(p, run.source_map.get(p.file_info.path, b"")),+ )+ for p in api_files+ if compute_page_id("api_contract", p.file_info.path) not in run.completed_ids+ ]++ # ---- Level 1: symbol_spotlight (allow-set filtered) ----+ parsed_by_path: dict[str, Any] = {p.file_info.path: p for p in run.parsed_files}+ top_symbols: list[tuple[Any, Any]] = []+ for file_path, sym_name in run.selection.symbol_spotlights:+ pf = parsed_by_path.get(file_path)+ if pf is None:+ continue+ sym = next((s for s in pf.symbols if s.name == sym_name), None)+ if sym is not None:+ top_symbols.append((sym, pf))++ level1 = [+ (+ compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}"),+ gen.generate_symbol_spotlight(+ sym, pf, run.pagerank, run.graph, source_map=run.source_map+ ),+ )+ for sym, pf in top_symbols+ if compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}")+ not in run.completed_ids+ ]+ return level0 + level1+++def _topo_order_code_files(run: _GenerationRun) -> None:+ """Reorder ``run.code_files`` so dependencies are generated before dependents."""+ code_file_paths = [p.file_info.path for p in run.code_files]+ graph = run.graph+ try:+ import networkx as nx # type: ignore[import]++ code_file_set = set(code_file_paths)+ dag = nx.DiGraph()+ dag.add_nodes_from(code_file_paths)+ for path_ in code_file_paths:+ if path_ in graph:+ for succ in graph.successors(path_):+ if succ in code_file_set:+ dag.add_edge(path_, succ) # path_ depends on succ++ if nx.is_directed_acyclic_graph(dag):+ # topological_sort yields u before v for each edge u→v (dependents+ # before dependencies). We want leaves first, so reverse.+ topo_order = list(reversed(list(nx.topological_sort(dag))))+ else:+ condensation = nx.condensation(dag)+ topo_order_scc = list(reversed(list(nx.topological_sort(condensation))))+ scc_members: dict[int, list[str]] = {+ n: list(condensation.nodes[n]["members"]) for n in condensation.nodes+ }+ topo_order = [node for scc_id in topo_order_scc for node in scc_members[scc_id]]++ priority_index = {p: i for i, p in enumerate(code_file_paths)}+ topo_order = [p for p in topo_order if p in priority_index]+ path_to_parsed = {p.file_info.path: p for p in run.code_files}+ run.code_files = [path_to_parsed[p] for p in topo_order if p in path_to_parsed]+ except Exception:+ pass # Keep existing priority order on any failure+++async def _prefetch_dependency_summaries(run: _GenerationRun) -> None:+ """Batch-prefetch dependency summaries from the vector store in one call."""+ if run.vector_store is None:+ return+ needed_deps: set[str] = set()+ for p in run.code_files:+ path_ = p.file_info.path+ if path_ not in run.graph:+ continue+ for dep in run.graph.successors(path_):+ if dep.startswith("external:"):+ continue+ if dep in run.completed_page_summaries:+ continue+ needed_deps.add(dep)+ if not needed_deps:+ return+ try:+ batch = await run.vector_store.get_page_summaries_by_paths(list(needed_deps))+ for dep_path, payload in batch.items():+ summary = payload.get("summary") if payload else None+ if summary:+ run.completed_page_summaries[dep_path] = summary+ except Exception as exc:+ log.debug("rag.batch_dep_prefetch_failed", error=str(exc))+++async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 2 (file_page): topo-ordered context assembly + tier routing.++ Context is assembled for ALL code files (module pages need it). Pages are+ emitted only for files in the selection allow-set. Tier-1 paths get the+ full LLM path; tier-2 paths get the deterministic template renderer.+ """+ gen = run.gen+ _topo_order_code_files(run)+ await _prefetch_dependency_summaries(run)++ coros: list[tuple[str, Any]] = []+ for p in run.code_files:+ ctx: FilePageContext = gen._assembler.assemble_file_page(+ p,+ run.graph,+ run.pagerank,+ run.betweenness,+ run.community,+ run.source_map.get(p.file_info.path, b""),+ git_meta=run.git_meta_map.get(p.file_info.path) if run.git_meta_map else None,+ page_summaries=run.completed_page_summaries,+ dead_code_findings=run.dead_code_by_file.get(p.file_info.path),+ decision_records=run.decisions_by_file.get(p.file_info.path),+ )+ run.file_page_contexts[p.file_info.path] = ctx+ path = p.file_info.path+ pid = compute_page_id("file_page", path)+ if path in run.sel_file_paths and pid not in run.completed_ids:+ if path in run.tier1_paths:+ coros.append((pid, gen._generate_file_page_from_ctx(p, ctx)))+ else:+ coros.append((pid, gen._generate_file_page_tier2(p, ctx)))+ return coros+++def build_level3_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 3 (scc_page), allow-set filtered."""+ gen = run.gen+ coros: list[tuple[str, Any]] = []+ for scc_id, scc_files in run.sel_scc_groups:+ fc_list = [+ run.file_page_contexts[f] for f in scc_files if f in run.file_page_contexts+ ]+ pid = compute_page_id("scc_page", scc_id)+ if pid not in run.completed_ids:+ coros.append((pid, gen.generate_scc_page(scc_id, scc_files, fc_list)))+ return coros+++def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 4 (module_page), allow-set filtered."""+ gen = run.gen+ coros: list[tuple[str, Any]] = []+ for mg in run.sel_module_groups:+ fcs = [+ run.file_page_contexts[fp]+ for fp in mg.file_paths+ if fp in run.file_page_contexts+ ]+ if not fcs:+ continue+ page_id = compute_page_id("module_page", mg.key)+ if page_id in run.completed_ids:+ continue+ coros.append(+ (+ page_id,+ gen.generate_module_page(+ mg.display,+ mg.language,+ fcs,+ run.graph,+ git_meta_map=run.git_meta_map,+ page_summaries=run.completed_page_summaries,+ decision_records=run.decisions_all,+ dead_code_findings=[+ d for fc in fcs for d in run.dead_code_by_file.get(fc.file_path, [])+ ],+ external_systems=run.external_systems,+ community_label=mg.label,+ community_cohesion=mg.cohesion,+ target_path=mg.key,+ ),+ )+ )+ return coros+++def build_level6_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 6 (repo_overview + architecture_diagram)."""+ gen = run.gen+ coros: list[tuple[str, Any]] = []+ if compute_page_id("repo_overview", run.repo_name) not in run.completed_ids:+ coros.append(+ (+ compute_page_id("repo_overview", run.repo_name),+ gen.generate_repo_overview(+ run.repo_structure,+ run.pagerank,+ run.sccs,+ run.community,+ git_meta_map=run.git_meta_map,+ graph_builder=run.graph_builder,+ repo_name=run.repo_name,+ external_systems=run.external_systems,+ decision_records=run.decisions_all[:10],+ ),+ )+ )+ if compute_page_id("architecture_diagram", run.repo_name) not in run.completed_ids:+ coros.append(+ (+ compute_page_id("architecture_diagram", run.repo_name),+ gen.generate_architecture_diagram(+ run.graph, run.pagerank, run.community, run.sccs, run.repo_name+ ),+ )+ )+ return coros+++def build_level7_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 7 (infra_page), allow-set filtered."""+ gen = run.gen+ infra_files = [+ p+ for p in run.parsed_files+ if _is_infra_file(p) and p.file_info.path in run.sel_infra_paths+ ]+ return [+ (+ compute_page_id("infra_page", p.file_info.path),+ gen.generate_infra_page(p, run.source_map.get(p.file_info.path, b"")),+ )+ for p in infra_files+ if compute_page_id("infra_page", p.file_info.path) not in run.completed_ids+ ]+++def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+ """Level 8 (curated onboarding collection)."""+ gen = run.gen+ coros: list[tuple[str, Any]] = []+ if not getattr(run.config, "enable_onboarding", True):+ return coros+ specs = _onboarding.iter_specs()+ if not specs:+ return coros+ if run.on_subphase is not None:+ with contextlib.suppress(Exception):+ run.on_subphase("onboarding", len(specs))+ signals = _onboarding.OnboardingSignals(+ repo_name=run.repo_name,+ repo_structure=run.repo_structure,+ parsed_files=tuple(run.parsed_files),+ source_map=run.source_map,+ graph_builder=run.graph_builder,+ pagerank=run.pagerank,+ betweenness=run.betweenness,+ community=run.community,+ sccs=tuple(run.sccs),+ git_meta_map=run.git_meta_map,+ dead_code_by_file=run.dead_code_by_file,+ decisions_all=tuple(run.decisions_all),+ external_systems=tuple(run.external_systems),+ completed_page_summaries=dict(run.completed_page_summaries),+ )+ for spec in specs:+ page_id = compute_page_id("onboarding", _onboarding.target_path(spec.slot))+ if page_id in run.completed_ids:+ continue+ coros.append((page_id, gen.generate_onboarding_page(spec, signals)))+ return corosdiff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pynew file mode 100644index 000000000..e46da8c16--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -0,0 +1,373 @@+"""Level-by-level orchestration for ``PageGenerator.generate_all``.++``run_generate_all`` builds a :class:`_GenerationRun` that holds the shared+per-run state (graph metrics, selection allow-sets, job bookkeeping, the+concurrency semaphore) and drives the ordered generation levels. The+per-level coroutine builders live in ``levels.py`` and read this state object.++Behaviour is identical to the previous single-method implementation; this is+purely a structural split to satisfy the project's 400-line ceiling.+"""++from __future__ import annotations++import asyncio+from collections.abc import Callable+from pathlib import Path+from typing import TYPE_CHECKING, Any++import structlog++from ..context_assembler import FilePageContext+from ..models import GeneratedPage+from . import levels as _levels+from .helpers import (+ _CODE_LANGUAGES,+ _is_infra_file,+ _select_clone_representatives,+ build_dead_code_map,+ build_decision_maps,+ overview_summary,+)+from .tiering import partition_file_tiers++if TYPE_CHECKING:+ from .core import PageGenerator++log = structlog.get_logger(__name__)+++class _GenerationRun:+ """Mutable per-call state for one ``generate_all`` invocation."""++ def __init__(+ self,+ gen: PageGenerator,+ *,+ parsed_files: list[Any],+ source_map: dict[str, bytes],+ graph_builder: Any,+ repo_structure: Any,+ repo_name: str,+ job_system: Any | None,+ on_page_done: Callable[[str], None] | None,+ on_total_known: Callable[[int], None] | None,+ on_subphase: Callable[[str, int | None], None] | None,+ git_meta_map: dict[str, dict] | None,+ resume: bool,+ repo_path: Path | str | None,+ dead_code_report: Any | None,+ decision_report: Any | None,+ external_systems: list[dict] | None,+ ) -> None:+ self.gen = gen+ self.config = gen._config+ self.vector_store = gen._vector_store+ self.parsed_files = parsed_files+ self.source_map = source_map+ self.graph_builder = graph_builder+ self.repo_structure = repo_structure+ self.repo_name = repo_name+ self.job_system = job_system+ self.on_page_done = on_page_done+ self.on_total_known = on_total_known+ self.on_subphase = on_subphase+ self.git_meta_map = git_meta_map+ self.resume = resume+ self.repo_path = repo_path+ self.external_systems = external_systems or []++ # ---- Graph metrics ----+ self.graph = graph_builder.graph()+ self.pagerank = graph_builder.pagerank()+ self.betweenness = graph_builder.betweenness_centrality()+ self.community = graph_builder.community_detection()+ self.sccs = graph_builder.strongly_connected_components()++ # ---- Per-file signal maps ----+ self.dead_code_by_file = build_dead_code_map(dead_code_report)+ self.decisions_by_file, self.decisions_all = build_decision_maps(decision_report)++ # ---- Run bookkeeping ----+ self.semaphore = asyncio.Semaphore(self.config.max_concurrency)+ self.completed_page_summaries: dict[str, str] = {}+ self.completed_ids: set[str] = set()+ self.job_id: str | None = None+ self.file_page_contexts: dict[str, FilePageContext] = {}++ # Selection allow-sets (populated by _compute_selection).+ self.selection: Any = None+ self.code_files: list[Any] = []+ self.sel_file_paths: set[str] = set()+ self.sel_api_paths: set[str] = set()+ self.sel_infra_paths: set[str] = set()+ self.sel_module_groups: list[Any] = []+ self.sel_scc_groups: list[Any] = []+ self.tier1_paths: set[str] = set()+ self.tier2_paths: set[str] = set()++ # ------------------------------------------------------------------+ # Setup+ # ------------------------------------------------------------------++ def _setup_job(self) -> None:+ """Create the resume job and, on resume, seed completed page ids."""+ if self.job_system is None:+ return+ repo_path_str = (+ str(Path(self.repo_path).resolve())+ if self.repo_path+ else str(getattr(self.repo_structure, "root_path", "."))+ )+ # On resume, query the vector store directly — it is the ground truth.+ if self.resume and self.vector_store is not None:+ # Note: caller drives this synchronously enough; resume seeding is+ # awaited in execute() to keep __init__ side-effect free.+ pass+ self.job_id = self.job_system.create_job(+ repo_path_str,+ self.config,+ self.gen._provider.provider_name,+ self.gen._provider.model_name,+ )++ async def _seed_resume(self) -> None:+ if (+ self.job_system is not None+ and self.resume+ and self.vector_store is not None+ ):+ self.completed_ids = await self.vector_store.list_page_ids()+ if self.completed_ids:+ log.info(+ "Resuming generation from vector store",+ already_completed=len(self.completed_ids),+ )++ def _compute_selection(self) -> None:+ """Run the selection subsystem and derive the level allow-sets."""+ code_files = [+ p+ for p in self.parsed_files+ if not p.file_info.is_api_contract+ and not _is_infra_file(p)+ and p.file_info.language in _CODE_LANGUAGES+ ]++ # Near-clone dedupe runs before scoring so clone losers never consume+ # scoring budget. Entry points are never dropped.+ parsed_files_for_selection = self.parsed_files+ if getattr(self.config, "dedupe_near_clones", True):+ drop_paths = _select_clone_representatives(code_files, self.pagerank)+ if drop_paths:+ log.info("page_selection.clone_dedupe", dropped=len(drop_paths))+ code_files = [p for p in code_files if p.file_info.path not in drop_paths]+ parsed_files_for_selection = [+ p for p in self.parsed_files if p.file_info.path not in drop_paths+ ]++ try:+ community_info_map = self.graph_builder.community_info() or {}+ except Exception:+ community_info_map = {}++ from ..selection import SelectionInputs, select_pages++ selection = select_pages(+ SelectionInputs(+ parsed_files=parsed_files_for_selection,+ pagerank=self.pagerank,+ betweenness=self.betweenness,+ community=self.community,+ community_info=community_info_map,+ sccs=list(self.sccs),+ git_meta_map=self.git_meta_map,+ config=self.config,+ )+ )++ self.selection = selection+ self.sel_file_paths = set(selection.file_page_paths)+ self.sel_api_paths = set(selection.api_contract_paths)+ self.sel_infra_paths = set(selection.infra_paths)+ self.sel_module_groups = list(selection.module_groups)+ self.sel_scc_groups = list(selection.scc_groups)++ # Tiered doc generation: split the selected file pages into a full-LLM+ # tier-1 and a deterministic template-only tier-2. When tier1_top_n is+ # None this puts every selected page in tier-1 (no behaviour change).+ self.tier1_paths, self.tier2_paths = partition_file_tiers(+ self.sel_file_paths,+ self.pagerank,+ getattr(self.config, "tier1_top_n", None),+ )++ # Sort code_files for stable level-2 ordering: selected files first+ # (so dep summaries land in the store earliest), then by PageRank desc.+ self.code_files = sorted(+ code_files,+ key=lambda p: (+ p.file_info.path not in self.sel_file_paths,+ not p.file_info.is_entry_point,+ -self.pagerank.get(p.file_info.path, 0.0),+ ),+ )++ def _announce_total(self) -> None:+ counts = self.selection.counts()+ estimated_total = (+ counts["api_contract"]+ + counts["symbol_spotlight"]+ + counts["file_page"]+ + counts["scc_page"]+ + counts["module_page"]+ + int(self.selection.emit_repo_overview)+ + int(self.selection.emit_arch_diagram)+ + counts["infra_page"]+ )+ remaining_total = max(0, estimated_total - len(self.completed_ids))+ if self.on_total_known is not None:+ self.on_total_known(remaining_total)+ if self.job_system is not None and self.job_id is not None:+ self.job_system.start_job(self.job_id, estimated_total)++ # ------------------------------------------------------------------+ # Level runner+ # ------------------------------------------------------------------++ async def run_level(+ self, named_coros: list[tuple[str, Any]], level: int+ ) -> list[GeneratedPage]:+ """Run one level's coroutines under the shared semaphore + embed batch."""+ if self.job_system is not None and self.job_id is not None:+ self.job_system.update_level(self.job_id, level)++ # Pages finished during this level, collected for a single batched+ # embed at the end. Embedding the whole wave in one call amortises the+ # embedder round-trip and the level drains before the next level's RAG+ # search runs, so there is no freshness regression.+ embed_items: list[tuple[str, str, dict]] = []++ async def guarded_named(page_id: str, coro: Any) -> Any:+ try:+ async with self.semaphore:+ result = await coro++ if isinstance(result, GeneratedPage):+ # Summary capture is cheap (string ops) — keep inline so+ # the next page's context assembly sees it immediately.+ self.completed_page_summaries[result.target_path] = overview_summary(+ result.content+ )+ # Progress tick fires the moment the page is ready.+ if self.on_page_done is not None:+ self.on_page_done(result.page_type)+ if self.vector_store is not None:+ embed_items.append(_embed_item(result))+ return result+ except Exception as exc:+ if self.job_system is not None and self.job_id is not None:+ self.job_system.fail_page(self.job_id, page_id, str(exc))+ log.error(+ "page_generation_failed",+ page_id=page_id,+ level=level,+ error=str(exc),+ )+ return exc # return as value so gather works++ tasks = [guarded_named(pid, c) for pid, c in named_coros]+ results = await asyncio.gather(*tasks)+ # Embed the whole level in one batch before declaring it done — the+ # next level's RAG search depends on these landing in the store.+ # Embedding is a RAG enhancement, not load-bearing, so failures are+ # swallowed at debug level.+ if embed_items and self.vector_store is not None:+ try:+ await self.vector_store.embed_batch(embed_items)+ except Exception as e:+ log.debug("rag.embed_batch_failed", count=len(embed_items), error=str(e))+ pages = [r for r in results if isinstance(r, GeneratedPage)]+ if self.job_system is not None and self.job_id is not None:+ for r in pages:+ self.job_system.complete_page(self.job_id, r.page_id)+ return pages++ # ------------------------------------------------------------------+ # Orchestration+ # ------------------------------------------------------------------++ async def execute(self) -> list[GeneratedPage]:+ self._setup_job()+ await self._seed_resume()+ self._compute_selection()+ self._announce_total()++ all_pages: list[GeneratedPage] = []++ # Levels 0 (api_contract) + 1 (symbol_spotlight) share no data+ # dependencies, so they run in one merged batch.+ level01 = _levels.build_level01_coros(self)+ all_pages.extend(await self.run_level(level01, 1))++ # Level 2 (file_page) needs context assembly + topo ordering.+ level2 = await _levels.build_level2_coros(self)+ all_pages.extend(await self.run_level(level2, 2))++ # Level 3 (scc_page).+ all_pages.extend(await self.run_level(_levels.build_level3_coros(self), 3))++ # Level 4 (module_page).+ all_pages.extend(await self.run_level(_levels.build_level4_coros(self), 4))++ # Levels 6 (repo_overview + architecture_diagram), 7 (infra_page),+ # and 8 (onboarding) share no data dependencies — run merged.+ final = (+ _levels.build_level6_coros(self)+ + _levels.build_level7_coros(self)+ + _levels.build_level8_coros(self)+ )+ final_pages = await self.run_level(final, 8)+ # Tag promoted onboarding slots (repo_overview / architecture_diagram).+ self.gen._tag_promoted_pages(final_pages)+ all_pages.extend(final_pages)++ # Post-generation: resolve backtick refs into wiki links + backlinks.+ try:+ from ..interlinking import attach_wiki_links_and_backlinks++ attach_wiki_links_and_backlinks(all_pages, self.parsed_files)+ except Exception as exc:+ log.debug("interlinking.failed", error=str(exc))++ if self.job_system is not None and self.job_id is not None:+ self.job_system.complete_job(self.job_id)++ log.info(+ "Generation complete",+ total_pages=len(all_pages),+ provider=self.gen._provider.provider_name,+ model=self.gen._provider.model_name,+ )+ return all_pages+++def _embed_item(page: GeneratedPage) -> tuple[str, str, dict]:+ """Build the ``(page_id, text, metadata)`` tuple for embedding."""+ summary = overview_summary(page.content)+ return (+ page.page_id,+ page.content,+ {+ "page_type": page.page_type,+ "target_path": page.target_path,+ "content": page.content[:600],+ "summary": summary,+ },+ )+++async def run_generate_all(gen: PageGenerator, **kwargs: Any) -> list[GeneratedPage]:+ """Entry point used by ``PageGenerator.generate_all``."""+ run = _GenerationRun(gen, **kwargs)+ return await run.execute()diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.pynew file mode 100644index 000000000..71441cfdb--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py@@ -0,0 +1,325 @@+"""Per-page-type generation methods, mixed into :class:`PageGenerator`.++Each method assembles a context, renders its Jinja template into a user+prompt, calls the provider, and wraps the response in a ``GeneratedPage``.+They are grouped here (rather than inline on the generator) purely to keep+each module under the project's 400-line ceiling.+"""++from __future__ import annotations++import uuid+from typing import Any++import structlog++from repowise.core.ingestion.models import ParsedFile, RepoStructure++from .. import onboarding as _onboarding+from ..context_assembler import FilePageContext+from ..models import GENERATION_LEVELS, GeneratedPage, compute_source_hash++log = structlog.get_logger(__name__)+++class PerTypeGenerationMixin:+ """Per-type ``generate_*`` methods. Requires the host to provide+ ``_assembler``, ``_render``, ``_call_provider`` and+ ``_build_generated_page`` (all supplied by :class:`PageGenerator`).+ """++ async def generate_file_page(+ self,+ parsed: ParsedFile,+ graph: Any,+ pagerank: dict[str, float],+ betweenness: dict[str, float],+ community: dict[str, int],+ source_bytes: bytes,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_file_page(+ parsed, graph, pagerank, betweenness, community, source_bytes+ )+ user_prompt = self._render("file_page.j2", ctx=ctx)+ response = await self._call_provider(+ "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path+ )+ return self._build_generated_page(+ "file_page",+ parsed.file_info.path,+ f"File: {parsed.file_info.path}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["file_page"],+ )++ async def generate_symbol_spotlight(+ self,+ symbol: Any,+ parsed: ParsedFile,+ pagerank: dict[str, float],+ graph: Any,+ source_map: dict[str, bytes] | None = None,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_symbol_spotlight(+ symbol,+ parsed,+ pagerank,+ graph,+ source_bytes=(source_map or {}).get(parsed.file_info.path, b""),+ )+ user_prompt = self._render("symbol_spotlight.j2", ctx=ctx)+ target = f"{parsed.file_info.path}::{symbol.name}"+ response = await self._call_provider(+ "symbol_spotlight", user_prompt, str(uuid.uuid4()), target_path=target+ )+ return self._build_generated_page(+ "symbol_spotlight",+ f"{parsed.file_info.path}::{symbol.name}",+ f"Symbol: {symbol.qualified_name}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["symbol_spotlight"],+ )++ async def generate_module_page(+ self,+ module_path: str,+ language: str,+ file_contexts: list[FilePageContext],+ graph: Any,+ git_meta_map: dict[str, dict] | None = None,+ page_summaries: dict[str, str] | None = None,+ decision_records: list[dict] | None = None,+ dead_code_findings: list[dict] | None = None,+ external_systems: list[dict] | None = None,+ community_label: str | None = None,+ community_cohesion: float | None = None,+ target_path: str | None = None,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_module_page(+ module_path,+ language,+ file_contexts,+ graph,+ page_summaries=page_summaries,+ git_meta_map=git_meta_map,+ decision_records=decision_records,+ dead_code_findings=dead_code_findings,+ external_systems=external_systems,+ community_label=community_label,+ community_cohesion=community_cohesion,+ )+ module_git_summary = None+ if git_meta_map:+ from collections import Counter++ file_paths = [fc.file_path for fc in file_contexts]+ metas = [git_meta_map[f] for f in file_paths if f in git_meta_map]+ if metas:+ owner_counts = Counter(+ m.get("primary_owner_name") for m in metas if m.get("primary_owner_name")+ )+ most_active = max(metas, key=lambda m: m.get("commit_count_90d", 0))+ module_git_summary = {+ "top_owners": [+ {"name": n, "file_count": c} for n, c in owner_counts.most_common(3)+ ],+ "most_active_file": most_active.get("file_path", ""),+ "most_active_commits_90d": most_active.get("commit_count_90d", 0),+ }+ user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)+ page_target = target_path or module_path+ response = await self._call_provider(+ "module_page", user_prompt, str(uuid.uuid4()), target_path=page_target+ )+ return self._build_generated_page(+ "module_page",+ page_target,+ f"Module: {module_path}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["module_page"],+ )++ async def generate_scc_page(+ self,+ scc_id: str,+ scc_files: list[str],+ file_contexts: list[FilePageContext],+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_scc_page(scc_id, scc_files, file_contexts)+ user_prompt = self._render("scc_page.j2", ctx=ctx)+ response = await self._call_provider(+ "scc_page", user_prompt, str(uuid.uuid4()), target_path=scc_id+ )+ return self._build_generated_page(+ "scc_page",+ scc_id,+ f"Circular Dependency: {scc_id}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["scc_page"],+ )++ async def generate_repo_overview(+ self,+ repo_structure: RepoStructure,+ pagerank: dict[str, float],+ sccs: list[Any],+ community: dict[str, int],+ git_meta_map: dict[str, dict] | None = None,+ graph_builder: Any | None = None,+ repo_name: str | None = None,+ external_systems: list[dict] | None = None,+ decision_records: list[dict] | None = None,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_repo_overview(+ repo_structure,+ pagerank,+ sccs,+ community,+ graph_builder=graph_builder,+ external_systems=external_systems,+ decision_records=decision_records,+ )+ repo_git_summary = None+ if git_meta_map:+ metas = list(git_meta_map.values())+ top_churn = sorted(metas, key=lambda m: m.get("commit_count_90d", 0), reverse=True)[:3]+ oldest = min(+ (m for m in metas if m.get("first_commit_at")),+ key=lambda m: m["first_commit_at"],+ default=None,+ )+ repo_git_summary = {+ "hotspot_count": sum(1 for m in metas if m.get("is_hotspot")),+ "stable_count": sum(1 for m in metas if m.get("is_stable")),+ "top_churn_files": [m.get("file_path", "") for m in top_churn],+ "oldest_file": oldest.get("file_path", "") if oldest else "",+ "oldest_file_age_days": oldest.get("age_days", 0) if oldest else 0,+ }+ if not repo_name:+ repo_name = getattr(repo_structure, "name", None) or "repo"+ user_prompt = self._render("repo_overview.j2", ctx=ctx, repo_git_summary=repo_git_summary)+ response = await self._call_provider(+ "repo_overview", user_prompt, str(uuid.uuid4()), target_path=repo_name+ )+ return self._build_generated_page(+ "repo_overview",+ repo_name,+ f"Repository Overview: {repo_name}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["repo_overview"],+ )++ async def generate_architecture_diagram(+ self,+ graph: Any,+ pagerank: dict[str, float],+ community: dict[str, int],+ sccs: list[Any],+ repo_name: str,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_architecture_diagram(+ graph, pagerank, community, sccs, repo_name+ )+ user_prompt = self._render("architecture_diagram.j2", ctx=ctx)+ response = await self._call_provider(+ "architecture_diagram", user_prompt, str(uuid.uuid4()), target_path=repo_name+ )+ return self._build_generated_page(+ "architecture_diagram",+ repo_name,+ f"Architecture Diagram: {repo_name}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["architecture_diagram"],+ )++ async def generate_api_contract(+ self,+ parsed: ParsedFile,+ source_bytes: bytes,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_api_contract(parsed, source_bytes)+ user_prompt = self._render("api_contract.j2", ctx=ctx)+ response = await self._call_provider(+ "api_contract", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path+ )+ return self._build_generated_page(+ "api_contract",+ parsed.file_info.path,+ f"API Contract: {parsed.file_info.path}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["api_contract"],+ )++ async def generate_onboarding_page(+ self,+ spec: _onboarding.SubkindSpec,+ signals: _onboarding.OnboardingSignals,+ ) -> GeneratedPage | None:+ """Generate one onboarding page from a registered subkind spec.++ Returns ``None`` when the subkind's gate fails (``build_context``+ returned ``None``) — the slot is silently skipped for this repo.+ """+ ctx = spec.build_context(signals)+ if ctx is None:+ log.debug("onboarding.gate_skipped", slot=spec.slot)+ return None++ template_name = f"onboarding/{spec.template}"+ user_prompt = self._render(template_name, ctx=ctx, slot=spec.slot)+ target = _onboarding.target_path(spec.slot)+ response = await self._call_provider(+ "onboarding", user_prompt, str(uuid.uuid4()), target_path=target+ )+ page = self._build_generated_page(+ "onboarding",+ target,+ spec.title,+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["onboarding"],+ )+ # Subkind discriminator lives in metadata; page_type alone is shared+ # across all six generated onboarding slots.+ page.metadata["subkind"] = spec.slot+ page.metadata["onboarding_slot"] = spec.slot+ return page++ @staticmethod+ def _tag_promoted_pages(pages: list[GeneratedPage]) -> None:+ """Tag repo_overview / architecture_diagram pages with their slot.++ Mutates each matching page's ``metadata["onboarding_slot"]`` so the+ UI groups them into the Onboarding folder without changing their+ underlying ``page_type``. Idempotent and tolerant of missing pages.+ """+ for page in pages:+ slot = _onboarding.PROMOTED_SLOTS.get(page.page_type)+ if slot is not None:+ page.metadata["onboarding_slot"] = slot++ async def generate_infra_page(+ self,+ parsed: ParsedFile,+ source_bytes: bytes,+ ) -> GeneratedPage:+ ctx = self._assembler.assemble_infra_page(parsed, source_bytes)+ user_prompt = self._render("infra_page.j2", ctx=ctx)+ response = await self._call_provider(+ "infra_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path+ )+ return self._build_generated_page(+ "infra_page",+ parsed.file_info.path,+ f"Infrastructure: {parsed.file_info.path}",+ response,+ compute_source_hash(user_prompt),+ GENERATION_LEVELS["infra_page"],+ )diff --git a/packages/core/src/repowise/core/generation/page_generator/prompts.py b/packages/core/src/repowise/core/generation/page_generator/prompts.pynew file mode 100644index 000000000..7e8e9dd5f--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/prompts.py@@ -0,0 +1,119 @@+"""System prompts and language metadata for the page generator.++System prompts are module-level constants — the same string per page type on+every call. This enables Anthropic server-side prefix caching.+"""++from __future__ import annotations++_LANGUAGE_NAMES = {+ "en": "English",+ "ru": "Russian",+ "es": "Spanish",+ "fr": "French",+ "de": "German",+ "zh": "Chinese",+ "ja": "Japanese",+ "ko": "Korean",+ "it": "Italian",+ "pt": "Portuguese",+ "nl": "Dutch",+ "pl": "Polish",+ "tr": "Turkish",+ "ar": "Arabic",+ "hi": "Hindi",+}++# ---------------------------------------------------------------------------+# System prompts — one per page type (constant strings for prefix caching)+# ---------------------------------------------------------------------------++SYSTEM_PROMPTS: dict[str, str] = {+ "file_page": (+ "You are repowise, an expert technical documentation generator. "+ "Your task is to produce comprehensive, accurate wiki pages from source code. "+ "Output markdown only. Do not include preamble or apologies. "+ "\n"+ "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "+ "state this file's job in one line. Use concrete architectural vocabulary "+ "(orchestrator, entry point, parser, adapter, dispatcher, factory, resolver, "+ "persistence layer, indexer, analyzer, etc.) and name what it produces or "+ "consumes. This sentence is what semantic search and grep both anchor on, "+ "so role keywords must appear at the very front — not buried in paragraph 3. "+ "Bad: 'This file contains the X class which is used by Y.' "+ "Good: 'X is the orchestrator for the indexing pipeline — it sequences "+ "traversal, parsing, graph analysis, git enrichment, and persistence.' "+ "If the user prompt lists Architectural signals (entry point, high PageRank, "+ "bridge), weave them into that first sentence rather than restating them. "+ "Required sections: ## Overview, ## Public API, ## Dependencies, ## Usage Notes."+ ),+ "symbol_spotlight": (+ "You are repowise, an expert technical documentation generator. "+ "Write a detailed spotlight page for a single code symbol. "+ "Output markdown only. "+ "Required sections: ## Purpose, ## Signature, ## Parameters, ## Returns, ## Example Usage."+ ),+ "module_page": (+ "You are repowise, an expert technical documentation generator. "+ "Write a module-level overview page summarising all files in the module. "+ "Output markdown only. "+ "\n"+ "CRITICAL — lead-with-role rule: the FIRST sentence under ## Overview must "+ "state the module's job in the larger system. Use architectural vocabulary "+ "(ingestion subsystem, generation pipeline, persistence layer, transport "+ "adapter, etc.) and name the inputs it consumes and the outputs it produces. "+ "Bad: 'The X module contains 15 files responsible for...' "+ "Good: 'The ingestion module is the entry stage of repowise's indexing "+ "pipeline — it traverses a repository, parses files into ASTs, extracts "+ "symbols, and yields ParsedFile objects for downstream analysis.' "+ "Required sections: ## Overview, ## Public API Summary, ## Architecture Notes."+ ),+ "scc_page": (+ "You are repowise, an expert technical documentation generator. "+ "Document this circular dependency cycle and provide actionable refactoring advice. "+ "Output markdown only. "+ "Required sections: ## Cycle Description, ## Files Involved, ## Why This Exists, "+ "## Refactoring Suggestions."+ ),+ "repo_overview": (+ "You are repowise, an expert technical documentation generator. "+ "Write a high-level repository overview suitable for onboarding new developers. "+ "Output markdown only. "+ "\n"+ "CRITICAL — lead-with-purpose rule: the FIRST sentence under ## Project "+ "Summary must answer 'what does this repository do, end-to-end?' in one "+ "concrete sentence. Name the inputs, the pipeline, and the outputs in "+ "architectural vocabulary. "+ "Bad: 'This repository implements a documentation tool with 15 packages.' "+ "Good: 'Repowise is a codebase documentation engine: it indexes a repository "+ "by traversing files, parsing code into ASTs, analyzing dependencies, and "+ "generating LLM-synthesised wiki pages served via MCP and a web UI.' "+ "Required sections: ## Project Summary, ## Technology Stack, ## Entry Points, ## Architecture."+ ),+ "architecture_diagram": (+ "You are repowise, an expert technical documentation generator. "+ "Generate an architecture overview with a Mermaid diagram. "+ "You MUST include a fenced mermaid block with graph TD showing key dependencies. "+ "Output markdown only."+ ),+ "api_contract": (+ "You are repowise, an expert technical documentation generator. "+ "Document this API contract file for developers integrating with the service. "+ "Output markdown only. "+ "Required sections: ## Overview, ## Endpoints, ## Schemas, ## Authentication, ## Examples."+ ),+ "infra_page": (+ "You are repowise, an expert technical documentation generator. "+ "Document this infrastructure file for DevOps and platform engineers. "+ "Output markdown only. "+ "Required sections: ## Purpose, ## Key Targets/Stages, ## Configuration, ## Operational Notes."+ ),+ "onboarding": (+ "You are repowise, an expert technical documentation generator producing "+ "a single page in a curated Onboarding collection that a new contributor "+ "or LLM agent reads first. "+ "Write concise, navigable prose grounded in the structured signals supplied. "+ "Do not invent file paths, symbol names, or rationale that is not in the context. "+ "Output markdown only — follow the exact section structure the user prompt prescribes."+ ),+}diff --git a/packages/core/src/repowise/core/generation/page_generator/tiering.py b/packages/core/src/repowise/core/generation/page_generator/tiering.pynew file mode 100644index 000000000..0480d9c1a--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/tiering.py@@ -0,0 +1,46 @@+"""Tiered doc-generation partition logic.++Splits the set of selected file pages into two tiers for large repos:++* **Tier-1** — the top ``tier1_top_n`` files by PageRank get full LLM+ generation (the existing, unchanged path).+* **Tier-2** — the long tail is rendered from a deterministic Jinja+ template and embedded for search, with no LLM call.++When ``tier1_top_n`` is ``None`` (the default) every selected file page is+tier-1, so behaviour is identical to the pre-tiering generator.+"""++from __future__ import annotations+++def partition_file_tiers(+ selected_paths: set[str],+ pagerank: dict[str, float],+ tier1_top_n: int | None,+) -> tuple[set[str], set[str]]:+ """Split selected file-page paths into (tier1, tier2) sets.++ Args:+ selected_paths: Allow-set of file paths chosen for a ``file_page``.+ pagerank: File-level PageRank scores (higher = more central).+ tier1_top_n: Cap on the number of full-LLM (tier-1) pages. ``None``+ or a value >= len(selected_paths) puts every page in+ tier-1, exactly reproducing the prior behaviour.++ Returns:+ ``(tier1_paths, tier2_paths)`` — a partition of ``selected_paths``.+ """+ if tier1_top_n is None or tier1_top_n >= len(selected_paths):+ return set(selected_paths), set()+ if tier1_top_n <= 0:+ return set(), set(selected_paths)++ # Rank by PageRank desc, then path for a deterministic tie-break.+ ranked = sorted(+ selected_paths,+ key=lambda p: (-pagerank.get(p, 0.0), p),+ )+ tier1 = set(ranked[:tier1_top_n])+ tier2 = set(ranked[tier1_top_n:])+ return tier1, tier2diff --git a/packages/core/src/repowise/core/generation/page_generator/validation.py b/packages/core/src/repowise/core/generation/page_generator/validation.pynew file mode 100644index 000000000..ffa27b461--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_generator/validation.py@@ -0,0 +1,233 @@+"""LLM-output validation — hallucination detection for generated pages.++Cross-checks backtick-quoted names in LLM output against the symbols,+exports, and imports of the source ``ParsedFile`` so that references the+model invented (but that do not exist in the AST) can be surfaced.+"""++from __future__ import annotations++import re++from repowise.core.ingestion.models import ParsedFile++# Common words that appear in backticks but are not code symbols.+_BACKTICK_SKIP = frozenset(+ {+ # Python builtins & keywords+ "True",+ "False",+ "None",+ "self",+ "cls",+ "super",+ "str",+ "int",+ "float",+ "bool",+ "list",+ "dict",+ "set",+ "tuple",+ "bytes",+ "object",+ "type",+ "Any",+ "Optional",+ "Union",+ "async",+ "await",+ "return",+ "yield",+ "import",+ "from",+ "class",+ "def",+ "if",+ "else",+ "for",+ "while",+ "try",+ "except",+ "raise",+ "with",+ "pass",+ "break",+ "continue",+ "lambda",+ "in",+ "not",+ "and",+ "or",+ "is",+ "del",+ "assert",+ "finally",+ "elif",+ "as",+ "global",+ "nonlocal",+ # JS/TS keywords+ "null",+ "undefined",+ "this",+ "const",+ "let",+ "var",+ "function",+ "export",+ "default",+ "extends",+ "implements",+ "interface",+ "enum",+ "new",+ "typeof",+ "instanceof",+ "void",+ "never",+ "string",+ "number",+ "boolean",+ "symbol",+ "bigint",+ "unknown",+ "readonly",+ "abstract",+ "static",+ "private",+ "protected",+ "public",+ "require",+ "module",+ "exports",+ "Promise",+ "Map",+ "Set",+ "Array",+ "Object",+ "Error",+ "Date",+ "RegExp",+ "JSON",+ "Math",+ "console",+ # Common tool/ecosystem names+ "pip",+ "npm",+ "npx",+ "yarn",+ "pnpm",+ "go",+ "rust",+ "python",+ "node",+ "cargo",+ "uv",+ "git",+ "docker",+ "make",+ # Common framework/lib names the LLM mentions in prose+ "FastAPI",+ "React",+ "Next",+ "Express",+ "Django",+ "Flask",+ "SQLAlchemy",+ "Pydantic",+ "Click",+ "Typer",+ "pytest",+ "asyncio",+ "pathlib",+ "dataclass",+ "dataclasses",+ }+)++# Regex: single-backtick references that look like identifiers.+_BACKTICK_REF_RE = re.compile(r"(?<!`)` *([A-Za-z_]\w*(?:\.\w+)*) *`(?!`)")++# Patterns that indicate the backtick content is a path, command, or+# value rather than a symbol reference — these should never be flagged.+_PATH_OR_CMD_RE = re.compile(+ r"[/\\]" # contains path separator+ r"|\.(?:py|ts|js|json|yaml|yml|toml|md|sh|sql|css|html)$" # file extension+ r"|^[a-z][\w-]*$" # all-lowercase with hyphens = CLI command/flag+)+++def _validate_symbol_references(+ content: str,+ parsed: ParsedFile,+) -> list[str]:+ """Cross-check backtick-quoted names in LLM output against actual symbols.++ Returns a list of warning strings for references that don't match any+ known symbol, export, or import in the ParsedFile. Designed to have low+ false-positive rates — only flags references that look like symbol names+ but can't be found anywhere in the file's AST, imports, or source text.+ """+ refs = set(_BACKTICK_REF_RE.findall(content))+ if not refs:+ return []++ # Build the known-names set from AST data+ known: set[str] = set()+ for s in parsed.symbols:+ known.add(s.name)+ known.add(s.qualified_name)+ # Decorator names are valid references (e.g. @app.command("init"))+ for dec in s.decorators:+ # Extract the decorator function name: "@app.command" → "command"+ dec_name = dec.lstrip("@").split("(")[0]+ known.add(dec_name)+ known.add(dec_name.split(".")[-1])+ known.update(parsed.exports)+ for imp in parsed.imports:+ if imp.module_path:+ # Add both the final component and intermediate segments+ parts = imp.module_path.split(".")+ known.update(parts)+ known.update(imp.imported_names)+ # Named bindings from import resolution+ for binding in getattr(imp, "bindings", []):+ known.add(binding.local_name)+ if binding.exported_name:+ known.add(binding.exported_name)++ # Also add all string literals from the source that look like identifiers+ # (catches Click command names, decorator arguments, dict keys, etc.)+ # The source is in the context, but we only have the parsed file here.+ # Use docstring and symbol names as a cheap approximation.+ if (+ hasattr(parsed, "file_info")+ and hasattr(parsed.file_info, "path")+ and parsed.docstring+ ):+ known.update(w for w in parsed.docstring.split() if w.isidentifier())++ warnings: list[str] = []+ for ref in refs:+ if ref in _BACKTICK_SKIP:+ continue+ # Skip short refs (1-2 chars are usually variables like `x`, `i`, `db`)+ if len(ref) <= 2:+ continue+ # Skip anything that looks like a path, file, or CLI command+ if _PATH_OR_CMD_RE.search(ref):+ continue+ # Skip all-uppercase (likely constants from other files: `MAX_RETRIES`)+ if ref.isupper():+ continue+ # Check against known names+ base = ref.split(".")[-1]+ if ref in known or base in known:+ continue+ # Skip if the ref is a substring of any known symbol (covers partial+ # references like `parse` when `parse_file` exists)+ if any(ref in k for k in known if len(k) > len(ref)):+ continue+ warnings.append(ref)+ return warningsdiff --git a/packages/core/src/repowise/core/generation/templates/file_page_tier2.j2 b/packages/core/src/repowise/core/generation/templates/file_page_tier2.j2new file mode 100644index 000000000..9d92d9487--- /dev/null+++ b/packages/core/src/repowise/core/generation/templates/file_page_tier2.j2@@ -0,0 +1,45 @@+{#- Deterministic (no-LLM) tier-2 file page.++ Rendered straight from the assembled FilePageContext for the long tail+ of selected files on large repos. No model call — the content is a+ structured, factual stub built entirely from parsed AST + graph signals,+ and is embedded for search like any other page.+-#}+# {{ ctx.file_path }}++## Overview++{% if ctx.docstring %}{{ ctx.docstring | trim }}+{% else %}`{{ ctx.file_path }}` is a {{ ctx.language }} {% if ctx.is_entry_point %}entry-point {% endif %}{% if ctx.is_test %}test {% endif %}source file in this repository.{% endif %}++{% set public_syms = ctx.symbols | selectattr("visibility", "equalto", "public") | list %}+{% if public_syms %}It exposes {{ public_syms | length }} public symbol{{ "s" if public_syms | length != 1 else "" }}{% if ctx.dependencies %} and depends on {{ ctx.dependencies | length }} internal module{{ "s" if ctx.dependencies | length != 1 else "" }}{% endif %}.{% endif %}++## Public API++{% if public_syms %}+| Symbol | Kind | Signature |+| --- | --- | --- |+{% for s in public_syms -%}+| `{{ s.name }}` | {{ s.kind }} | {{ (s.signature or "") | replace("|", "\\|") | replace("\n", " ") | truncate(120, true, "…") }} |+{% endfor %}+{% else %}+_No public symbols were extracted from this file._+{% endif %}++## Dependencies++{% if ctx.dependencies %}+{% for dep in ctx.dependencies -%}+- `{{ dep }}`+{% endfor %}+{% else %}+_No internal dependencies resolved._+{% endif %}++## Usage Notes++{% if ctx.dependents %}Imported by {{ ctx.dependents | length }} file{{ "s" if ctx.dependents | length != 1 else "" }} in this repository.{% else %}No internal callers were resolved for this file.{% endif %}+{% if ctx.community_label %} Part of the **{{ ctx.community_label }}** module cluster.{% endif %}++<!-- Generated deterministically (tier-2, no LLM). -->diff --git a/packages/core/src/repowise/core/ingestion/git_indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer.pydeleted file mode 100644index 54b85532c..000000000--- a/packages/core/src/repowise/core/ingestion/git_indexer.py+++ /dev/null@@ -1,1070 +0,0 @@-"""Git history indexer for the repowise ingestion pipeline.--Mines git history into the git_metadata table. Uses gitpython for git-operations. Parallelizes per-file git log calls with asyncio.Semaphore(20).--Non-blocking: if git is unavailable or repo has no history, log a warning-and return an empty summary. All downstream features degrade gracefully.-"""--from __future__ import annotations--import asyncio-import contextlib-import enum-import json-import math-import re-import time-from collections import Counter, defaultdict-from collections.abc import Callable-from dataclasses import dataclass-from datetime import UTC, datetime, timedelta-from pathlib import Path-from typing import Any--import structlog--from .languages.registry import REGISTRY as _LANG_REGISTRY--logger = structlog.get_logger(__name__)---class GitIndexTier(enum.StrEnum):- """Public knob for git-indexing depth on large repositories.-- ``FULL`` (the default) preserves today's behavior — last *commit_depth*- commits with co-change accumulation and extended blame. ``ESSENTIAL``- runs only the cheap baseline (last commit, no co-change graph) and is- intended for the fast orchestrator path; the implementation that- actually switches on this tier is delivered in a follow-up phase- addressing large-repo scale. The enum is published in this revision so- plugin authors and downstream tooling can target it now.- """-- ESSENTIAL = "essential"- FULL = "full"--# Silence GitPython's _CatFileContentStream.__del__ ValueError spam.-# When git cat-file streams are GC'd after the subprocess pipe is closed,-# their __del__ tries to drain remaining bytes and hits a closed file.-# This is harmless but floods stderr with tracebacks.-try:- from git.cmd import _CatFileContentStream-- _orig_del = _CatFileContentStream.__del__-- def _quiet_del(self: Any) -> None:- with contextlib.suppress(ValueError, OSError):- _orig_del(self)-- _CatFileContentStream.__del__ = _quiet_del # type: ignore[assignment]-except Exception:- pass # git not installed — nothing to patch--# Commit message prefixes that are ALWAYS skipped (no signal value).-_HARD_SKIP_PREFIXES = ("Merge ",)--# Conventional-commit prefixes normally skipped — but kept if the message-# contains a decision-signal keyword (e.g. "build: migrate from webpack to vite").-_SOFT_SKIP_PREFIXES = ("Bump ", "chore:", "ci:", "style:", "build:", "release:")--# Lightweight subset of decision-signal keywords (mirrors decision_extractor.py).-# Used to rescue soft-skipped commits that carry architectural intent.-_DECISION_SIGNAL_WORDS: frozenset[str] = frozenset(- {- "migrate",- "migration",- "switch to",- "replace",- "refactor",- "adopt",- "introduce",- "deprecate",- "remove",- "upgrade",- "rewrite",- "extract",- "convert",- "transition",- }-)--_SKIP_AUTHORS = ("dependabot", "renovate", "github-actions")-_MIN_MESSAGE_LEN = 12--# Default per-file commit history depth.-_DEFAULT_COMMIT_LIMIT: int = 500--# Per-file persisted contributor / commit fan-out. Previously hard-coded-# to 5 / 10 inline, which silently hid co-owners and meaningful history on-# multi-team modules. 50 is generous enough that any realistic UI surface-# can render the full list while keeping the JSON blob bounded.-_MAX_TOP_AUTHORS: int = 50-_MAX_SIGNIFICANT_COMMITS: int = 50--# Co-change pair extraction widens the window because individual files-# may only co-change a handful of times in 500 commits — well below the-# ``min_count`` threshold. On low-churn repos the 500-commit window-# produced 0 co-change pairs every run; 2000 commits captures enough-# history for the decay-weighted score to clear the bar without-# meaningfully blowing up wall-clock time (single `git log` call).-_DEFAULT_CO_CHANGE_COMMIT_LIMIT: int = 2000--# Minimum decay-weighted co-occurrence weight for a pair to be recorded.-# Was 3 historically; on repos with sparse change history (libraries,-# stable services) that produced empty co-change tables. Two recent-# co-changes is enough signal to surface in the UI, and the dashboard-# already sorts partners by weight so the ranking is unaffected.-_DEFAULT_CO_CHANGE_MIN_COUNT: int = 2--# Commits that touch a very large number of files (mass renames,-# copyright header sweeps, code-mod runs) produce O(N^2) pairs and-# contribute no useful co-change signal. Skip pair generation for any-# commit above this threshold. The decay-weighted score already-# de-prioritises mass-edit commits, but the pairs are materialised-# first — for a worst-case 500 files/commit × 2000 commits run that's-# 250M pairs ≈ 16 GB RAM. The cap is a memory safeguard, not a-# correctness change: typical commits sit well under 20 files.-_MAX_FILES_PER_COMMIT_FOR_COCHANGE: int = 200--# Commit message classification regexes (Phase 2.2).-_COMMIT_CATEGORIES: dict[str, re.Pattern[str]] = {- "feature": re.compile(- r"\b(add|implement|introduce|create|new|feat)\b",- re.IGNORECASE,- ),- "refactor": re.compile(- r"\b(refactor|restructure|cleanup|clean.up|rename|reorganize|extract|simplify|move)\b",- re.IGNORECASE,- ),- "fix": re.compile(- r"\b(fix|bug|patch|hotfix|revert|regression|broken|crash|error)\b",- re.IGNORECASE,- ),- "dependency": re.compile(- r"\b(upgrade|bump|update.dep|migrate.to|switch.to|dependency|dependencies)\b",- re.IGNORECASE,- ),-}--# Co-change temporal decay: half-life ~125 days (lambda for exp(-t/tau)).-_CO_CHANGE_DECAY_TAU: float = 180.0--# Hotspot temporal decay: half-life for exponentially weighted churn score.-HOTSPOT_HALFLIFE_DAYS: float = 180.0--# Regex to extract PR/MR numbers from commit messages.-# Matches: "#123", "Merge pull request #456", "(#789)", "!42" (GitLab MR)-_PR_NUMBER_RE = re.compile(r"(?:pull request |)\#(\d+)|\(#(\d+)\)|!(\d+)")--# Allowlist of extensions for which per-file git indexing (blame, commit-# history, hotspot/stable classification) is worth running. Anything NOT in-# this set is skipped — data, config, markup, dotfiles, and binaries add no-# documentation value, and git blame on large JSON/YAML files is very slow.-# Co-change detection still runs across ALL tracked files regardless.-# Derived from the centralised LanguageRegistry — all code language extensions.-_CODE_EXTENSIONS: frozenset[str] = _LANG_REGISTRY.all_code_extensions()--# Files larger than this skip git blame. blame is O(lines) and blocks the-# executor thread — for large files the commit-based ownership estimate is-# used as a fallback instead.-_MAX_BLAME_SIZE_BYTES: int = 100 * 1024 # 100 KB--# Maximum seconds to wait for a single file's git indexing. If exceeded the-# file is recorded with whatever data was collected before the timeout and the-# semaphore slot is released so other files can proceed.-_FILE_INDEX_TIMEOUT_SECS: float = 45.0---@dataclass-class _CommitRec:- """Lightweight commit record parsed from ``git log --numstat``."""-- sha: str- author_name: str- author_email: str- ts: int # unix epoch- is_merge: bool- subject: str- added: int = 0- deleted: int = 0---_RENAME_RE = re.compile(r"\{(.+?) => (.+?)\}")---def _extract_rename_paths(stat_path: str, known_paths: set[str]) -> tuple[str | None, str | None]:- """Extract old/new paths from a git numstat rename line and add to *known_paths*.-- Git ``--numstat`` with ``--follow`` emits rename lines like::-- 10\t5\t{old => new}/shared_suffix- 10\t5\told_dir/{old_name => new_name}.py-- This helper parses both forms, adds both expanded paths to *known_paths*,- and returns ``(old_path, new_path)`` so the caller can attribute churn to- the correct file. Returns ``(None, None)`` if the pattern is not found.- """- m = _RENAME_RE.search(stat_path)- if m:- prefix = stat_path[: m.start()]- suffix = stat_path[m.end() :]- old_path = prefix + m.group(1) + suffix- new_path = prefix + m.group(2) + suffix- known_paths.add(old_path)- known_paths.add(new_path)- return old_path, new_path- return None, None---def _should_skip_index(file_path: str) -> bool:- """Return True for files where per-file git indexing should be skipped.-- Uses an allowlist: only files with known source-code extensions are indexed.- Everything else (data, config, markup, dotfiles, binaries) is skipped.- """- return Path(file_path).suffix.lower() not in _CODE_EXTENSIONS---@dataclass-class GitIndexSummary:- files_indexed: int- hotspots: int- stable_files: int- duration_seconds: float = 0.0---class GitIndexer:- """Mines git history into the git_metadata table.-- Uses gitpython (already a dependency) for git operations.- Parallelizes per-file git log calls with asyncio.Semaphore(20).-- Non-blocking: if git is unavailable or repo has no history, log a warning- and return an empty summary. All downstream features degrade gracefully.- """-- def __init__(- self,- repo_path: str | Path,- *,- commit_limit: int | None = None,- follow_renames: bool = False,- ) -> None:- self.repo_path = Path(repo_path)- self.commit_limit = commit_limit or _DEFAULT_COMMIT_LIMIT- self.follow_renames = follow_renames-- async def index_repo(- self,- repo_id: str,- on_start: Callable[[int], None] | None = None,- on_file_done: Callable[[], None] | None = None,- on_commit_done: Callable[[], None] | None = None,- on_co_change_start: Callable[[int], None] | None = None,- on_co_change_done: Callable[[], None] | None = None,- ) -> tuple[GitIndexSummary, list[dict]]:- """Full index of all tracked files. Returns summary + list of metadata dicts- ready for bulk upsert.-- Optional progress callbacks (all thread-safe, called from the event loop- or executor threads):- on_start(total) — fired once with the total number of tracked files- on_file_done() — fired after each file is indexed- on_co_change_start(total) — fired once with actual commit count for- co-change analysis- on_commit_done() — fired after each commit is processed during- co-change analysis- on_co_change_done() — fired the moment co-change accumulation- completes (BEFORE per-file git indexing- finishes). Lets callers stop a co_change- phase timer without including the wallclock- cost of the still-running per-file walk.- """- start = time.monotonic()- repo = self._get_repo()- if repo is None:- return GitIndexSummary(0, 0, 0, 0.0), []-- tracked_files = self._get_tracked_files(repo)- if not tracked_files:- return GitIndexSummary(0, 0, 0, 0.0), []-- # Only run expensive per-file indexing (git log + blame) on code files.- # Data/config/markup files are skipped here but still passed to- # _compute_co_changes so co-change relationships remain complete.- indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]-- if on_start is not None:- on_start(len(indexable_files))-- # Use a dedicated executor so timed-out threads don't block- # asyncio.run() cleanup (the default executor waits for ALL threads).- from concurrent.futures import ThreadPoolExecutor-- executor = ThreadPoolExecutor(max_workers=20, thread_name_prefix="git-idx")- semaphore = asyncio.Semaphore(20)- loop = asyncio.get_event_loop()-- # Build a repo-wide commit index in ONE git log subprocess when- # rename-tracking is off (the default). Each per-file worker then- # reads its commits from this shared dict instead of spawning its- # own ``git log -- <file>`` — turning O(files) process spawns- # into O(1) on large repos.- commit_index: dict[str, list[_CommitRec]] = {}- if not self.follow_renames:- from .git_commit_index import load_commit_index-- commit_index = load_commit_index(- repo, self.commit_limit, set(indexable_files)- )-- def _index_one_sync(file_path: str) -> dict:- """Use a per-thread Repo to avoid shared-handle issues on Windows."""- try:- import git as gitpython-- thread_repo = gitpython.Repo(- self.repo_path,- search_parent_directories=True,- )- try:- precomputed = (- commit_index.get(file_path) if commit_index else None- )- return self._index_file(- file_path, thread_repo, precomputed_commits=precomputed- )- finally:- thread_repo.close()- except Exception:- return {"file_path": file_path}-- async def index_one(file_path: str) -> dict:- async with semaphore:- try:- result = await asyncio.wait_for(- loop.run_in_executor(executor, _index_one_sync, file_path),- timeout=_FILE_INDEX_TIMEOUT_SECS,- )- except TimeoutError:- logger.debug(- "Git indexing timed out for file — using partial data",- path=file_path,- timeout=_FILE_INDEX_TIMEOUT_SECS,- )- result = {"file_path": file_path}- except Exception as exc:- logger.debug(- "Git indexing failed for file",- path=file_path,- error=str(exc),- )- result = {"file_path": file_path}- if on_file_done is not None:- on_file_done()- return result-- # Run per-file indexing and co-change analysis in parallel — they are- # independent (co-change only needs tracked_files, not per-file results).- file_tasks = [index_one(fp) for fp in indexable_files]-- async def _co_change_task() -> dict[str, list[dict]]:- # Co-change uses a wider window than per-file history (set- # by ``self.commit_limit``) so low-churn repos surface- # any pairs at all. Defaults live in module constants so- # tests and re-index paths can override without touching- # this call site.- result = await loop.run_in_executor(- executor,- self._compute_co_changes,- repo,- set(tracked_files),- max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),- _DEFAULT_CO_CHANGE_MIN_COUNT,- on_commit_done,- on_co_change_start,- )- # Fire the done callback the instant accumulation finishes so- # any co_change phase timer stops here, NOT after the parallel- # per-file git walk also completes (it can run for many more- # minutes on large repos).- if on_co_change_done is not None:- try:- on_co_change_done()- except Exception:- pass- return result-- metadata_list, co_changes = await asyncio.gather(- asyncio.gather(*file_tasks, return_exceptions=True),- _co_change_task(),- )-- # Abandon any timed-out threads immediately instead of letting- # asyncio.run() block for minutes during default-executor cleanup.- executor.shutdown(wait=False, cancel_futures=True)-- # Filter out failures- results: list[dict] = []- for r in metadata_list:- if isinstance(r, Exception):- logger.warning("Failed to index file", error=str(r))- else:- results.append(r)-- # Merge co-change partners into per-file metadata- for meta in results:- fp = meta["file_path"]- if fp in co_changes:- meta["co_change_partners_json"] = json.dumps(co_changes[fp])-- # Compute percentiles- self._compute_percentiles(results)-- duration = time.monotonic() - start- hotspots = sum(1 for m in results if m.get("is_hotspot", False))- stable = sum(1 for m in results if m.get("is_stable", False))-- summary = GitIndexSummary(- files_indexed=len(results),- hotspots=hotspots,- stable_files=stable,- duration_seconds=duration,- )- # Explicitly close the Repo to shut down the persistent git cat-file- # process. Without this, garbage-collected _CatFileContentStream objects- # try to read from an already-closed pipe and spam ValueError tracebacks.- repo.close()-- logger.info(- "Git indexing complete",- files=summary.files_indexed,- hotspots=summary.hotspots,- stable=summary.stable_files,- duration=f"{summary.duration_seconds:.1f}s",- )- return summary, results-- async def index_changed_files(self, changed_file_paths: list[str]) -> list[dict]:- """Incremental update: re-index only changed files.- Also re-index any file whose co_change_partners include a changed file.- """- repo = self._get_repo()- if repo is None:- return []-- loop = asyncio.get_event_loop()- semaphore = asyncio.Semaphore(20)-- def _index_one_sync(file_path: str) -> dict:- """Use a per-thread Repo to avoid shared-handle issues on Windows."""- try:- import git as gitpython-- thread_repo = gitpython.Repo(- self.repo_path,- search_parent_directories=True,- )- try:- return self._index_file(file_path, thread_repo)- finally:- thread_repo.close()- except Exception:- return {"file_path": file_path}-- async def index_one(file_path: str) -> dict:- async with semaphore:- try:- return await asyncio.wait_for(- loop.run_in_executor(None, _index_one_sync, file_path),- timeout=_FILE_INDEX_TIMEOUT_SECS,- )- except (TimeoutError, Exception) as exc:- logger.debug(- "Git indexing failed for changed file",- path=file_path,- error=str(exc),- )- return {"file_path": file_path}-- tasks = [index_one(fp) for fp in changed_file_paths]- results_raw = await asyncio.gather(*tasks, return_exceptions=True)-- results: list[dict] = []- for r in results_raw:- if isinstance(r, Exception):- logger.warning("Failed to index changed file", error=str(r))- else:- results.append(r)-- repo.close()- return results-- # ------------------------------------------------------------------- # Internal methods- # -------------------------------------------------------------------- def _get_repo(self) -> Any | None:- try:- import git as gitpython-- return gitpython.Repo(self.repo_path, search_parent_directories=True)- except Exception as exc:- logger.warning(- "Git unavailable or not a repository",- path=str(self.repo_path),- error=str(exc),- )- return None-- def _get_tracked_files(self, repo: Any) -> list[str]:- try:- output = repo.git.ls_files()- return [f for f in output.splitlines() if f.strip()]- except Exception as exc:- logger.warning("Failed to list tracked files", error=str(exc))- return []-- def _index_file(- self,- file_path: str,- repo: Any,- precomputed_commits: list[_CommitRec] | None = None,- ) -> dict:- """Index a single file's git history. Runs in executor.-- Uses a single ``git log --numstat`` call to collect commit metadata,- line-churn stats, and merge-commit flag in one subprocess instead of- the previous three separate calls (_get_commits, _get_line_stats,- _get_merge_commit_count).-- When *precomputed_commits* is provided (the default when called- from the batched full-repo path), the per-file ``git log``- subprocess is skipped entirely — eliminating the dominant- process-spawn cost on large repos.- """- now = datetime.now(UTC)- ninety_days_ago = now - timedelta(days=90)- thirty_days_ago = now - timedelta(days=30)- ninety_days_ago_ts = ninety_days_ago.timestamp()- thirty_days_ago_ts = thirty_days_ago.timestamp()-- meta: dict[str, Any] = {- "file_path": file_path,- "commit_count_total": 0,- "commit_count_90d": 0,- "commit_count_30d": 0,- "commit_count_capped": False,- "first_commit_at": None,- "last_commit_at": None,- "primary_owner_name": None,- "primary_owner_email": None,- "primary_owner_commit_pct": None,- "top_authors_json": "[]",- "significant_commits_json": "[]",- "co_change_partners_json": "[]",- "commit_categories_json": "{}",- "is_hotspot": False,- "is_stable": False,- "churn_percentile": 0.0,- "age_days": 0,- # Phase 2 fields- "lines_added_90d": 0,- "lines_deleted_90d": 0,- "avg_commit_size": 0.0,- "recent_owner_name": None,- "recent_owner_commit_pct": None,- "bus_factor": 0,- "contributor_count": 0,- # Phase 3 fields- "original_path": None,- "merge_commit_count_90d": 0,- # Temporal hotspot score (exponentially decayed churn)- "temporal_hotspot_score": 0.0,- }-- orig_path: str | None = None- if precomputed_commits is not None:- commits: list[_CommitRec] = precomputed_commits- else:- try:- # Single git log call: header line + optional numstat lines per commit.- # Format: NUL-delimited record per commit so we can split reliably.- # Fields: sha, author name, author email, unix timestamp, parent SHAs, subject- log_args: list[str] = []- if self.follow_renames:- log_args.append("--follow")- log_args += [- f"-{self.commit_limit}",- "--numstat",- "--format=%x00%H%x1f%an%x1f%ae%x1f%ct%x1f%P%x1f%s",- "--",- file_path,- ]- raw = repo.git.log(*log_args)- except Exception:- return meta-- if not raw.strip():- return meta-- # Parse records — each starts with a \x00 marker line followed by- # zero-or-more numstat lines (added\tdeleted\tpath).- # When --follow is active, older commits may reference previous file- # names so we track all names seen via rename markers ("{old => new}").- known_paths: set[str] = {file_path}- # When --follow is active, seed known_paths with the original path- # so that numstat lines referencing the old name (without a rename- # marker in the log window) are still counted.- if self.follow_renames:- orig_path = self._detect_original_path(file_path, repo)- if orig_path:- known_paths.add(orig_path)- commits = []- current: _CommitRec | None = None-- for line in raw.splitlines():- if line.startswith("\x00"):- parts = line.lstrip("\x00").split("\x1f")- if len(parts) >= 6:- sha, an, ae, ct, parents, subj = (- parts[0],- parts[1],- parts[2],- parts[3],- parts[4],- parts[5],- )- try:- ts = int(ct)- except ValueError:- ts = 0- current = _CommitRec(- sha=sha,- author_name=an or "unknown",- author_email=ae,- ts=ts,- is_merge=len(parents.split()) > 1,- subject=subj,- )- commits.append(current)- elif current is not None and line.strip():- # numstat line: added\tdeleted\tpath — only count the target file.- # With --follow, git may emit rename lines like- # "10\t5\t{old_dir => new_dir}/file.py" or just the old path.- numstat_parts = line.split("\t")- if len(numstat_parts) >= 3:- stat_path = numstat_parts[2]- # Track renamed paths so we can match them in older commits.- # For rename lines the raw stat_path (with {old => new}) won't- # match known_paths directly — check the expanded new path instead.- match_path = stat_path- if "=>" in stat_path:- _old, _new = _extract_rename_paths(stat_path, known_paths)- match_path = _new or stat_path- if match_path in known_paths or match_path == file_path:- try:- current.added += int(numstat_parts[0]) if numstat_parts[0] != "-" else 0- current.deleted += (- int(numstat_parts[1]) if numstat_parts[1] != "-" else 0- )- except ValueError:- pass-- if not commits:- return meta-- meta["commit_count_total"] = len(commits)- meta["commit_count_capped"] = len(commits) >= self.commit_limit-- try:- timestamps = [c.ts for c in commits if c.ts > 0]- if timestamps:- first_ts = min(timestamps)- last_ts = max(timestamps)- meta["first_commit_at"] = datetime.fromtimestamp(first_ts, tz=UTC)- meta["last_commit_at"] = datetime.fromtimestamp(last_ts, tz=UTC)- meta["age_days"] = (now - datetime.fromtimestamp(first_ts, tz=UTC)).days-- author_counts: Counter[str] = Counter()- author_emails: dict[str, str] = {}- recent_author_counts: Counter[str] = Counter()-- for c in commits:- is_recent_90 = c.ts >= ninety_days_ago_ts- is_recent_30 = c.ts >= thirty_days_ago_ts- if is_recent_90:- meta["commit_count_90d"] += 1- recent_author_counts[c.author_name] += 1- meta["lines_added_90d"] += c.added- meta["lines_deleted_90d"] += c.deleted- if c.is_merge:- meta["merge_commit_count_90d"] += 1- if is_recent_30:- meta["commit_count_30d"] += 1- author_counts[c.author_name] += 1- if c.author_name not in author_emails and c.author_email:- author_emails[c.author_name] = c.author_email-- c90 = meta["commit_count_90d"]- total_churn = meta["lines_added_90d"] + meta["lines_deleted_90d"]- meta["avg_commit_size"] = total_churn / c90 if c90 > 0 else 0.0-- # Temporal hotspot score: exponentially decayed per-commit churn.- # Each commit contributes weight * clamped_lines where weight decays- # with a half-life of HOTSPOT_HALFLIFE_DAYS days from now.- _ln2 = math.log(2)- temporal_score = 0.0- for c in commits:- age_days = max((now.timestamp() - c.ts) / 86400.0, 0.0)- weight = math.exp(-_ln2 * age_days / HOTSPOT_HALFLIFE_DAYS)- lines = min((c.added + c.deleted) / 100.0, 3.0)- temporal_score += weight * lines- meta["temporal_hotspot_score"] = temporal_score-- # Contributor count & bus factor- meta["contributor_count"] = len(author_counts)- total_commits = sum(author_counts.values())- if total_commits > 0:- threshold = total_commits * 0.8- running = 0- bus = 0- for _name, cnt in author_counts.most_common():- running += cnt- bus += 1- if running >= threshold:- break- meta["bus_factor"] = bus-- # Top authors — keep a generous slice (50) so the API can show- # the full contributor surface for a file; the old top-5 cap- # silently hid co-owners on multi-team modules.- top_authors = []- for name, count in author_counts.most_common(_MAX_TOP_AUTHORS):- top_authors.append(- {- "name": name,- "email": author_emails.get(name, ""),- "commit_count": count,- }- )- meta["top_authors_json"] = json.dumps(top_authors)-- if top_authors:- primary = top_authors[0]- meta["primary_owner_name"] = primary["name"]- meta["primary_owner_email"] = primary["email"]- meta["primary_owner_commit_pct"] = (- primary["commit_count"] / total_commits if total_commits > 0 else 0.0- )-- # Recent owner (90d)- if recent_author_counts:- recent_top = recent_author_counts.most_common(1)[0]- meta["recent_owner_name"] = recent_top[0]- recent_total = sum(recent_author_counts.values())- meta["recent_owner_commit_pct"] = (- recent_top[1] / recent_total if recent_total > 0 else 0.0- )-- # Blame ownership (overrides author-based if available).- # Skipped for large files — git blame is O(lines) and can block the- # executor thread for many seconds on files > 100 KB.- try:- file_size = (self.repo_path / file_path).stat().st_size- if file_size <= _MAX_BLAME_SIZE_BYTES:- blame_name, blame_email, blame_pct = self._get_blame_ownership(file_path, repo)- if blame_name:- meta["primary_owner_name"] = blame_name- meta["primary_owner_email"] = blame_email- meta["primary_owner_commit_pct"] = blame_pct- except Exception:- pass # blame is best-effort-- # Significant commits + classification + PR extraction- sig_commits: list[dict[str, Any]] = []- sig_full = False- category_counts: Counter[str] = Counter()- for c in commits:- msg = c.subject[:200]- if not sig_full and self._is_significant_commit(msg, c.author_name):- entry: dict[str, Any] = {- "sha": c.sha[:8],- "date": datetime.fromtimestamp(c.ts, tz=UTC).isoformat() if c.ts else "",- "message": msg,- "author": c.author_name,- }- pr_match = _PR_NUMBER_RE.search(msg)- if pr_match:- pr_num = pr_match.group(1) or pr_match.group(2) or pr_match.group(3)- entry["pr_number"] = int(pr_num)- sig_commits.append(entry)- if len(sig_commits) >= _MAX_SIGNIFICANT_COMMITS:- sig_full = True- # Classify ALL commits for accurate category ratios- for cat, pattern in _COMMIT_CATEGORIES.items():- if pattern.search(msg):- category_counts[cat] += 1- break-- meta["significant_commits_json"] = json.dumps(sig_commits)- meta["commit_categories_json"] = json.dumps(dict(category_counts))-- # Original path detection (rename tracking) — reuse the result- # from the known_paths seeding above to avoid a duplicate subprocess.- if self.follow_renames and orig_path:- meta["original_path"] = orig_path-- # Stable classification- if meta["commit_count_total"] > 10 and meta["commit_count_90d"] == 0:- meta["is_stable"] = True-- except Exception:- logger.debug("git_indexer_partial_failure", file_path=file_path, exc_info=True)-- return meta-- def _detect_original_path(self, file_path: str, repo: Any) -> str | None:- """If --follow reveals the file was renamed, return its earliest prior path."""- try:- raw = repo.git.log(- "--follow",- f"-{self.commit_limit}",- "--format=",- "--name-only",- "--",- file_path,- )- except Exception:- return None-- # Paths appear newest-first; the last distinct path is the original.- prev_path: str | None = None- for line in raw.splitlines():- p = line.strip()- if p and p != file_path:- prev_path = p # keep overwriting — last one is oldest- return prev_path-- def _get_blame_ownership(- self, file_path: str, repo: Any- ) -> tuple[str | None, str | None, float | None]:- """Compute primary owner from git blame (who wrote the most lines)."""- try:- blame = repo.blame("HEAD", file_path)- except Exception:- return None, None, None-- line_counts: Counter[str] = Counter()- emails: dict[str, str] = {}- total_lines = 0-- for commit, lines in blame:- name = commit.author.name or "unknown"- count = len(lines)- line_counts[name] += count- total_lines += count- if name not in emails and commit.author.email:- emails[name] = commit.author.email-- if not line_counts or total_lines == 0:- return None, None, None-- top_name = line_counts.most_common(1)[0][0]- pct = line_counts[top_name] / total_lines- return top_name, emails.get(top_name), pct-- def _is_significant_commit(self, message: str, author: str) -> bool:- """Return True if the commit is considered significant.-- Filtering rules:- 1. Always skip messages shorter than _MIN_MESSAGE_LEN characters.- 2. Always skip merge commits and bot authors (no useful signal).- 3. Conventional-commit prefixes (chore:, ci:, style:, build:,- release:, Bump) are normally skipped — UNLESS the message also- contains a decision-signal keyword (e.g. "build: migrate from- webpack to vite"). This rescues architecturally meaningful- commits that happen to use a low-signal prefix.- """- msg = message.strip()- if len(msg) < _MIN_MESSAGE_LEN:- return False- # Always skip merge commits- for prefix in _HARD_SKIP_PREFIXES:- if msg.startswith(prefix):- return False- # Always skip bot authors- author_lower = author.lower()- for skip in _SKIP_AUTHORS:- if skip in author_lower:- return False- # Soft-skip conventional prefixes unless decision signal present- for prefix in _SOFT_SKIP_PREFIXES:- if msg.startswith(prefix):- msg_lower = msg.lower()- return any(word in msg_lower for word in _DECISION_SIGNAL_WORDS)- return True-- def _compute_co_changes(- self,- repo: Any,- all_files: set[str],- commit_limit: int = _DEFAULT_CO_CHANGE_COMMIT_LIMIT,- min_count: int = _DEFAULT_CO_CHANGE_MIN_COUNT,- on_commit_done: Callable[[], None] | None = None,- on_co_change_start: Callable[[int], None] | None = None,- ) -> dict[str, list[dict]]:- """Walk recent commits and record co-occurrence pairs for tracked files.-- Uses a single ``git log --name-only`` call instead of spawning one- ``git diff`` subprocess per commit — O(1) processes vs O(commit_limit).-- Applies exponential temporal decay so recent co-changes weigh more- than ancient ones. The ``%ct`` format captures commit timestamps.-- on_co_change_start(total) is called once with the actual number of- commits found. on_commit_done() is called after each commit block is- processed. Both are invoked from a thread-pool thread; callers must- ensure thread safety (Rich Progress is thread-safe).- """- pair_scores: defaultdict[tuple[str, str], float] = defaultdict(float)- pair_last_date: dict[tuple[str, str], int] = {} # pair → latest Unix ts- now_ts = time.time()-- try:- # %x00 = commit separator, %ct = committer timestamp (Unix epoch).- raw = repo.git.log(- f"-{commit_limit}",- "--name-only",- "--no-merges",- "--format=%x00%ct",- )- except Exception:- return {}-- # Count actual commits so the caller can set an accurate progress total.- actual_commits = raw.count("\x00")- if on_co_change_start is not None:- on_co_change_start(actual_commits)-- current: set[str] = set()- current_ts: int = 0-- def _flush_commit() -> None:- if len(current) < 2:- return- if len(current) > _MAX_FILES_PER_COMMIT_FOR_COCHANGE:- # Mass-edit commit — skip pair generation entirely (see- # constant docstring). Logged at debug for traceability.- logger.debug(- "co_change_skip_oversized_commit",- files_in_commit=len(current),- threshold=_MAX_FILES_PER_COMMIT_FOR_COCHANGE,- )- return- age_days = max((now_ts - current_ts) / 86400.0, 0.0)- weight = math.exp(-age_days / _CO_CHANGE_DECAY_TAU)- sorted_files = sorted(current)- for i in range(len(sorted_files)):- for j in range(i + 1, len(sorted_files)):- pair = (sorted_files[i], sorted_files[j])- pair_scores[pair] += weight- if pair not in pair_last_date or current_ts > pair_last_date[pair]:- pair_last_date[pair] = current_ts-- for line in raw.splitlines():- if line == "\x00" or line.startswith("\x00"):- # Commit boundary — flush previous, parse timestamp.- _flush_commit()- current = set()- # Extract Unix timestamp after the \x00 marker- ts_part = line.lstrip("\x00").strip()- try:- current_ts = int(ts_part)- except (ValueError, TypeError):- current_ts = 0- if on_commit_done is not None:- on_commit_done()- else:- path = line.strip()- if path and path in all_files:- current.add(path)-- _flush_commit() # final commit-- # Build result: for each file, list partners above threshold.- # min_count is compared against raw weight (decay-adjusted).- # A score of 3.0 roughly equals 3 recent co-changes or many older ones.- result: dict[str, list[dict]] = defaultdict(list)- for (a, b), score in pair_scores.items():- if score >= min_count:- last_ts = pair_last_date.get((a, b), 0)- last_date = (- datetime.fromtimestamp(last_ts, tz=UTC).strftime("%Y-%m-%d")- if last_ts > 0- else None- )- entry_a = {- "file_path": b,- "co_change_count": round(score, 2),- "last_co_change": last_date,- }- entry_b = {- "file_path": a,- "co_change_count": round(score, 2),- "last_co_change": last_date,- }- result[a].append(entry_a)- result[b].append(entry_b)-- # Sort partners by score descending- for fp in result:- result[fp].sort(key=lambda x: x["co_change_count"], reverse=True)-- # Lightweight observability: every audit so far has had to- # answer "why are co-change tables empty?" by guessing. Logging- # the filter funnel lets future debugging look at one log line.- logger.debug(- "co_change_computed",- commits=actual_commits,- tracked_files=len(all_files),- pairs_considered=len(pair_scores),- pairs_above_threshold=sum(1 for s in pair_scores.values() if s >= min_count),- files_with_partners=len(result),- min_count=min_count,- commit_limit=commit_limit,- )-- return dict(result)-- @staticmethod- def _compute_percentiles(metadata_list: list[dict]) -> None:- """Compute churn_percentile and is_hotspot. Mutates in place.-- Primary sort key is temporal_hotspot_score (exponentially decayed churn);- commit_count_90d is used as a tiebreak, matching the SQL PERCENT_RANK path.- """- if not metadata_list:- return-- # Sort by temporal_hotspot_score (primary) then commit_count_90d (tiebreak)- sorted_by_churn = sorted(- range(len(metadata_list)),- key=lambda i: (- metadata_list[i].get("temporal_hotspot_score") or 0.0,- metadata_list[i].get("commit_count_90d", 0),- ),- )-- total = len(metadata_list)- for rank, idx in enumerate(sorted_by_churn):- metadata_list[idx]["churn_percentile"] = rank / total if total > 0 else 0.0-- # Hotspot: top 25% churn (i.e., churn_percentile >= 0.75)- for meta in metadata_list:- commit_90d = meta.get("commit_count_90d", 0)- churn_pct = meta.get("churn_percentile", 0.0)- if churn_pct >= 0.75 and commit_90d > 0:- meta["is_hotspot"] = Truediff --git a/packages/core/src/repowise/core/ingestion/git_indexer/README.md b/packages/core/src/repowise/core/ingestion/git_indexer/README.mdnew file mode 100644index 000000000..a2db8eb97--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/README.md@@ -0,0 +1,59 @@+# `git_indexer`++Mines git history into per-file metadata (ownership, churn, hotspots,+co-change partners) for the ingestion pipeline.++## Purpose++Turn a repository's commit history into the signals the wiki and health+layers consume: commit counts and recency windows, contributor/bus-factor,+significant commits, a temporal hotspot score, and co-change relationships.++## Public API++Imported via `repowise.core.ingestion.git_indexer`:++- `GitIndexer` — the orchestrator (`index_repo`, `index_changed_files`).+- `GitIndexTier` — `FULL` (default) or `ESSENTIAL` indexing depth.+- `GitIndexSummary` — counts + duration returned by `index_repo`.+- `backfill_full_tier(indexer, repo_id, *, job_store=None)` — promote an+ ESSENTIAL index to FULL as a resumable phase.+- Module-level building blocks (unit-tested directly): `index_file`,+ `compute_co_changes`, `compute_percentiles`, `get_blame_ownership`,+ `is_significant_commit`, `detect_original_path`.++## Tiers++| Tier | Per-file history | `git blame` ownership | Co-change walk |+|------|------------------|-----------------------|----------------|+| `FULL` (default) | yes | yes | yes |+| `ESSENTIAL` | yes | no (commit-author fallback) | no |++`ESSENTIAL` is for the fast orchestrator path (`--mode fast`) on very large+repos: it skips the two O(repo) signals so a first index lands quickly, then+`backfill_full_tier` fills in blame + co-change later.++## Internal layout++| Module | Contents |+|--------|----------|+| `tiers.py` | `GitIndexTier` enum + `includes_blame` / `includes_co_change` |+| `_constants.py` | commit-depth defaults, decay half-lives, skip heuristics, GitPython noise patch |+| `records.py` | `_CommitRec`, `GitIndexSummary`, rename/skip path helpers |+| `file_history.py` | `index_file` — per-file parse + base metrics (blame gated by tier) |+| `enrich.py` | blame ownership, commit significance, rename detection, percentiles |+| `co_change.py` | `compute_co_changes` — repo-wide decay-weighted pair walk |+| `indexer.py` | `GitIndexer` class wiring the above; back-compat instance shims |+| `backfill.py` | `backfill_full_tier` resumable ESSENTIAL→FULL promotion |++## Extension points++A downstream indexer can call the module functions directly with its own+executor, or subclass nothing and just pass a different `GitIndexTier`.+`backfill_full_tier` accepts any `JobStore` for checkpoint/resume.++## Tests++`tests/unit/test_git_indexer.py` (per-function behaviour),+`tests/unit/ingestion/test_git_indexer_tiers.py` (tier gating + backfill),+`tests/integration/test_git_intelligence_integration.py` (end-to-end).diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py b/packages/core/src/repowise/core/ingestion/git_indexer/__init__.pynew file mode 100644index 000000000..b6f53e27a--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/__init__.py@@ -0,0 +1,58 @@+"""Git history indexer for the repowise ingestion pipeline.++Mines git history into the git_metadata table. Uses gitpython for git+operations and parallelises per-file git log calls.++Non-blocking: if git is unavailable or the repo has no history, a warning is+logged and an empty summary returned. All downstream features degrade+gracefully.++The indexer is tiered (:class:`GitIndexTier`): ``FULL`` (default) preserves the+historical behaviour exactly, while ``ESSENTIAL`` skips the two expensive+signals — per-file ``git blame`` and the repo-wide co-change walk — for fast+large-repo indexing. :func:`~.backfill.backfill_full_tier` promotes an+ESSENTIAL index to FULL afterwards.++This package was split out of a single 1k-line module; the public import path+(``repowise.core.ingestion.git_indexer``) is unchanged.+"""++from __future__ import annotations++from ._constants import HOTSPOT_HALFLIFE_DAYS+from .backfill import BACKFILL_PHASE, backfill_full_tier+from .co_change import compute_co_changes+from .enrich import (+ compute_percentiles,+ detect_original_path,+ get_blame_ownership,+ is_significant_commit,+)+from .file_history import index_file+from .indexer import GitIndexer+from .records import (+ GitIndexSummary,+ _CommitRec,+ _extract_rename_paths,+ _should_skip_index,+)+from .tiers import GitIndexTier++__all__ = [+ "BACKFILL_PHASE",+ "HOTSPOT_HALFLIFE_DAYS",+ "GitIndexSummary",+ "GitIndexTier",+ "GitIndexer",+ # Re-exported for ``git_commit_index`` which builds the shared commit index.+ "_CommitRec",+ "_extract_rename_paths",+ "_should_skip_index",+ "backfill_full_tier",+ "compute_co_changes",+ "compute_percentiles",+ "detect_original_path",+ "get_blame_ownership",+ "index_file",+ "is_significant_commit",+]diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py b/packages/core/src/repowise/core/ingestion/git_indexer/_constants.pynew file mode 100644index 000000000..81334eeae--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py@@ -0,0 +1,144 @@+"""Module constants, regexes, and the GitPython noise patch for git indexing.++Kept in one place so the per-tier modules share a single source of truth for+the commit-depth defaults, decay half-lives, and skip heuristics.+"""++from __future__ import annotations++import contextlib+import re+from typing import Any++from ..languages.registry import REGISTRY as _LANG_REGISTRY++# Silence GitPython's _CatFileContentStream.__del__ ValueError spam.+# When git cat-file streams are GC'd after the subprocess pipe is closed,+# their __del__ tries to drain remaining bytes and hits a closed file.+# This is harmless but floods stderr with tracebacks.+try:+ from git.cmd import _CatFileContentStream++ _orig_del = _CatFileContentStream.__del__++ def _quiet_del(self: Any) -> None:+ with contextlib.suppress(ValueError, OSError):+ _orig_del(self)++ _CatFileContentStream.__del__ = _quiet_del # type: ignore[assignment]+except Exception:+ pass # git not installed — nothing to patch++# Commit message prefixes that are ALWAYS skipped (no signal value).+_HARD_SKIP_PREFIXES = ("Merge ",)++# Conventional-commit prefixes normally skipped — but kept if the message+# contains a decision-signal keyword (e.g. "build: migrate from webpack to vite").+_SOFT_SKIP_PREFIXES = ("Bump ", "chore:", "ci:", "style:", "build:", "release:")++# Lightweight subset of decision-signal keywords (mirrors decision_extractor.py).+# Used to rescue soft-skipped commits that carry architectural intent.+_DECISION_SIGNAL_WORDS: frozenset[str] = frozenset(+ {+ "migrate",+ "migration",+ "switch to",+ "replace",+ "refactor",+ "adopt",+ "introduce",+ "deprecate",+ "remove",+ "upgrade",+ "rewrite",+ "extract",+ "convert",+ "transition",+ }+)++_SKIP_AUTHORS = ("dependabot", "renovate", "github-actions")+_MIN_MESSAGE_LEN = 12++# Default per-file commit history depth.+_DEFAULT_COMMIT_LIMIT: int = 500++# Per-file persisted contributor / commit fan-out. Previously hard-coded+# to 5 / 10 inline, which silently hid co-owners and meaningful history on+# multi-team modules. 50 is generous enough that any realistic UI surface+# can render the full list while keeping the JSON blob bounded.+_MAX_TOP_AUTHORS: int = 50+_MAX_SIGNIFICANT_COMMITS: int = 50++# Co-change pair extraction widens the window because individual files+# may only co-change a handful of times in 500 commits — well below the+# ``min_count`` threshold. On low-churn repos the 500-commit window+# produced 0 co-change pairs every run; 2000 commits captures enough+# history for the decay-weighted score to clear the bar without+# meaningfully blowing up wall-clock time (single `git log` call).+_DEFAULT_CO_CHANGE_COMMIT_LIMIT: int = 2000++# Minimum decay-weighted co-occurrence weight for a pair to be recorded.+# Was 3 historically; on repos with sparse change history (libraries,+# stable services) that produced empty co-change tables. Two recent+# co-changes is enough signal to surface in the UI, and the dashboard+# already sorts partners by weight so the ranking is unaffected.+_DEFAULT_CO_CHANGE_MIN_COUNT: int = 2++# Commits that touch a very large number of files (mass renames,+# copyright header sweeps, code-mod runs) produce O(N^2) pairs and+# contribute no useful co-change signal. Skip pair generation for any+# commit above this threshold. The decay-weighted score already+# de-prioritises mass-edit commits, but the pairs are materialised+# first — for a worst-case 500 files/commit x 2000 commits run that's+# 250M pairs ~= 16 GB RAM. The cap is a memory safeguard, not a+# correctness change: typical commits sit well under 20 files.+_MAX_FILES_PER_COMMIT_FOR_COCHANGE: int = 200++# Commit message classification regexes (Phase 2.2).+_COMMIT_CATEGORIES: dict[str, re.Pattern[str]] = {+ "feature": re.compile(+ r"\b(add|implement|introduce|create|new|feat)\b",+ re.IGNORECASE,+ ),+ "refactor": re.compile(+ r"\b(refactor|restructure|cleanup|clean.up|rename|reorganize|extract|simplify|move)\b",+ re.IGNORECASE,+ ),+ "fix": re.compile(+ r"\b(fix|bug|patch|hotfix|revert|regression|broken|crash|error)\b",+ re.IGNORECASE,+ ),+ "dependency": re.compile(+ r"\b(upgrade|bump|update.dep|migrate.to|switch.to|dependency|dependencies)\b",+ re.IGNORECASE,+ ),+}++# Co-change temporal decay: half-life ~125 days (lambda for exp(-t/tau)).+_CO_CHANGE_DECAY_TAU: float = 180.0++# Hotspot temporal decay: half-life for exponentially weighted churn score.+HOTSPOT_HALFLIFE_DAYS: float = 180.0++# Regex to extract PR/MR numbers from commit messages.+# Matches: "#123", "Merge pull request #456", "(#789)", "!42" (GitLab MR)+_PR_NUMBER_RE = re.compile(r"(?:pull request |)\#(\d+)|\(#(\d+)\)|!(\d+)")++# Allowlist of extensions for which per-file git indexing (blame, commit+# history, hotspot/stable classification) is worth running. Anything NOT in+# this set is skipped — data, config, markup, dotfiles, and binaries add no+# documentation value, and git blame on large JSON/YAML files is very slow.+# Co-change detection still runs across ALL tracked files regardless.+# Derived from the centralised LanguageRegistry — all code language extensions.+_CODE_EXTENSIONS: frozenset[str] = _LANG_REGISTRY.all_code_extensions()++# Files larger than this skip git blame. blame is O(lines) and blocks the+# executor thread — for large files the commit-based ownership estimate is+# used as a fallback instead.+_MAX_BLAME_SIZE_BYTES: int = 100 * 1024 # 100 KB++# Maximum seconds to wait for a single file's git indexing. If exceeded the+# file is recorded with whatever data was collected before the timeout and the+# semaphore slot is released so other files can proceed.+_FILE_INDEX_TIMEOUT_SECS: float = 45.0diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/backfill.py b/packages/core/src/repowise/core/ingestion/git_indexer/backfill.pynew file mode 100644index 000000000..fb23d4b9a--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/backfill.py@@ -0,0 +1,83 @@+"""Background backfill worker — promote an ESSENTIAL index to FULL.++When a large repo is indexed with :class:`~.tiers.GitIndexTier.ESSENTIAL`, the+expensive signals (per-file blame, repo-wide co-change) are skipped to get the+user a usable index fast. ``backfill_full_tier`` runs those signals afterwards+as a separate, resumable phase.++It is checkpoint-aware: when a :class:`JobStore` is supplied it records a+``git.backfill`` job, advances the cursor as it completes, and marks the job+COMPLETED / FAILED so a crashed backfill can be detected and re-run. The+worker is deliberately a thin scaffold — it re-runs FULL indexing for the repo+— so the resume contract and wiring are exercised now; finer-grained+per-file resume is a follow-up.+"""++from __future__ import annotations++from typing import TYPE_CHECKING++import structlog++from .records import GitIndexSummary+from .tiers import GitIndexTier++if TYPE_CHECKING:+ from repowise.core.persistence._interfaces.job_store import JobStore++ from .indexer import GitIndexer++logger = structlog.get_logger(__name__)++__all__ = ["BACKFILL_PHASE", "backfill_full_tier"]++# Phase name used for the JobStore record so resume detection can find it.+BACKFILL_PHASE = "git.backfill"+++async def backfill_full_tier(+ indexer: GitIndexer,+ repo_id: str,+ *,+ job_store: JobStore | None = None,+) -> tuple[GitIndexSummary, list[dict]]:+ """Run the FULL-tier git signals for *repo_id* and return the result.++ *indexer* may have been constructed at any tier; this forces FULL for the+ duration of the backfill and restores the original tier afterwards. When+ *job_store* is provided the run is bracketed by a resumable job record.+ """+ from repowise.core.persistence._interfaces.job_store import JobState++ job_id: str | None = None+ if job_store is not None:+ job = await job_store.create_job(+ repository_id=repo_id,+ phase=BACKFILL_PHASE,+ metadata={"tier": GitIndexTier.FULL.value},+ )+ job_id = job.id+ await job_store.update_state(job_id, JobState.RUNNING)++ original_tier = indexer.tier+ indexer.tier = GitIndexTier.FULL+ try:+ summary, results = await indexer.index_repo(repo_id)+ except Exception as exc:+ if job_store is not None and job_id is not None:+ await job_store.update_state(job_id, JobState.FAILED, error=str(exc))+ logger.warning("git_backfill_failed", repo_id=repo_id, error=str(exc))+ raise+ finally:+ indexer.tier = original_tier++ if job_store is not None and job_id is not None:+ await job_store.update_state(+ job_id, JobState.COMPLETED, cursor=str(summary.files_indexed)+ )+ logger.info(+ "git_backfill_complete",+ repo_id=repo_id,+ files=summary.files_indexed,+ )+ return summary, resultsdiff --git a/packages/core/src/repowise/core/ingestion/git_indexer/co_change.py b/packages/core/src/repowise/core/ingestion/git_indexer/co_change.pynew file mode 100644index 000000000..311192ec9--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/co_change.py@@ -0,0 +1,154 @@+"""Repo-wide co-change accumulation (FULL-tier signal).++A single ``git log --name-only`` walk records decay-weighted co-occurrence+pairs across all tracked files. This is the second of the two expensive+signals the ESSENTIAL tier defers; the FULL tier (and the backfill worker)+run it.+"""++from __future__ import annotations++import math+import time+from collections import defaultdict+from collections.abc import Callable+from datetime import UTC, datetime+from typing import Any++import structlog++from ._constants import (+ _CO_CHANGE_DECAY_TAU,+ _DEFAULT_CO_CHANGE_COMMIT_LIMIT,+ _DEFAULT_CO_CHANGE_MIN_COUNT,+ _MAX_FILES_PER_COMMIT_FOR_COCHANGE,+)++logger = structlog.get_logger(__name__)++__all__ = ["compute_co_changes"]+++def compute_co_changes(+ repo: Any,+ all_files: set[str],+ commit_limit: int = _DEFAULT_CO_CHANGE_COMMIT_LIMIT,+ min_count: int = _DEFAULT_CO_CHANGE_MIN_COUNT,+ on_commit_done: Callable[[], None] | None = None,+ on_co_change_start: Callable[[int], None] | None = None,+) -> dict[str, list[dict]]:+ """Walk recent commits and record co-occurrence pairs for tracked files.++ Uses a single ``git log --name-only`` call instead of spawning one+ ``git diff`` subprocess per commit — O(1) processes vs O(commit_limit).++ Applies exponential temporal decay so recent co-changes weigh more than+ ancient ones. ``on_co_change_start(total)`` is called once with the actual+ number of commits found; ``on_commit_done()`` after each commit block.+ Both run from a thread-pool thread; callers must ensure thread safety.+ """+ pair_scores: defaultdict[tuple[str, str], float] = defaultdict(float)+ pair_last_date: dict[tuple[str, str], int] = {} # pair → latest Unix ts+ now_ts = time.time()++ try:+ # %x00 = commit separator, %ct = committer timestamp (Unix epoch).+ raw = repo.git.log(+ f"-{commit_limit}",+ "--name-only",+ "--no-merges",+ "--format=%x00%ct",+ )+ except Exception:+ return {}++ actual_commits = raw.count("\x00")+ if on_co_change_start is not None:+ on_co_change_start(actual_commits)++ current: set[str] = set()+ current_ts: int = 0++ def _flush_commit() -> None:+ nonlocal current_ts+ if len(current) < 2:+ return+ if len(current) > _MAX_FILES_PER_COMMIT_FOR_COCHANGE:+ # Mass-edit commit — skip pair generation entirely (see constant+ # docstring). Logged at debug for traceability.+ logger.debug(+ "co_change_skip_oversized_commit",+ files_in_commit=len(current),+ threshold=_MAX_FILES_PER_COMMIT_FOR_COCHANGE,+ )+ return+ age_days = max((now_ts - current_ts) / 86400.0, 0.0)+ weight = math.exp(-age_days / _CO_CHANGE_DECAY_TAU)+ sorted_files = sorted(current)+ for i in range(len(sorted_files)):+ for j in range(i + 1, len(sorted_files)):+ pair = (sorted_files[i], sorted_files[j])+ pair_scores[pair] += weight+ if pair not in pair_last_date or current_ts > pair_last_date[pair]:+ pair_last_date[pair] = current_ts++ for line in raw.splitlines():+ if line == "\x00" or line.startswith("\x00"):+ # Commit boundary — flush previous, parse timestamp.+ _flush_commit()+ current = set()+ ts_part = line.lstrip("\x00").strip()+ try:+ current_ts = int(ts_part)+ except (ValueError, TypeError):+ current_ts = 0+ if on_commit_done is not None:+ on_commit_done()+ else:+ path = line.strip()+ if path and path in all_files:+ current.add(path)++ _flush_commit() # final commit++ # Build result: for each file, list partners above threshold.+ result: dict[str, list[dict]] = defaultdict(list)+ for (a, b), score in pair_scores.items():+ if score >= min_count:+ last_ts = pair_last_date.get((a, b), 0)+ last_date = (+ datetime.fromtimestamp(last_ts, tz=UTC).strftime("%Y-%m-%d")+ if last_ts > 0+ else None+ )+ result[a].append(+ {+ "file_path": b,+ "co_change_count": round(score, 2),+ "last_co_change": last_date,+ }+ )+ result[b].append(+ {+ "file_path": a,+ "co_change_count": round(score, 2),+ "last_co_change": last_date,+ }+ )++ # Sort partners by score descending+ for fp in result:+ result[fp].sort(key=lambda x: x["co_change_count"], reverse=True)++ logger.debug(+ "co_change_computed",+ commits=actual_commits,+ tracked_files=len(all_files),+ pairs_considered=len(pair_scores),+ pairs_above_threshold=sum(1 for s in pair_scores.values() if s >= min_count),+ files_with_partners=len(result),+ min_count=min_count,+ commit_limit=commit_limit,+ )++ return dict(result)diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py b/packages/core/src/repowise/core/ingestion/git_indexer/enrich.pynew file mode 100644index 000000000..1dbf8a320--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/enrich.py@@ -0,0 +1,142 @@+"""FULL-tier enrichment helpers: blame ownership, commit significance,+rename detection, and churn percentiles.++These are pure functions (no GitIndexer instance state) so each can be unit+tested in isolation. The cheap baseline (commit counts, authors, temporal+score) lives in :mod:`file_history`; the expensive blame call and the+significance/percentile logic that only the FULL tier needs live here.+"""++from __future__ import annotations++from collections import Counter+from typing import Any++from ._constants import (+ _DECISION_SIGNAL_WORDS,+ _HARD_SKIP_PREFIXES,+ _MIN_MESSAGE_LEN,+ _SKIP_AUTHORS,+ _SOFT_SKIP_PREFIXES,+)++__all__ = [+ "compute_percentiles",+ "detect_original_path",+ "get_blame_ownership",+ "is_significant_commit",+]+++def detect_original_path(repo: Any, file_path: str, commit_limit: int) -> str | None:+ """If --follow reveals the file was renamed, return its earliest prior path."""+ try:+ raw = repo.git.log(+ "--follow",+ f"-{commit_limit}",+ "--format=",+ "--name-only",+ "--",+ file_path,+ )+ except Exception:+ return None++ # Paths appear newest-first; the last distinct path is the original.+ prev_path: str | None = None+ for line in raw.splitlines():+ p = line.strip()+ if p and p != file_path:+ prev_path = p # keep overwriting — last one is oldest+ return prev_path+++def get_blame_ownership(+ repo: Any, file_path: str+) -> tuple[str | None, str | None, float | None]:+ """Compute primary owner from git blame (who wrote the most lines)."""+ try:+ blame = repo.blame("HEAD", file_path)+ except Exception:+ return None, None, None++ line_counts: Counter[str] = Counter()+ emails: dict[str, str] = {}+ total_lines = 0++ for commit, lines in blame:+ name = commit.author.name or "unknown"+ count = len(lines)+ line_counts[name] += count+ total_lines += count+ if name not in emails and commit.author.email:+ emails[name] = commit.author.email++ if not line_counts or total_lines == 0:+ return None, None, None++ top_name = line_counts.most_common(1)[0][0]+ pct = line_counts[top_name] / total_lines+ return top_name, emails.get(top_name), pct+++def is_significant_commit(message: str, author: str) -> bool:+ """Return True if the commit is considered significant.++ Filtering rules:+ 1. Always skip messages shorter than _MIN_MESSAGE_LEN characters.+ 2. Always skip merge commits and bot authors (no useful signal).+ 3. Conventional-commit prefixes (chore:, ci:, style:, build:,+ release:, Bump) are normally skipped — UNLESS the message also+ contains a decision-signal keyword (e.g. "build: migrate from+ webpack to vite"). This rescues architecturally meaningful+ commits that happen to use a low-signal prefix.+ """+ msg = message.strip()+ if len(msg) < _MIN_MESSAGE_LEN:+ return False+ # Always skip merge commits+ for prefix in _HARD_SKIP_PREFIXES:+ if msg.startswith(prefix):+ return False+ # Always skip bot authors+ author_lower = author.lower()+ for skip in _SKIP_AUTHORS:+ if skip in author_lower:+ return False+ # Soft-skip conventional prefixes unless decision signal present+ for prefix in _SOFT_SKIP_PREFIXES:+ if msg.startswith(prefix):+ msg_lower = msg.lower()+ return any(word in msg_lower for word in _DECISION_SIGNAL_WORDS)+ return True+++def compute_percentiles(metadata_list: list[dict]) -> None:+ """Compute churn_percentile and is_hotspot. Mutates in place.++ Primary sort key is temporal_hotspot_score (exponentially decayed churn);+ commit_count_90d is used as a tiebreak, matching the SQL PERCENT_RANK path.+ """+ if not metadata_list:+ return++ # Sort by temporal_hotspot_score (primary) then commit_count_90d (tiebreak)+ sorted_by_churn = sorted(+ range(len(metadata_list)),+ key=lambda i: (+ metadata_list[i].get("temporal_hotspot_score") or 0.0,+ metadata_list[i].get("commit_count_90d", 0),+ ),+ )++ total = len(metadata_list)+ for rank, idx in enumerate(sorted_by_churn):+ metadata_list[idx]["churn_percentile"] = rank / total if total > 0 else 0.0++ # Hotspot: top 25% churn (i.e., churn_percentile >= 0.75)+ for meta in metadata_list:+ commit_90d = meta.get("commit_count_90d", 0)+ churn_pct = meta.get("churn_percentile", 0.0)+ if churn_pct >= 0.75 and commit_90d > 0:+ meta["is_hotspot"] = Truediff --git a/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py b/packages/core/src/repowise/core/ingestion/git_indexer/file_history.pynew file mode 100644index 000000000..b88ab48ab--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/file_history.py@@ -0,0 +1,340 @@+"""Per-file git history indexing — the ESSENTIAL baseline plus optional+FULL-tier blame enrichment.++``index_file`` parses a single file's commit history (from a precomputed+commit index when available, else a per-file ``git log``) and derives commit+counts, authorship, line churn, the temporal hotspot score, and significant+commits. The expensive ``git blame`` ownership pass is gated by+*include_blame* so the ESSENTIAL tier can skip it.+"""++from __future__ import annotations++import json+import math+from collections import Counter+from datetime import UTC, datetime, timedelta+from pathlib import Path+from typing import Any++import structlog++from ._constants import (+ _COMMIT_CATEGORIES,+ _MAX_BLAME_SIZE_BYTES,+ _MAX_SIGNIFICANT_COMMITS,+ _MAX_TOP_AUTHORS,+ _PR_NUMBER_RE,+ HOTSPOT_HALFLIFE_DAYS,+)+from .enrich import detect_original_path, get_blame_ownership, is_significant_commit+from .records import _CommitRec, _extract_rename_paths++logger = structlog.get_logger(__name__)++__all__ = ["index_file", "new_meta"]+++def new_meta(file_path: str) -> dict[str, Any]:+ """Return the default metadata dict for *file_path* (all fields zeroed)."""+ return {+ "file_path": file_path,+ "commit_count_total": 0,+ "commit_count_90d": 0,+ "commit_count_30d": 0,+ "commit_count_capped": False,+ "first_commit_at": None,+ "last_commit_at": None,+ "primary_owner_name": None,+ "primary_owner_email": None,+ "primary_owner_commit_pct": None,+ "top_authors_json": "[]",+ "significant_commits_json": "[]",+ "co_change_partners_json": "[]",+ "commit_categories_json": "{}",+ "is_hotspot": False,+ "is_stable": False,+ "churn_percentile": 0.0,+ "age_days": 0,+ # Phase 2 fields+ "lines_added_90d": 0,+ "lines_deleted_90d": 0,+ "avg_commit_size": 0.0,+ "recent_owner_name": None,+ "recent_owner_commit_pct": None,+ "bus_factor": 0,+ "contributor_count": 0,+ # Phase 3 fields+ "original_path": None,+ "merge_commit_count_90d": 0,+ # Temporal hotspot score (exponentially decayed churn)+ "temporal_hotspot_score": 0.0,+ }+++def _parse_per_file_log(+ repo: Any,+ file_path: str,+ *,+ commit_limit: int,+ follow_renames: bool,+) -> tuple[list[_CommitRec], str | None]:+ """Run a per-file ``git log --numstat`` and parse it into commit records.++ Returns ``(commits, original_path)``. Only used when no precomputed commit+ index is available (i.e. when rename-tracking is on).+ """+ log_args: list[str] = []+ if follow_renames:+ log_args.append("--follow")+ log_args += [+ f"-{commit_limit}",+ "--numstat",+ "--format=%x00%H%x1f%an%x1f%ae%x1f%ct%x1f%P%x1f%s",+ "--",+ file_path,+ ]+ try:+ raw = repo.git.log(*log_args)+ except Exception:+ return [], None++ if not raw.strip():+ return [], None++ known_paths: set[str] = {file_path}+ orig_path: str | None = None+ if follow_renames:+ orig_path = detect_original_path(repo, file_path, commit_limit)+ if orig_path:+ known_paths.add(orig_path)++ commits: list[_CommitRec] = []+ current: _CommitRec | None = None+ for line in raw.splitlines():+ if line.startswith("\x00"):+ parts = line.lstrip("\x00").split("\x1f")+ if len(parts) >= 6:+ sha, an, ae, ct, parents, subj = parts[:6]+ try:+ ts = int(ct)+ except ValueError:+ ts = 0+ current = _CommitRec(+ sha=sha,+ author_name=an or "unknown",+ author_email=ae,+ ts=ts,+ is_merge=len(parents.split()) > 1,+ subject=subj,+ )+ commits.append(current)+ elif current is not None and line.strip():+ numstat_parts = line.split("\t")+ if len(numstat_parts) >= 3:+ stat_path = numstat_parts[2]+ match_path = stat_path+ if "=>" in stat_path:+ _old, _new = _extract_rename_paths(stat_path, known_paths)+ match_path = _new or stat_path+ if match_path in known_paths or match_path == file_path:+ try:+ current.added += (+ int(numstat_parts[0]) if numstat_parts[0] != "-" else 0+ )+ current.deleted += (+ int(numstat_parts[1]) if numstat_parts[1] != "-" else 0+ )+ except ValueError:+ pass+ return commits, orig_path+++def index_file(+ repo: Any,+ file_path: str,+ *,+ repo_path: Path,+ commit_limit: int,+ follow_renames: bool,+ include_blame: bool = True,+ precomputed_commits: list[_CommitRec] | None = None,+) -> dict:+ """Index a single file's git history. Runs in executor.++ When *precomputed_commits* is provided (the default when called from the+ batched full-repo path), the per-file ``git log`` subprocess is skipped+ entirely — eliminating the dominant process-spawn cost on large repos.++ *include_blame* gates the FULL-tier ``git blame`` ownership pass; the+ ESSENTIAL tier sets it False and falls back to commit-author ownership.+ """+ now = datetime.now(UTC)+ ninety_days_ago_ts = (now - timedelta(days=90)).timestamp()+ thirty_days_ago_ts = (now - timedelta(days=30)).timestamp()++ meta = new_meta(file_path)++ orig_path: str | None = None+ if precomputed_commits is not None:+ commits: list[_CommitRec] = precomputed_commits+ else:+ commits, orig_path = _parse_per_file_log(+ repo,+ file_path,+ commit_limit=commit_limit,+ follow_renames=follow_renames,+ )++ if not commits:+ return meta++ meta["commit_count_total"] = len(commits)+ meta["commit_count_capped"] = len(commits) >= commit_limit++ try:+ timestamps = [c.ts for c in commits if c.ts > 0]+ if timestamps:+ first_ts = min(timestamps)+ last_ts = max(timestamps)+ meta["first_commit_at"] = datetime.fromtimestamp(first_ts, tz=UTC)+ meta["last_commit_at"] = datetime.fromtimestamp(last_ts, tz=UTC)+ meta["age_days"] = (now - datetime.fromtimestamp(first_ts, tz=UTC)).days++ author_counts: Counter[str] = Counter()+ author_emails: dict[str, str] = {}+ recent_author_counts: Counter[str] = Counter()++ for c in commits:+ if c.ts >= ninety_days_ago_ts:+ meta["commit_count_90d"] += 1+ recent_author_counts[c.author_name] += 1+ meta["lines_added_90d"] += c.added+ meta["lines_deleted_90d"] += c.deleted+ if c.is_merge:+ meta["merge_commit_count_90d"] += 1+ if c.ts >= thirty_days_ago_ts:+ meta["commit_count_30d"] += 1+ author_counts[c.author_name] += 1+ if c.author_name not in author_emails and c.author_email:+ author_emails[c.author_name] = c.author_email++ c90 = meta["commit_count_90d"]+ total_churn = meta["lines_added_90d"] + meta["lines_deleted_90d"]+ meta["avg_commit_size"] = total_churn / c90 if c90 > 0 else 0.0++ # Temporal hotspot score: exponentially decayed per-commit churn.+ _ln2 = math.log(2)+ temporal_score = 0.0+ for c in commits:+ age_days = max((now.timestamp() - c.ts) / 86400.0, 0.0)+ weight = math.exp(-_ln2 * age_days / HOTSPOT_HALFLIFE_DAYS)+ lines = min((c.added + c.deleted) / 100.0, 3.0)+ temporal_score += weight * lines+ meta["temporal_hotspot_score"] = temporal_score++ # Contributor count & bus factor+ meta["contributor_count"] = len(author_counts)+ total_commits = sum(author_counts.values())+ if total_commits > 0:+ threshold = total_commits * 0.8+ running = 0+ bus = 0+ for _name, cnt in author_counts.most_common():+ running += cnt+ bus += 1+ if running >= threshold:+ break+ meta["bus_factor"] = bus++ # Top authors — keep a generous slice (50) so the API can show the+ # full contributor surface for a file.+ top_authors = []+ for name, count in author_counts.most_common(_MAX_TOP_AUTHORS):+ top_authors.append(+ {+ "name": name,+ "email": author_emails.get(name, ""),+ "commit_count": count,+ }+ )+ meta["top_authors_json"] = json.dumps(top_authors)++ if top_authors:+ primary = top_authors[0]+ meta["primary_owner_name"] = primary["name"]+ meta["primary_owner_email"] = primary["email"]+ meta["primary_owner_commit_pct"] = (+ primary["commit_count"] / total_commits if total_commits > 0 else 0.0+ )++ # Recent owner (90d)+ if recent_author_counts:+ recent_top = recent_author_counts.most_common(1)[0]+ meta["recent_owner_name"] = recent_top[0]+ recent_total = sum(recent_author_counts.values())+ meta["recent_owner_commit_pct"] = (+ recent_top[1] / recent_total if recent_total > 0 else 0.0+ )++ # Blame ownership (FULL tier only; overrides author-based if available).+ # Skipped for large files — git blame is O(lines) and can block the+ # executor thread for many seconds on files > 100 KB.+ if include_blame:+ try:+ file_size = (repo_path / file_path).stat().st_size+ if file_size <= _MAX_BLAME_SIZE_BYTES:+ blame_name, blame_email, blame_pct = get_blame_ownership(+ repo, file_path+ )+ if blame_name:+ meta["primary_owner_name"] = blame_name+ meta["primary_owner_email"] = blame_email+ meta["primary_owner_commit_pct"] = blame_pct+ except Exception:+ pass # blame is best-effort++ # Significant commits + classification + PR extraction+ sig_commits: list[dict[str, Any]] = []+ sig_full = False+ category_counts: Counter[str] = Counter()+ for c in commits:+ msg = c.subject[:200]+ if not sig_full and is_significant_commit(msg, c.author_name):+ entry: dict[str, Any] = {+ "sha": c.sha[:8],+ "date": datetime.fromtimestamp(c.ts, tz=UTC).isoformat()+ if c.ts+ else "",+ "message": msg,+ "author": c.author_name,+ }+ pr_match = _PR_NUMBER_RE.search(msg)+ if pr_match:+ pr_num = pr_match.group(1) or pr_match.group(2) or pr_match.group(3)+ entry["pr_number"] = int(pr_num)+ sig_commits.append(entry)+ if len(sig_commits) >= _MAX_SIGNIFICANT_COMMITS:+ sig_full = True+ # Classify ALL commits for accurate category ratios+ for cat, pattern in _COMMIT_CATEGORIES.items():+ if pattern.search(msg):+ category_counts[cat] += 1+ break++ meta["significant_commits_json"] = json.dumps(sig_commits)+ meta["commit_categories_json"] = json.dumps(dict(category_counts))++ # Original path detection (rename tracking) — reuse the result from the+ # known_paths seeding above to avoid a duplicate subprocess.+ if follow_renames and orig_path:+ meta["original_path"] = orig_path++ # Stable classification+ if meta["commit_count_total"] > 10 and meta["commit_count_90d"] == 0:+ meta["is_stable"] = True++ except Exception:+ logger.debug("git_indexer_partial_failure", file_path=file_path, exc_info=True)++ return metadiff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.pynew file mode 100644index 000000000..480965e29--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py@@ -0,0 +1,377 @@+"""GitIndexer — orchestrates per-file history + co-change accumulation.++The class wires together the tier modules: :mod:`file_history` for per-file+metadata, :mod:`co_change` for the repo-wide pair walk, and :mod:`enrich` for+percentiles. The :class:`~.tiers.GitIndexTier` passed at construction decides+which expensive signals run (blame, co-change); ESSENTIAL skips both.+"""++from __future__ import annotations++import asyncio+import contextlib+import json+import time+from collections.abc import Callable+from pathlib import Path+from typing import Any++import structlog++from ._constants import (+ _DEFAULT_CO_CHANGE_COMMIT_LIMIT,+ _DEFAULT_CO_CHANGE_MIN_COUNT,+ _DEFAULT_COMMIT_LIMIT,+ _FILE_INDEX_TIMEOUT_SECS,+)+from .co_change import compute_co_changes+from .enrich import compute_percentiles+from .file_history import index_file+from .records import GitIndexSummary, _CommitRec, _should_skip_index+from .tiers import GitIndexTier++logger = structlog.get_logger(__name__)++__all__ = ["GitIndexer"]+++class GitIndexer:+ """Mines git history into the git_metadata table.++ Uses gitpython (already a dependency) for git operations.+ Parallelizes per-file git log calls with asyncio.Semaphore(20).++ Non-blocking: if git is unavailable or repo has no history, log a warning+ and return an empty summary. All downstream features degrade gracefully.++ *tier* selects indexing depth: ``FULL`` (default) preserves historical+ behaviour; ``ESSENTIAL`` skips per-file blame and the co-change walk for+ fast large-repo indexing (backfill the rest later — see :mod:`backfill`).+ """++ def __init__(+ self,+ repo_path: str | Path,+ *,+ commit_limit: int | None = None,+ follow_renames: bool = False,+ tier: GitIndexTier = GitIndexTier.FULL,+ ) -> None:+ self.repo_path = Path(repo_path)+ self.commit_limit = commit_limit or _DEFAULT_COMMIT_LIMIT+ self.follow_renames = follow_renames+ self.tier = tier++ async def index_repo(+ self,+ repo_id: str,+ on_start: Callable[[int], None] | None = None,+ on_file_done: Callable[[], None] | None = None,+ on_commit_done: Callable[[], None] | None = None,+ on_co_change_start: Callable[[int], None] | None = None,+ on_co_change_done: Callable[[], None] | None = None,+ ) -> tuple[GitIndexSummary, list[dict]]:+ """Full index of all tracked files. Returns summary + list of metadata+ dicts ready for bulk upsert.++ Optional progress callbacks (all thread-safe, called from the event+ loop or executor threads):+ on_start(total) — fired once with the total number of tracked files+ on_file_done() — fired after each file is indexed+ on_co_change_start(total) — fired once with actual commit count for+ co-change analysis+ on_commit_done() — fired after each commit is processed during+ co-change analysis+ on_co_change_done() — fired the moment co-change accumulation+ completes (BEFORE per-file git indexing+ finishes).+ """+ start = time.monotonic()+ repo = self._get_repo()+ if repo is None:+ return GitIndexSummary(0, 0, 0, 0.0), []++ tracked_files = self._get_tracked_files(repo)+ if not tracked_files:+ return GitIndexSummary(0, 0, 0, 0.0), []++ # Only run expensive per-file indexing (git log + blame) on code files.+ indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]++ if on_start is not None:+ on_start(len(indexable_files))++ from concurrent.futures import ThreadPoolExecutor++ executor = ThreadPoolExecutor(max_workers=20, thread_name_prefix="git-idx")+ semaphore = asyncio.Semaphore(20)+ loop = asyncio.get_event_loop()++ # Build a repo-wide commit index in ONE git log subprocess when+ # rename-tracking is off (the default). Each per-file worker then reads+ # its commits from this shared dict instead of spawning its own+ # ``git log -- <file>`` — turning O(files) process spawns into O(1).+ commit_index: dict[str, list[_CommitRec]] = {}+ if not self.follow_renames:+ from ..git_commit_index import load_commit_index++ commit_index = load_commit_index(+ repo, self.commit_limit, set(indexable_files)+ )++ include_blame = self.tier.includes_blame++ def _index_one_sync(file_path: str) -> dict:+ """Use a per-thread Repo to avoid shared-handle issues on Windows."""+ try:+ import git as gitpython++ thread_repo = gitpython.Repo(+ self.repo_path, search_parent_directories=True+ )+ try:+ precomputed = commit_index.get(file_path) if commit_index else None+ return index_file(+ thread_repo,+ file_path,+ repo_path=self.repo_path,+ commit_limit=self.commit_limit,+ follow_renames=self.follow_renames,+ include_blame=include_blame,+ precomputed_commits=precomputed,+ )+ finally:+ thread_repo.close()+ except Exception:+ return {"file_path": file_path}++ async def index_one(file_path: str) -> dict:+ async with semaphore:+ try:+ result = await asyncio.wait_for(+ loop.run_in_executor(executor, _index_one_sync, file_path),+ timeout=_FILE_INDEX_TIMEOUT_SECS,+ )+ except TimeoutError:+ logger.debug(+ "Git indexing timed out for file — using partial data",+ path=file_path,+ timeout=_FILE_INDEX_TIMEOUT_SECS,+ )+ result = {"file_path": file_path}+ except Exception as exc:+ logger.debug(+ "Git indexing failed for file", path=file_path, error=str(exc)+ )+ result = {"file_path": file_path}+ if on_file_done is not None:+ on_file_done()+ return result++ file_tasks = [index_one(fp) for fp in indexable_files]++ async def _co_change_task() -> dict[str, list[dict]]:+ # ESSENTIAL tier defers co-change entirely (the expensive repo-wide+ # walk) — return empty and let a FULL backfill fill it in.+ if not self.tier.includes_co_change:+ if on_co_change_done is not None:+ with contextlib.suppress(Exception):+ on_co_change_done()+ return {}+ result = await loop.run_in_executor(+ executor,+ compute_co_changes,+ repo,+ set(tracked_files),+ max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),+ _DEFAULT_CO_CHANGE_MIN_COUNT,+ on_commit_done,+ on_co_change_start,+ )+ if on_co_change_done is not None:+ with contextlib.suppress(Exception):+ on_co_change_done()+ return result++ metadata_list, co_changes = await asyncio.gather(+ asyncio.gather(*file_tasks, return_exceptions=True),+ _co_change_task(),+ )++ # Abandon any timed-out threads immediately instead of letting+ # asyncio.run() block for minutes during default-executor cleanup.+ executor.shutdown(wait=False, cancel_futures=True)++ results: list[dict] = []+ for r in metadata_list:+ if isinstance(r, Exception):+ logger.warning("Failed to index file", error=str(r))+ else:+ results.append(r)++ # Merge co-change partners into per-file metadata+ for meta in results:+ fp = meta["file_path"]+ if fp in co_changes:+ meta["co_change_partners_json"] = json.dumps(co_changes[fp])++ compute_percentiles(results)++ duration = time.monotonic() - start+ hotspots = sum(1 for m in results if m.get("is_hotspot", False))+ stable = sum(1 for m in results if m.get("is_stable", False))++ summary = GitIndexSummary(+ files_indexed=len(results),+ hotspots=hotspots,+ stable_files=stable,+ duration_seconds=duration,+ )+ repo.close()++ logger.info(+ "Git indexing complete",+ tier=self.tier.value,+ files=summary.files_indexed,+ hotspots=summary.hotspots,+ stable=summary.stable_files,+ duration=f"{summary.duration_seconds:.1f}s",+ )+ return summary, results++ async def index_changed_files(self, changed_file_paths: list[str]) -> list[dict]:+ """Incremental update: re-index only changed files."""+ repo = self._get_repo()+ if repo is None:+ return []++ loop = asyncio.get_event_loop()+ semaphore = asyncio.Semaphore(20)+ include_blame = self.tier.includes_blame++ def _index_one_sync(file_path: str) -> dict:+ try:+ import git as gitpython++ thread_repo = gitpython.Repo(+ self.repo_path, search_parent_directories=True+ )+ try:+ return index_file(+ thread_repo,+ file_path,+ repo_path=self.repo_path,+ commit_limit=self.commit_limit,+ follow_renames=self.follow_renames,+ include_blame=include_blame,+ )+ finally:+ thread_repo.close()+ except Exception:+ return {"file_path": file_path}++ async def index_one(file_path: str) -> dict:+ async with semaphore:+ try:+ return await asyncio.wait_for(+ loop.run_in_executor(None, _index_one_sync, file_path),+ timeout=_FILE_INDEX_TIMEOUT_SECS,+ )+ except (TimeoutError, Exception) as exc:+ logger.debug(+ "Git indexing failed for changed file",+ path=file_path,+ error=str(exc),+ )+ return {"file_path": file_path}++ tasks = [index_one(fp) for fp in changed_file_paths]+ results_raw = await asyncio.gather(*tasks, return_exceptions=True)++ results: list[dict] = []+ for r in results_raw:+ if isinstance(r, Exception):+ logger.warning("Failed to index changed file", error=str(r))+ else:+ results.append(r)++ repo.close()+ return results++ # ------------------------------------------------------------------+ # Internal methods+ # ------------------------------------------------------------------++ def _get_repo(self) -> Any | None:+ try:+ import git as gitpython++ return gitpython.Repo(self.repo_path, search_parent_directories=True)+ except Exception as exc:+ logger.warning(+ "Git unavailable or not a repository",+ path=str(self.repo_path),+ error=str(exc),+ )+ return None++ def _get_tracked_files(self, repo: Any) -> list[str]:+ try:+ output = repo.git.ls_files()+ return [f for f in output.splitlines() if f.strip()]+ except Exception as exc:+ logger.warning("Failed to list tracked files", error=str(exc))+ return []++ # ------------------------------------------------------------------+ # Backward-compatible instance shims+ #+ # The per-tier logic now lives in module-level functions (file_history,+ # co_change, enrich) so it can be unit tested directly. These thin methods+ # preserve the historical instance API for existing callers and tests.+ # ------------------------------------------------------------------++ def _index_file(+ self,+ file_path: str,+ repo: Any,+ precomputed_commits: list[_CommitRec] | None = None,+ ) -> dict:+ return index_file(+ repo,+ file_path,+ repo_path=self.repo_path,+ commit_limit=self.commit_limit,+ follow_renames=self.follow_renames,+ include_blame=self.tier.includes_blame,+ precomputed_commits=precomputed_commits,+ )++ def _get_blame_ownership(+ self, file_path: str, repo: Any+ ) -> tuple[str | None, str | None, float | None]:+ from .enrich import get_blame_ownership++ return get_blame_ownership(repo, file_path)++ def _is_significant_commit(self, message: str, author: str) -> bool:+ from .enrich import is_significant_commit++ return is_significant_commit(message, author)++ def _compute_co_changes(+ self,+ repo: Any,+ all_files: set[str],+ commit_limit: int = _DEFAULT_CO_CHANGE_COMMIT_LIMIT,+ min_count: int = _DEFAULT_CO_CHANGE_MIN_COUNT,+ on_commit_done: Callable[[], None] | None = None,+ on_co_change_start: Callable[[int], None] | None = None,+ ) -> dict[str, list[dict]]:+ return compute_co_changes(+ repo, all_files, commit_limit, min_count, on_commit_done, on_co_change_start+ )++ @staticmethod+ def _compute_percentiles(metadata_list: list[dict]) -> None:+ compute_percentiles(metadata_list)diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/records.py b/packages/core/src/repowise/core/ingestion/git_indexer/records.pynew file mode 100644index 000000000..dc3b97397--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/records.py@@ -0,0 +1,76 @@+"""Lightweight data records and path helpers shared across git-index tiers."""++from __future__ import annotations++import re+from dataclasses import dataclass+from pathlib import Path++from ._constants import _CODE_EXTENSIONS++__all__ = [+ "GitIndexSummary",+ "_CommitRec",+ "_extract_rename_paths",+ "_should_skip_index",+]+++@dataclass+class _CommitRec:+ """Lightweight commit record parsed from ``git log --numstat``."""++ sha: str+ author_name: str+ author_email: str+ ts: int # unix epoch+ is_merge: bool+ subject: str+ added: int = 0+ deleted: int = 0+++@dataclass+class GitIndexSummary:+ files_indexed: int+ hotspots: int+ stable_files: int+ duration_seconds: float = 0.0+++_RENAME_RE = re.compile(r"\{(.+?) => (.+?)\}")+++def _extract_rename_paths(+ stat_path: str, known_paths: set[str]+) -> tuple[str | None, str | None]:+ """Extract old/new paths from a git numstat rename line and add to *known_paths*.++ Git ``--numstat`` with ``--follow`` emits rename lines like::++ 10\t5\t{old => new}/shared_suffix+ 10\t5\told_dir/{old_name => new_name}.py++ This helper parses both forms, adds both expanded paths to *known_paths*,+ and returns ``(old_path, new_path)`` so the caller can attribute churn to+ the correct file. Returns ``(None, None)`` if the pattern is not found.+ """+ m = _RENAME_RE.search(stat_path)+ if m:+ prefix = stat_path[: m.start()]+ suffix = stat_path[m.end() :]+ old_path = prefix + m.group(1) + suffix+ new_path = prefix + m.group(2) + suffix+ known_paths.add(old_path)+ known_paths.add(new_path)+ return old_path, new_path+ return None, None+++def _should_skip_index(file_path: str) -> bool:+ """Return True for files where per-file git indexing should be skipped.++ Uses an allowlist: only files with known source-code extensions are indexed.+ Everything else (data, config, markup, dotfiles, binaries) is skipped.+ """+ return Path(file_path).suffix.lower() not in _CODE_EXTENSIONSdiff --git a/packages/core/src/repowise/core/ingestion/git_indexer/tiers.py b/packages/core/src/repowise/core/ingestion/git_indexer/tiers.pynew file mode 100644index 000000000..51ac3cec8--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/git_indexer/tiers.py@@ -0,0 +1,41 @@+"""Git-indexing depth tiers.++``GitIndexTier`` is the public knob that lets large-repo callers trade+completeness for speed. ``FULL`` (the default) preserves the historical+behaviour exactly. ``ESSENTIAL`` runs only the cheap baseline — per-file+commit history with no per-file ``git blame`` and no co-change accumulation —+so the fast orchestrator path can index a 30k-file repo quickly and backfill+the expensive signals later (see :mod:`backfill`).+"""++from __future__ import annotations++import enum++__all__ = ["GitIndexTier"]+++class GitIndexTier(enum.StrEnum):+ """Public knob for git-indexing depth on large repositories."""++ ESSENTIAL = "essential"+ FULL = "full"++ @property+ def includes_blame(self) -> bool:+ """Whether per-file ``git blame`` ownership runs in this tier.++ Blame is O(lines) and the dominant per-file cost; ESSENTIAL skips it+ and falls back to commit-author ownership.+ """+ return self is GitIndexTier.FULL++ @property+ def includes_co_change(self) -> bool:+ """Whether co-change pair accumulation runs in this tier.++ The repo-wide co-change walk is the other expensive signal; ESSENTIAL+ skips it and leaves ``co_change_partners_json`` empty until a FULL+ backfill fills it in.+ """+ return self is GitIndexTier.FULLdiff --git a/packages/core/src/repowise/core/ingestion/graph.py b/packages/core/src/repowise/core/ingestion/graph.pydeleted file mode 100644index 26b246b3a..000000000--- a/packages/core/src/repowise/core/ingestion/graph.py+++ /dev/null@@ -1,929 +0,0 @@-"""Dependency graph builder for the repowise ingestion pipeline.--GraphBuilder constructs a directed graph from ParsedFile objects with two-tiers of nodes:-- File-level nodes:- "file" — every source file- "external" — third-party / unresolvable imports (prefix "external:")-- Symbol-level nodes:- "symbol" — functions, classes, methods, interfaces, etc.- keyed by Symbol.id (e.g. "src/app.py::main")--Edge types:- "imports" — file-to-file import relationship- "defines" — file-to-symbol containment- "has_method" — class-to-method ownership- "calls" — symbol-to-symbol call relationship (with confidence)--After calling build(), graph metrics are available:- pagerank() — dict[path, float]- strongly_connected_components() — list[frozenset[str]]- betweenness_centrality() — dict[path, float]-"""--from __future__ import annotations--import json-from pathlib import Path-from typing import Any--import networkx as nx-import structlog--from .models import ParsedFile-from .resolvers import ResolverContext, resolve_import-from .resolvers.go import read_go_module_path, read_go_modules-from .type_ref_resolution import resolve_type_refs--log = structlog.get_logger(__name__)--_LARGE_REPO_THRESHOLD = 30_000 # nodes — above this, algorithms are expensive--# Path segments that mark a file as low-value for stem-based import resolution.-_LOW_VALUE_PATH_SEGMENTS = frozenset(- {- "tests",- "test",- "_tests",- "__tests__",- "testing",- "test_apps",- "testdata",- "test_data",- "fixtures",- "examples",- "example",- "samples",- "sample",- "scripts",- "benchmarks",- "bench",- "docs",- "doc",- }-)---def _stem_priority(path: str, stem: str) -> tuple[int, int, int, str]:- """Sort key for choosing among files that share an import stem.-- Lower tuples sort first; callers take ``candidates[0]`` as the resolution.- """- path_obj = Path(path)- parts = path_obj.parts- if path_obj.name == "__init__.py":- parent_match = 0- else:- parent_dir = parts[-2].lower() if len(parts) >= 2 else ""- parent_match = 0 if parent_dir == stem else 1- low_value = 1 if any(seg.lower() in _LOW_VALUE_PATH_SEGMENTS for seg in parts) else 0- return (parent_match, low_value, len(parts), path)---class GraphBuilder:- """Build a dependency graph from a collection of ParsedFile objects.-- Usage::-- builder = GraphBuilder()- for parsed in parsed_files:- builder.add_file(parsed)- graph = builder.build()- pr = builder.pagerank()- """-- def __init__(self, repo_path: Path | str | None = None) -> None:- self._graph: nx.DiGraph = nx.DiGraph()- self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile- self._built = False- self._repo_path: Path | None = Path(repo_path) if repo_path else None- self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)-- # Community / flow / metric caches (invalidated on build)- self._community_cache: dict[str, int] | None = None- self._symbol_community_cache: dict[str, int] | None = None- self._community_info_cache: dict[int, Any] | None = None- self._community_algo: str = ""- self._pagerank_cache: dict[str, float] | None = None- self._betweenness_cache: dict[str, float] | None = None- self._symbol_pagerank_cache: dict[str, float] | None = None- self._symbol_betweenness_cache: dict[str, float] | None = None- self._execution_flow_cache: Any | None = None-- def set_tsconfig_resolver(self, resolver: Any) -> None:- """Attach a :class:`TsconfigResolver` for TS/JS path-alias resolution."""- self._tsconfig_resolver = resolver-- # ------------------------------------------------------------------- # Building- # -------------------------------------------------------------------- def add_file(self, parsed: ParsedFile) -> None:- """Register one parsed file and its symbols in the graph."""- path = parsed.file_info.path- self._parsed_files[path] = parsed- self._built = False # invalidate cached metrics-- # --- File node ---- self._graph.add_node(- path,- node_type="file",- language=parsed.file_info.language,- symbol_count=len(parsed.symbols),- has_error=bool(parsed.parse_errors),- is_test=parsed.file_info.is_test,- is_entry_point=parsed.file_info.is_entry_point,- )-- # --- Symbol nodes ---- for sym in parsed.symbols:- self._graph.add_node(- sym.id,- node_type="symbol",- kind=sym.kind,- name=sym.name,- qualified_name=sym.qualified_name,- file_path=path,- start_line=sym.start_line,- end_line=sym.end_line,- visibility=sym.visibility,- is_async=sym.is_async,- language=sym.language,- parent_name=sym.parent_name,- signature=sym.signature,- decorators=sym.decorators,- is_exported_symbol=sym.is_exported_symbol,- )-- # DEFINES edge: file → symbol- self._graph.add_edge(- path,- sym.id,- edge_type="defines",- )-- # HAS_METHOD edge: class/struct → method- if sym.parent_name and sym.kind == "method":- parent_id = f"{path}::{sym.parent_name}"- if parent_id in self._graph:- self._graph.add_edge(- parent_id,- sym.id,- edge_type="has_method",- )-- # --- Synthetic module-level symbol for top-level calls ---- module_sym_id = f"{path}::__module__"- self._graph.add_node(- module_sym_id,- node_type="symbol",- kind="module",- name="__module__",- file_path=path,- start_line=0,- end_line=0,- visibility="private",- language=parsed.file_info.language,- )- self._graph.add_edge(path, module_sym_id, edge_type="defines")-- def build(self, progress: Any | None = None) -> nx.DiGraph:- """Resolve imports and calls, add edges. Returns the finalized graph.-- *progress* is an optional ``ProgressCallback`` (duck-typed). When- provided, sub-phase events ``graph.imports`` / ``graph.heritage`` /- ``graph.calls`` are emitted so the CLI can surface per-file progress- instead of a single opaque "0/1" bar over the whole build.- """- # Invalidate cached metrics- self._community_cache = None- self._symbol_community_cache = None- self._community_info_cache = None- self._community_algo = ""- self._pagerank_cache = None- self._betweenness_cache = None- self._symbol_pagerank_cache = None- self._symbol_betweenness_cache = None- self._execution_flow_cache = None-- # Clear import/call edges but keep structural edges (defines, has_method)- edges_to_remove = [- (u, v)- for u, v, d in self._graph.edges(data=True)- if d.get("edge_type") not in ("defines", "has_method")- ]- self._graph.remove_edges_from(edges_to_remove)-- # Build lookup tables for import resolution- path_set = set(self._parsed_files.keys())- stem_map = self._build_stem_map(path_set)-- # Construct resolver context- go_modules = read_go_modules(self._repo_path)- ctx = ResolverContext(- path_set=path_set,- stem_map=stem_map,- graph=self._graph,- repo_path=self._repo_path,- tsconfig_resolver=self._tsconfig_resolver,- go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),- go_modules=go_modules,- has_sfc_files=any(- p.endswith((".vue", ".svelte", ".astro")) for p in path_set- ),- parsed_files=self._parsed_files,- )-- # --- Phase 1 prelude: language-specific warmups ---- # Some languages need an expensive one-time index built before any- # per-file import resolution can run. Doing it here, under a dedicated- # phase event, keeps the cost out of ``graph.imports`` and surfaces- # a meaningful progress label instead of a stuck per-file bar.- from .graph_warmups import run_warmups-- run_warmups(self._parsed_files, ctx, progress=progress)-- # --- Phase 1: Resolve file-level imports ---- import_targets: dict[str, set[str]] = {} # file → set of imported files-- # Per-language import-resolution timing — surfaces which language is- # actually dominating the import loop on multi-language repos- # (audit #30). Persisted to state.json for after-the-fact analysis.- import time as _t- lang_import_time: dict[str, float] = {}-- if progress:- progress.on_phase_start("graph.imports", len(self._parsed_files))- for path, parsed in self._parsed_files.items():- _lang = parsed.file_info.language- _t0 = _t.monotonic()- file_imports: set[str] = set()- for imp in parsed.imports:- target = resolve_import(imp.module_path, path, parsed.file_info.language, ctx)- if target:- imp.resolved_file = target- file_imports.add(target)- # Aggregate imported_names on parallel edges- if self._graph.has_edge(path, target):- existing = self._graph[path][target].get("imported_names", [])- merged = list(set(existing + imp.imported_names))- self._graph[path][target]["imported_names"] = merged- else:- self._graph.add_edge(- path,- target,- edge_type="imports",- imported_names=list(imp.imported_names),- )- import_targets[path] = file_imports- lang_import_time[_lang] = lang_import_time.get(_lang, 0.0) + (_t.monotonic() - _t0)- if progress:- progress.on_item_done("graph.imports")- if progress:- _phase_done = getattr(progress, "on_phase_done", None)- if _phase_done is not None:- _phase_done("graph.imports")- if lang_import_time:- log.info(- "import_resolution_per_language",- seconds={k: round(v, 2) for k, v in lang_import_time.items()},- )-- # --- Phase 1b: Resolve non-import type references (e.g. C# ctor params) ---- # Lives between import resolution and heritage so the type-use- # edges feed into heritage's import_targets propagation if a- # future language emits them, and so dead-code reachability- # sees every edge before any analysis pass runs.- if progress:- progress.on_phase_start("graph.type_refs", len(self._parsed_files))- type_use_counts = resolve_type_refs(self._parsed_files, ctx, self._graph)- for path in self._parsed_files:- for _, target, data in self._graph.out_edges(path, data=True):- # Treat both real ``using`` imports and synthesised- # type-use edges as import-like for downstream heritage- # / call resolution. Without this, type-use edges would- # only contribute to file-reachability — not to- # ``import_targets`` which gates cross-file call /- # heritage lookups.- if data.get("edge_type") in ("imports", "type_use"):- import_targets.setdefault(path, set()).add(target)- if progress:- _phase_done = getattr(progress, "on_phase_done", None)- if _phase_done is not None:- _phase_done("graph.type_refs")- del type_use_counts # used only for logging inside resolve_type_refs-- # --- Phase 1c: Resolve C# member-access reads ---- # Bind `var x = new T(...); ... x.Prop` style property reads- # to T's defining file. Cuts the largest single bucket of C#- # unused_export false positives (audit #23).- self._resolve_member_reads(progress=progress)-- # --- Phase 2: Resolve heritage (extends/implements) ---- self._resolve_heritage(import_targets, progress=progress)-- # --- Phase 3: Resolve symbol-level calls ---- self._resolve_calls(import_targets, progress=progress)-- self._built = True-- # Count edge types for logging- edge_counts: dict[str, int] = {}- for _, _, d in self._graph.edges(data=True):- et = d.get("edge_type", "imports")- edge_counts[et] = edge_counts.get(et, 0) + 1-- file_nodes = sum(- 1 for _, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"- )- symbol_nodes = sum(- 1 for _, d in self._graph.nodes(data=True) if d.get("node_type") == "symbol"- )-- log.info(- "Graph built",- file_nodes=file_nodes,- symbol_nodes=symbol_nodes,- edges=self._graph.number_of_edges(),- edge_types=edge_counts,- )- return self._graph-- def _resolve_heritage(- self,- import_targets: dict[str, set[str]],- progress: Any | None = None,- ) -> None:- """Resolve heritage relations and add EXTENDS/IMPLEMENTS edges."""- from .heritage_resolver import HeritageResolver-- resolver = HeritageResolver(self._parsed_files, import_targets)- total_resolved = 0-- files_with_heritage = [- (p, pf) for p, pf in self._parsed_files.items() if pf.heritage- ]- if progress:- progress.on_phase_start("graph.heritage", len(files_with_heritage))- for path, parsed in files_with_heritage:- resolved = resolver.resolve_file(path, parsed.heritage)- for rh in resolved:- if rh.child_id in self._graph and rh.parent_id in self._graph:- if not self._graph.has_edge(rh.child_id, rh.parent_id):- self._graph.add_edge(- rh.child_id,- rh.parent_id,- edge_type=rh.edge_type,- confidence=rh.confidence,- )- total_resolved += 1- else:- existing = self._graph[rh.child_id][rh.parent_id]- if rh.confidence > existing.get("confidence", 0):- existing["confidence"] = rh.confidence- if progress:- progress.on_item_done("graph.heritage")-- if progress:- _phase_done = getattr(progress, "on_phase_done", None)- if _phase_done is not None:- _phase_done("graph.heritage")- log.info("Heritage edges resolved", total=total_resolved)-- def _resolve_member_reads(self, progress: Any | None = None) -> None:- """Phase 1c: emit ``reads`` edges for C# property / member access.-- Runs after type-use resolution so the dead-code analyser sees- member access as evidence of reachability. The pass is C#-only- today (the lever is largest there); the helper module is set- up to receive other languages via additional strategies.- """- from .languages.csharp_member_reads import (- build_csharp_type_to_file,- collect_csharp_source_texts,- resolve_csharp_member_reads,- )-- has_csharp = any(- pf.file_info.language == "csharp" for pf in self._parsed_files.values()- )- if not has_csharp:- return-- phase = "graph.member_reads"- if progress:- progress.on_phase_start(phase, None)- try:- cs_texts = collect_csharp_source_texts(self._parsed_files)- type_to_file = build_csharp_type_to_file(self._parsed_files)- added = resolve_csharp_member_reads(self._graph, cs_texts, type_to_file)- log.info("member_read_edges", language="csharp", added=added)- except Exception as exc:- log.warning("member_reads_failed", error=str(exc))- finally:- if progress:- done = getattr(progress, "on_phase_done", None)- if callable(done):- done(phase)-- def _resolve_calls(- self,- import_targets: dict[str, set[str]],- progress: Any | None = None,- ) -> None:- """Run three-tier call resolution and add CALLS edges to the graph."""- from .call_resolver import CallResolver-- resolver = CallResolver(self._parsed_files, import_targets)- total_resolved = 0-- files_with_calls = [- (p, pf) for p, pf in self._parsed_files.items() if pf.calls- ]- if progress:- progress.on_phase_start("graph.calls", len(files_with_calls))- for path, parsed in files_with_calls:- resolved = resolver.resolve_file(path, parsed.calls)- for rc in resolved:- if rc.caller_id in self._graph and rc.callee_id in self._graph:- if not self._graph.has_edge(rc.caller_id, rc.callee_id):- self._graph.add_edge(- rc.caller_id,- rc.callee_id,- edge_type="calls",- confidence=rc.confidence,- )- total_resolved += 1- else:- existing = self._graph[rc.caller_id][rc.callee_id]- if rc.confidence > existing.get("confidence", 0):- existing["confidence"] = rc.confidence- if progress:- progress.on_item_done("graph.calls")-- if progress:- _phase_done = getattr(progress, "on_phase_done", None)- if _phase_done is not None:- _phase_done("graph.calls")- log.info("Call edges resolved", total=total_resolved)-- def graph(self) -> nx.DiGraph:- """Return the graph (building it first if necessary)."""- if not self._built:- self.build()- return self._graph-- # ------------------------------------------------------------------- # Graph metrics- # -------------------------------------------------------------------- def strongly_connected_components(self) -> list[frozenset[str]]:- """Return SCCs as a list of frozensets."""- return [frozenset(scc) for scc in nx.strongly_connected_components(self.file_subgraph())]-- def betweenness_centrality(self) -> dict[str, float]:- """Return betweenness centrality for file nodes (cached)."""- if self._betweenness_cache is not None:- return self._betweenness_cache- g = self.file_subgraph()- n = g.number_of_nodes()- if n == 0:- self._betweenness_cache = {}- return self._betweenness_cache- if n > _LARGE_REPO_THRESHOLD:- k = min(500, n)- self._betweenness_cache = nx.betweenness_centrality(g, k=k, normalized=True)- else:- self._betweenness_cache = nx.betweenness_centrality(g, normalized=True)- return self._betweenness_cache-- def community_detection(self) -> dict[str, int]:- """Assign a community ID to each file node."""- if self._community_cache is not None:- return self._community_cache-- from repowise.core.analysis.communities import detect_file_communities-- try:- assignment, info, algo = detect_file_communities(self._graph)- self._community_cache = assignment- self._community_info_cache = info- self._community_algo = algo- except Exception as exc:- log.warning("community_detection_failed", error=str(exc))- file_nodes = [- n for n, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"- ]- self._community_cache = {n: 0 for n in file_nodes}- self._community_info_cache = {}- self._community_algo = "failed"- return self._community_cache-- def symbol_communities(self) -> dict[str, int]:- """Assign a community ID to each symbol node using call/heritage edges."""- if self._symbol_community_cache is not None:- return self._symbol_community_cache-- from repowise.core.analysis.communities import detect_symbol_communities-- try:- self._symbol_community_cache = detect_symbol_communities(self._graph)- except Exception as exc:- log.warning("symbol_community_detection_failed", error=str(exc))- self._symbol_community_cache = {}- return self._symbol_community_cache-- def community_info(self) -> dict[int, Any]:- """Return metadata for each file-level community."""- if self._community_info_cache is None:- self.community_detection()- return self._community_info_cache or {}-- def execution_flows(self, config: Any | None = None) -> Any:- """Trace execution flows from entry-point symbols (cached when ``config`` is None)."""- from repowise.core.analysis.execution_flows import (- ExecutionFlowReport,- trace_execution_flows,- )-- # Only the no-config path is cached — callers that pass custom- # FlowConfig still get a fresh trace.- if config is None and self._execution_flow_cache is not None:- return self._execution_flow_cache-- file_cd = self.community_detection()- merged_cd: dict[str, int] = dict(file_cd)-- sym_cd = self.symbol_communities()- merged_cd.update(sym_cd)-- for node_id in self._graph.nodes():- if node_id not in merged_cd and "::" in node_id:- file_path = node_id.split("::")[0]- if file_path in file_cd:- merged_cd[node_id] = file_cd[file_path]-- try:- report = trace_execution_flows(self._graph, merged_cd, config)- except Exception as exc:- log.warning("execution_flow_tracing_failed", error=str(exc))- report = ExecutionFlowReport(- total_entry_points_scored=0,- total_flows=0,- flows=[],- )- if config is None:- self._execution_flow_cache = report- return report-- async def compute_metrics_parallel(self) -> None:- """Eagerly populate all metric caches with fan-out parallelism.-- Runs PageRank, betweenness, file/symbol community detection in- parallel via ``asyncio.gather`` + ``asyncio.to_thread`` (the- scipy- and igraph-backed kernels release the GIL during heavy- compute, so true parallelism is achievable). Execution flows then- run after, since they depend on the community caches.-- Calling this is optional — every metric falls back to lazy- computation, so existing call sites keep working unchanged.- """- import asyncio as _asyncio-- await _asyncio.gather(- _asyncio.to_thread(self.pagerank),- _asyncio.to_thread(self.betweenness_centrality),- _asyncio.to_thread(self.symbol_pagerank),- _asyncio.to_thread(self.symbol_betweenness_centrality),- _asyncio.to_thread(self.community_detection),- _asyncio.to_thread(self.symbol_communities),- )- # execution_flows reads from the community caches just primed above.- await _asyncio.to_thread(self.execution_flows)-- # ------------------------------------------------------------------- # Serialisation- # -------------------------------------------------------------------- def to_json(self) -> dict[str, Any]:- """Serialize the graph to a JSON-compatible dict (node-link format)."""- return nx.node_link_data(self.graph())-- async def persist(self, db_path: Path, repo_id: str) -> None:- """Persist the graph to an SQLite database."""- import aiosqlite- import sqlite3-- pr = self.pagerank()- bc = self.betweenness_centrality()- scc_map = self._build_scc_map()- cd = self.community_detection()- g = self.graph()-- async with aiosqlite.connect(db_path) as db:- await db.executescript("""- CREATE TABLE IF NOT EXISTS graph_nodes (- repo_id TEXT NOT NULL,- path TEXT NOT NULL,- language TEXT,- symbol_count INTEGER,- has_error INTEGER,- pagerank REAL,- betweenness REAL,- scc_id INTEGER,- community_id INTEGER DEFAULT 0,- PRIMARY KEY (repo_id, path)- );- CREATE TABLE IF NOT EXISTS graph_edges (- repo_id TEXT NOT NULL,- source_path TEXT NOT NULL,- target_path TEXT NOT NULL,- imported_names TEXT,- PRIMARY KEY (repo_id, source_path, target_path)- );- """)-- try:- await db.execute(- "ALTER TABLE graph_nodes ADD COLUMN community_id INTEGER DEFAULT 0"- )- except sqlite3.OperationalError:- pass-- node_rows = [- (- repo_id,- path,- data.get("language", ""),- data.get("symbol_count", 0),- int(data.get("has_error", False)),- pr.get(path, 0.0),- bc.get(path, 0.0),- scc_map.get(path, 0),- cd.get(path, cd.get(data.get("file_path", ""), 0)),- )- for path, data in g.nodes(data=True)- ]- await db.executemany(- "INSERT OR REPLACE INTO graph_nodes VALUES (?,?,?,?,?,?,?,?,?)",- node_rows,- )-- edge_rows = [- (- repo_id,- src,- dst,- json.dumps(data.get("imported_names", [])),- )- for src, dst, data in g.edges(data=True)- ]- await db.executemany(- "INSERT OR REPLACE INTO graph_edges VALUES (?,?,?,?)",- edge_rows,- )-- await db.commit()-- log.info(- "Graph persisted",- db_path=str(db_path),- nodes=len(node_rows),- edges=len(edge_rows),- )-- # ------------------------------------------------------------------- # Internal helpers- # -------------------------------------------------------------------- def _build_stem_map(self, path_set: set[str]) -> dict[str, list[str]]:- """Map import-stems to candidate file paths, sorted best-first."""- buckets: dict[str, list[str]] = {}- for p in path_set:- path_obj = Path(p)- if path_obj.name == "__init__.py":- parent = path_obj.parent.name- if not parent:- continue- stem = parent.lower()- else:- stem = path_obj.stem.lower()- buckets.setdefault(stem, []).append(p)-- for stem, paths in buckets.items():- paths.sort(key=lambda candidate: _stem_priority(candidate, stem))- return buckets-- # ------------------------------------------------------------------- # Co-change edges- # -------------------------------------------------------------------- def add_co_change_edges(self, git_meta_map: dict, min_count: int = 3) -> int:- """Add co_changes edges from git metadata. Returns count of edges added."""- count = 0- seen: set[tuple[str, str]] = set()-- for file_path, meta in git_meta_map.items():- co_json = meta.get("co_change_partners_json", "[]")- if isinstance(co_json, str):- try:- partners = json.loads(co_json)- except Exception:- partners = []- else:- partners = co_json-- for partner in partners:- partner_path = partner.get("file_path", "")- co_count = partner.get("co_change_count", 0)- if co_count < min_count:- continue- if partner_path not in self._graph:- continue-- pair = tuple(sorted([file_path, partner_path]))- if pair in seen:- continue- seen.add(pair)-- if not self._graph.has_edge(file_path, partner_path) and not self._graph.has_edge(- partner_path, file_path- ):- self._graph.add_edge(- file_path,- partner_path,- edge_type="co_changes",- weight=co_count,- imported_names=[],- )- count += 1-- log.info("Co-change edges added", count=count)- return count-- def update_co_change_edges(self, updated_meta: dict, min_count: int = 3) -> None:- """Remove old co_changes edges for updated files, add new ones."""- edges_to_remove = []- for u, v, data in self._graph.edges(data=True):- if data.get("edge_type") == "co_changes" and (u in updated_meta or v in updated_meta):- edges_to_remove.append((u, v))- self._graph.remove_edges_from(edges_to_remove)- self.add_co_change_edges(updated_meta, min_count)-- # ------------------------------------------------------------------- # Dynamic-hint edges- # -------------------------------------------------------------------- def add_dynamic_edges(self, edges: list) -> None:- """Add dynamic-hint edges to the graph. Each edge is a DynamicEdge."""- for e in edges:- if e.source not in self._graph:- continue- if e.target not in self._graph:- self._graph.add_node(e.target)- sub_type = e.edge_type or "dynamic"- graph_edge_type = (- sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"- )- self._graph.add_edge(- e.source,- e.target,- edge_type=graph_edge_type,- hint_source=e.hint_source,- weight=e.weight,- )-- # ------------------------------------------------------------------- # Framework-aware synthetic edges- # -------------------------------------------------------------------- def add_framework_edges(self, tech_stack: list[str] | None = None) -> int:- """Add synthetic edges for framework-mediated relationships.-- Returns the number of edges added.- """- from .framework_edges import add_framework_edges-- path_set = set(self._parsed_files.keys())- stem_map = self._build_stem_map(path_set)-- go_modules = read_go_modules(self._repo_path)- ctx = ResolverContext(- path_set=path_set,- stem_map=stem_map,- graph=self._graph,- repo_path=self._repo_path,- tsconfig_resolver=self._tsconfig_resolver,- go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),- go_modules=go_modules,- has_sfc_files=any(- p.endswith((".vue", ".svelte", ".astro")) for p in path_set- ),- parsed_files=self._parsed_files,- )-- count = add_framework_edges(self._graph, self._parsed_files, ctx, tech_stack)- if count:- log.info("Framework edges added", count=count)- return count-- def file_subgraph(self) -> nx.DiGraph:- """Return a subgraph containing only file-level nodes and import edges."""- g = self.graph()- file_nodes = [- n for n, d in g.nodes(data=True) if d.get("node_type", "file") in ("file", "external")- ]- sub = g.subgraph(file_nodes).copy()- edges_to_remove = [- (u, v) for u, v, d in sub.edges(data=True) if d.get("edge_type") in ("co_changes",)- ]- sub.remove_edges_from(edges_to_remove)- return sub-- def pagerank(self, alpha: float = 0.85) -> dict[str, float]:- """Return PageRank scores for file nodes only (cached)."""- if self._pagerank_cache is not None:- return self._pagerank_cache- filtered = self.file_subgraph()- if filtered.number_of_nodes() == 0:- self._pagerank_cache = {}- return self._pagerank_cache-- try:- self._pagerank_cache = nx.pagerank(filtered, alpha=alpha)- except nx.PowerIterationFailedConvergence:- log.warning("PageRank did not converge, using uniform scores")- n = filtered.number_of_nodes()- self._pagerank_cache = {node: 1.0 / n for node in filtered.nodes()}- return self._pagerank_cache-- def symbol_subgraph(self) -> nx.DiGraph:- """Return a subgraph of symbol nodes connected by call + heritage edges.-- File-to-symbol ``defines`` edges and class-to-method ``has_method``- ownership edges are dropped so that the resulting centrality- scores reflect call/heritage flow rather than containment.- """- g = self.graph()- symbol_nodes = [- n for n, d in g.nodes(data=True) if d.get("node_type") == "symbol"- ]- sub = g.subgraph(symbol_nodes).copy()- edges_to_remove = [- (u, v)- for u, v, d in sub.edges(data=True)- if d.get("edge_type") not in ("calls", "extends", "implements")- ]- sub.remove_edges_from(edges_to_remove)- return sub-- def symbol_pagerank(self, alpha: float = 0.85) -> dict[str, float]:- """Return PageRank scores for symbol nodes only (cached).-- Computed on the call/heritage symbol subgraph — this is what the- UI's per-symbol "graph metrics" panel reads. Without it every- symbol shows ``Not indexed in graph``.- """- if self._symbol_pagerank_cache is not None:- return self._symbol_pagerank_cache- sub = self.symbol_subgraph()- if sub.number_of_nodes() == 0:- self._symbol_pagerank_cache = {}- return self._symbol_pagerank_cache- try:- self._symbol_pagerank_cache = nx.pagerank(sub, alpha=alpha)- except nx.PowerIterationFailedConvergence:- log.warning("Symbol PageRank did not converge, using uniform scores")- n = sub.number_of_nodes()- self._symbol_pagerank_cache = {node: 1.0 / n for node in sub.nodes()}- return self._symbol_pagerank_cache-- def symbol_betweenness_centrality(self) -> dict[str, float]:- """Return betweenness centrality for symbol nodes (cached)."""- if self._symbol_betweenness_cache is not None:- return self._symbol_betweenness_cache- sub = self.symbol_subgraph()- n = sub.number_of_nodes()- if n == 0:- self._symbol_betweenness_cache = {}- return self._symbol_betweenness_cache- if n > _LARGE_REPO_THRESHOLD:- k = min(500, n)- self._symbol_betweenness_cache = nx.betweenness_centrality(- sub, k=k, normalized=True- )- else:- self._symbol_betweenness_cache = nx.betweenness_centrality(- sub, normalized=True- )- return self._symbol_betweenness_cache-- def _build_scc_map(self) -> dict[str, int]:- """Assign a numeric SCC ID to each node."""- result: dict[str, int] = {}- for scc_id, scc in enumerate(nx.strongly_connected_components(self.graph())):- for node in scc:- result[node] = scc_id- return resultdiff --git a/packages/core/src/repowise/core/ingestion/graph/README.md b/packages/core/src/repowise/core/ingestion/graph/README.mdnew file mode 100644index 000000000..33147a499--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/README.md@@ -0,0 +1,71 @@+# `graph`++Builds the directed dependency graph for a repository from `ParsedFile`+objects and exposes the graph metrics (PageRank, betweenness, communities,+degrees) that downstream selection, generation, and risk analysis consume.++## Purpose++`GraphBuilder` is the analysis stage between parsing and generation. It adds+file/symbol nodes, resolves imports, heritage, member reads, and calls into a+NetworkX `DiGraph`, then computes centrality and community metrics over+file- and symbol-level subgraphs.++## Public API++```python+from repowise.core.ingestion.graph import GraphBuilder++b = GraphBuilder(repo_path)+for parsed in parsed_files:+ b.add_file(parsed)+b.build()+pr = b.pagerank() # dict[path, float]+bc = b.betweenness_centrality()+cd = b.community_detection()+ind = b.in_degree() # dict[path, int]+```++Edge augmentation: `add_co_change_edges`, `add_dynamic_edges`,+`add_framework_edges`. Serialisation: `to_json`, `persist`.++### Large-repo SQL metric routing++For large repos the file-level metrics are materialized to the `graph_metrics`+table (see `persistence.crud.batch_upsert_graph_metrics`). After that:++- `b.file_metrics_snapshot()` returns the `node_id → {pagerank, betweenness,+ community_id, in_degree, out_degree}` dict written to SQL.+- `b.load_metrics_from_sql(rows)` pre-fills the metric caches from the+ materialized snapshot, so the expensive NetworkX kernels are never recomputed+ — reads are served from SQL.+- `b.release_graph()` drops the in-memory NetworkX object once metrics are+ loaded, for callers that no longer need traversal (e.g. the fast-mode+ pipeline, which generates no docs).++This is gated by the pipeline knob `OrchestratorMode` / `sql_backed_metrics`+(default off in standard mode; on in fast mode). Standard generation keeps the+graph live because it traverses it for context assembly.++## Internal layout++- `builder.py` — `GraphBuilder` core: node/edge construction + `build()`.+- `_metrics.py` — `MetricsMixin`: centrality, communities, degrees, SQL routing.+- `_resolvers.py` — `ResolveMixin`: heritage / member-read / call passes.+- `_edges.py` — `EdgesMixin`: co-change / dynamic / framework edges.+- `_serialize.py` — `SerializeMixin`: node-link JSON + SQLite export.+- `_stem.py` — import-stem priority + stem-map construction.++## Extension points++- New edge kind: add a method to `EdgesMixin` or a resolution pass to+ `ResolveMixin`, then call it from `build()`.+- New metric: add a cached method to `MetricsMixin` and include it in+ `file_metrics_snapshot()` if it should be materialized.++## Tests++- `tests/unit/ingestion/test_graph.py`+- `tests/unit/ingestion/test_graph_metrics_sql.py`+- `tests/integration/persistence/test_graph_metrics_pg.py`+"""diff --git a/packages/core/src/repowise/core/ingestion/graph/__init__.py b/packages/core/src/repowise/core/ingestion/graph/__init__.pynew file mode 100644index 000000000..0772cdfca--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/__init__.py@@ -0,0 +1,40 @@+"""Dependency graph builder for the repowise ingestion pipeline.++GraphBuilder constructs a directed graph from ParsedFile objects with two+tiers of nodes:++ File-level nodes:+ "file" — every source file+ "external" — third-party / unresolvable imports (prefix "external:")++ Symbol-level nodes:+ "symbol" — functions, classes, methods, interfaces, etc.+ keyed by Symbol.id (e.g. "src/app.py::main")++Edge types:+ "imports" — file-to-file import relationship+ "defines" — file-to-symbol containment+ "has_method" — class-to-method ownership+ "calls" — symbol-to-symbol call relationship (with confidence)++After calling build(), graph metrics are available:+ pagerank() — dict[path, float]+ strongly_connected_components() — list[frozenset[str]]+ betweenness_centrality() — dict[path, float]+ in_degree() / out_degree() — dict[path, int]++Internal layout (kept under the 400-line ceiling):+ builder.py — GraphBuilder core (nodes, edges, build lifecycle)+ _metrics.py — centrality / community / degree metrics (+ SQL routing)+ _resolvers.py — heritage / member-read / call resolution passes+ _edges.py — co-change / dynamic / framework edges+ _serialize.py — node-link JSON + SQLite export+ _stem.py — import-stem resolution helpers+"""++from __future__ import annotations++from ._stem import _stem_priority, build_stem_map+from .builder import GraphBuilder++__all__ = ["GraphBuilder", "_stem_priority", "build_stem_map"]diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.pynew file mode 100644index 000000000..f159acd98--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py@@ -0,0 +1,121 @@+"""Co-change, dynamic-hint, and framework-aware edge construction.++Mixed into :class:`GraphBuilder`. These passes add edges that static import+resolution cannot infer: git co-change relations, dynamic-dispatch hints, and+framework-mediated wiring.+"""++from __future__ import annotations++import json++import structlog++from ..resolvers import ResolverContext+from ..resolvers.go import read_go_module_path, read_go_modules+from ._stem import build_stem_map++log = structlog.get_logger(__name__)+++class EdgesMixin:+ """Non-import edge construction for :class:`GraphBuilder`."""++ def add_co_change_edges(self, git_meta_map: dict, min_count: int = 3) -> int:+ """Add co_changes edges from git metadata. Returns count of edges added."""+ count = 0+ seen: set[tuple[str, str]] = set()++ for file_path, meta in git_meta_map.items():+ co_json = meta.get("co_change_partners_json", "[]")+ if isinstance(co_json, str):+ try:+ partners = json.loads(co_json)+ except Exception:+ partners = []+ else:+ partners = co_json++ for partner in partners:+ partner_path = partner.get("file_path", "")+ co_count = partner.get("co_change_count", 0)+ if co_count < min_count:+ continue+ if partner_path not in self._graph:+ continue++ pair = tuple(sorted([file_path, partner_path]))+ if pair in seen:+ continue+ seen.add(pair)++ if not self._graph.has_edge(file_path, partner_path) and not self._graph.has_edge(+ partner_path, file_path+ ):+ self._graph.add_edge(+ file_path,+ partner_path,+ edge_type="co_changes",+ weight=co_count,+ imported_names=[],+ )+ count += 1++ log.info("Co-change edges added", count=count)+ return count++ def update_co_change_edges(self, updated_meta: dict, min_count: int = 3) -> None:+ """Remove old co_changes edges for updated files, add new ones."""+ edges_to_remove = []+ for u, v, data in self._graph.edges(data=True):+ if data.get("edge_type") == "co_changes" and (u in updated_meta or v in updated_meta):+ edges_to_remove.append((u, v))+ self._graph.remove_edges_from(edges_to_remove)+ self.add_co_change_edges(updated_meta, min_count)++ def add_dynamic_edges(self, edges: list) -> None:+ """Add dynamic-hint edges to the graph. Each edge is a DynamicEdge."""+ for e in edges:+ if e.source not in self._graph:+ continue+ if e.target not in self._graph:+ self._graph.add_node(e.target)+ sub_type = e.edge_type or "dynamic"+ graph_edge_type = (+ sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"+ )+ self._graph.add_edge(+ e.source,+ e.target,+ edge_type=graph_edge_type,+ hint_source=e.hint_source,+ weight=e.weight,+ )++ def add_framework_edges(self, tech_stack: list[str] | None = None) -> int:+ """Add synthetic edges for framework-mediated relationships.++ Returns the number of edges added.+ """+ from ..framework_edges import add_framework_edges++ path_set = set(self._parsed_files.keys())+ stem_map = build_stem_map(path_set)++ go_modules = read_go_modules(self._repo_path)+ ctx = ResolverContext(+ path_set=path_set,+ stem_map=stem_map,+ graph=self._graph,+ repo_path=self._repo_path,+ tsconfig_resolver=self._tsconfig_resolver,+ go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),+ go_modules=go_modules,+ has_sfc_files=any(p.endswith((".vue", ".svelte", ".astro")) for p in path_set),+ parsed_files=self._parsed_files,+ )++ count = add_framework_edges(self._graph, self._parsed_files, ctx, tech_stack)+ if count:+ log.info("Framework edges added", count=count)+ return countdiff --git a/packages/core/src/repowise/core/ingestion/graph/_metrics.py b/packages/core/src/repowise/core/ingestion/graph/_metrics.pynew file mode 100644index 000000000..052a8de7f--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/_metrics.py@@ -0,0 +1,325 @@+"""Graph metric computation (PageRank, betweenness, communities, degrees).++These methods are mixed into :class:`GraphBuilder`. They read the metric+caches and ``_graph`` set up in the builder's ``__init__``.++Large-repo SQL routing+----------------------+When metric values have been materialized to SQL (the ``graph_metrics``+table), :meth:`load_metrics_from_sql` pre-fills the file-level caches so the+expensive NetworkX kernels (notably betweenness on 30k+ nodes) are never+recomputed — subsequent reads are served straight from the materialized+snapshot. The structural graph stays available for traversal.+"""++from __future__ import annotations++from typing import Any++import networkx as nx+import structlog++log = structlog.get_logger(__name__)++_LARGE_REPO_THRESHOLD = 30_000 # nodes — above this, algorithms are expensive+++class MetricsMixin:+ """Centrality + community + degree metrics for :class:`GraphBuilder`."""++ # ------------------------------------------------------------------+ # SQL-backed metric routing (large repos)+ # ------------------------------------------------------------------++ def load_metrics_from_sql(self, metrics: dict[str, dict[str, Any]]) -> None:+ """Pre-fill the file-level metric caches from a materialized snapshot.++ *metrics* maps ``node_id`` → a dict with any of ``pagerank``,+ ``betweenness``, ``community_id``, ``in_degree``, ``out_degree``.+ After this call, the corresponding metric methods return the snapshot+ values without recomputing them via NetworkX.+ """+ self._pagerank_cache = {n: float(m.get("pagerank", 0.0)) for n, m in metrics.items()}+ self._betweenness_cache = {+ n: float(m.get("betweenness", 0.0)) for n, m in metrics.items()+ }+ self._community_cache = {n: int(m.get("community_id", 0)) for n, m in metrics.items()}+ self._in_degree_cache = {n: int(m.get("in_degree", 0)) for n, m in metrics.items()}+ self._out_degree_cache = {n: int(m.get("out_degree", 0)) for n, m in metrics.items()}+ log.info("graph.metrics_loaded_from_sql", nodes=len(metrics))++ def file_metrics_snapshot(self) -> dict[str, dict[str, Any]]:+ """Return the file-level metrics as a ``node_id → metrics`` dict.++ Used to materialize the ``graph_metrics`` table. Computes every metric+ from NetworkX (or returns the cached/SQL-loaded values when present).+ """+ pr = self.pagerank()+ bc = self.betweenness_centrality()+ cd = self.community_detection()+ ind = self.in_degree()+ outd = self.out_degree()+ nodes = set(pr) | set(bc) | set(cd) | set(ind) | set(outd)+ return {+ n: {+ "pagerank": pr.get(n, 0.0),+ "betweenness": bc.get(n, 0.0),+ "community_id": cd.get(n, 0),+ "in_degree": ind.get(n, 0),+ "out_degree": outd.get(n, 0),+ }+ for n in nodes+ }++ # ------------------------------------------------------------------+ # Subgraphs+ # ------------------------------------------------------------------++ def file_subgraph(self) -> nx.DiGraph:+ """Return a subgraph containing only file-level nodes and import edges."""+ g = self.graph()+ file_nodes = [+ n for n, d in g.nodes(data=True) if d.get("node_type", "file") in ("file", "external")+ ]+ sub = g.subgraph(file_nodes).copy()+ edges_to_remove = [+ (u, v) for u, v, d in sub.edges(data=True) if d.get("edge_type") in ("co_changes",)+ ]+ sub.remove_edges_from(edges_to_remove)+ return sub++ def symbol_subgraph(self) -> nx.DiGraph:+ """Return a subgraph of symbol nodes connected by call + heritage edges.++ File-to-symbol ``defines`` edges and class-to-method ``has_method``+ ownership edges are dropped so that the resulting centrality+ scores reflect call/heritage flow rather than containment.+ """+ g = self.graph()+ symbol_nodes = [n for n, d in g.nodes(data=True) if d.get("node_type") == "symbol"]+ sub = g.subgraph(symbol_nodes).copy()+ edges_to_remove = [+ (u, v)+ for u, v, d in sub.edges(data=True)+ if d.get("edge_type") not in ("calls", "extends", "implements")+ ]+ sub.remove_edges_from(edges_to_remove)+ return sub++ # ------------------------------------------------------------------+ # File-level metrics+ # ------------------------------------------------------------------++ def strongly_connected_components(self) -> list[frozenset[str]]:+ """Return SCCs as a list of frozensets."""+ return [frozenset(scc) for scc in nx.strongly_connected_components(self.file_subgraph())]++ def pagerank(self, alpha: float = 0.85) -> dict[str, float]:+ """Return PageRank scores for file nodes only (cached)."""+ if self._pagerank_cache is not None:+ return self._pagerank_cache+ filtered = self.file_subgraph()+ if filtered.number_of_nodes() == 0:+ self._pagerank_cache = {}+ return self._pagerank_cache++ try:+ self._pagerank_cache = nx.pagerank(filtered, alpha=alpha)+ except nx.PowerIterationFailedConvergence:+ log.warning("PageRank did not converge, using uniform scores")+ n = filtered.number_of_nodes()+ self._pagerank_cache = {node: 1.0 / n for node in filtered.nodes()}+ return self._pagerank_cache++ def betweenness_centrality(self) -> dict[str, float]:+ """Return betweenness centrality for file nodes (cached)."""+ if self._betweenness_cache is not None:+ return self._betweenness_cache+ g = self.file_subgraph()+ n = g.number_of_nodes()+ if n == 0:+ self._betweenness_cache = {}+ return self._betweenness_cache+ if n > _LARGE_REPO_THRESHOLD:+ k = min(500, n)+ self._betweenness_cache = nx.betweenness_centrality(g, k=k, normalized=True)+ else:+ self._betweenness_cache = nx.betweenness_centrality(g, normalized=True)+ return self._betweenness_cache++ def in_degree(self) -> dict[str, int]:+ """Return in-degree (number of importers) for each file node (cached)."""+ if self._in_degree_cache is not None:+ return self._in_degree_cache+ g = self.file_subgraph()+ self._in_degree_cache = {n: int(d) for n, d in g.in_degree()}+ return self._in_degree_cache++ def out_degree(self) -> dict[str, int]:+ """Return out-degree (number of dependencies) for each file node (cached)."""+ if self._out_degree_cache is not None:+ return self._out_degree_cache+ g = self.file_subgraph()+ self._out_degree_cache = {n: int(d) for n, d in g.out_degree()}+ return self._out_degree_cache++ # ------------------------------------------------------------------+ # Communities+ # ------------------------------------------------------------------++ def community_detection(self) -> dict[str, int]:+ """Assign a community ID to each file node."""+ if self._community_cache is not None:+ return self._community_cache++ from repowise.core.analysis.communities import detect_file_communities++ try:+ assignment, info, algo = detect_file_communities(self._graph)+ self._community_cache = assignment+ self._community_info_cache = info+ self._community_algo = algo+ except Exception as exc:+ log.warning("community_detection_failed", error=str(exc))+ file_nodes = [+ n for n, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"+ ]+ self._community_cache = {n: 0 for n in file_nodes}+ self._community_info_cache = {}+ self._community_algo = "failed"+ return self._community_cache++ def symbol_communities(self) -> dict[str, int]:+ """Assign a community ID to each symbol node using call/heritage edges."""+ if self._symbol_community_cache is not None:+ return self._symbol_community_cache++ from repowise.core.analysis.communities import detect_symbol_communities++ try:+ self._symbol_community_cache = detect_symbol_communities(self._graph)+ except Exception as exc:+ log.warning("symbol_community_detection_failed", error=str(exc))+ self._symbol_community_cache = {}+ return self._symbol_community_cache++ def community_info(self) -> dict[int, Any]:+ """Return metadata for each file-level community."""+ if self._community_info_cache is None:+ self.community_detection()+ return self._community_info_cache or {}++ # ------------------------------------------------------------------+ # Symbol-level metrics+ # ------------------------------------------------------------------++ def symbol_pagerank(self, alpha: float = 0.85) -> dict[str, float]:+ """Return PageRank scores for symbol nodes only (cached).++ Computed on the call/heritage symbol subgraph — this is what the+ UI's per-symbol "graph metrics" panel reads. Without it every+ symbol shows ``Not indexed in graph``.+ """+ if self._symbol_pagerank_cache is not None:+ return self._symbol_pagerank_cache+ sub = self.symbol_subgraph()+ if sub.number_of_nodes() == 0:+ self._symbol_pagerank_cache = {}+ return self._symbol_pagerank_cache+ try:+ self._symbol_pagerank_cache = nx.pagerank(sub, alpha=alpha)+ except nx.PowerIterationFailedConvergence:+ log.warning("Symbol PageRank did not converge, using uniform scores")+ n = sub.number_of_nodes()+ self._symbol_pagerank_cache = {node: 1.0 / n for node in sub.nodes()}+ return self._symbol_pagerank_cache++ def symbol_betweenness_centrality(self) -> dict[str, float]:+ """Return betweenness centrality for symbol nodes (cached)."""+ if self._symbol_betweenness_cache is not None:+ return self._symbol_betweenness_cache+ sub = self.symbol_subgraph()+ n = sub.number_of_nodes()+ if n == 0:+ self._symbol_betweenness_cache = {}+ return self._symbol_betweenness_cache+ if n > _LARGE_REPO_THRESHOLD:+ k = min(500, n)+ self._symbol_betweenness_cache = nx.betweenness_centrality(sub, k=k, normalized=True)+ else:+ self._symbol_betweenness_cache = nx.betweenness_centrality(sub, normalized=True)+ return self._symbol_betweenness_cache++ # ------------------------------------------------------------------+ # Execution flows + bulk priming+ # ------------------------------------------------------------------++ def execution_flows(self, config: Any | None = None) -> Any:+ """Trace execution flows from entry-point symbols (cached when ``config`` is None)."""+ from repowise.core.analysis.execution_flows import (+ ExecutionFlowReport,+ trace_execution_flows,+ )++ # Only the no-config path is cached — callers that pass custom+ # FlowConfig still get a fresh trace.+ if config is None and self._execution_flow_cache is not None:+ return self._execution_flow_cache++ file_cd = self.community_detection()+ merged_cd: dict[str, int] = dict(file_cd)++ sym_cd = self.symbol_communities()+ merged_cd.update(sym_cd)++ for node_id in self._graph.nodes():+ if node_id not in merged_cd and "::" in node_id:+ file_path = node_id.split("::")[0]+ if file_path in file_cd:+ merged_cd[node_id] = file_cd[file_path]++ try:+ report = trace_execution_flows(self._graph, merged_cd, config)+ except Exception as exc:+ log.warning("execution_flow_tracing_failed", error=str(exc))+ report = ExecutionFlowReport(+ total_entry_points_scored=0,+ total_flows=0,+ flows=[],+ )+ if config is None:+ self._execution_flow_cache = report+ return report++ async def compute_metrics_parallel(self) -> None:+ """Eagerly populate all metric caches with fan-out parallelism.++ Runs PageRank, betweenness, file/symbol community detection in+ parallel via ``asyncio.gather`` + ``asyncio.to_thread`` (the+ scipy- and igraph-backed kernels release the GIL during heavy+ compute, so true parallelism is achievable). Execution flows then+ run after, since they depend on the community caches.++ Calling this is optional — every metric falls back to lazy+ computation, so existing call sites keep working unchanged.+ """+ import asyncio as _asyncio++ await _asyncio.gather(+ _asyncio.to_thread(self.pagerank),+ _asyncio.to_thread(self.betweenness_centrality),+ _asyncio.to_thread(self.symbol_pagerank),+ _asyncio.to_thread(self.symbol_betweenness_centrality),+ _asyncio.to_thread(self.community_detection),+ _asyncio.to_thread(self.symbol_communities),+ )+ # execution_flows reads from the community caches just primed above.+ await _asyncio.to_thread(self.execution_flows)++ def _build_scc_map(self) -> dict[str, int]:+ """Assign a numeric SCC ID to each node."""+ result: dict[str, int] = {}+ for scc_id, scc in enumerate(nx.strongly_connected_components(self.graph())):+ for node in scc:+ result[node] = scc_id+ return resultdiff --git a/packages/core/src/repowise/core/ingestion/graph/_resolvers.py b/packages/core/src/repowise/core/ingestion/graph/_resolvers.pynew file mode 100644index 000000000..164f86243--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/_resolvers.py@@ -0,0 +1,135 @@+"""Heritage / member-read / call edge resolution for :class:`GraphBuilder`.++Each pass reads ``self._parsed_files`` and mutates ``self._graph`` in place,+emitting EXTENDS/IMPLEMENTS, ``reads``, and ``calls`` edges respectively.+"""++from __future__ import annotations++from typing import Any++import structlog++log = structlog.get_logger(__name__)+++class ResolveMixin:+ """Symbol-level edge resolution passes run during ``build()``."""++ def _resolve_heritage(+ self,+ import_targets: dict[str, set[str]],+ progress: Any | None = None,+ ) -> None:+ """Resolve heritage relations and add EXTENDS/IMPLEMENTS edges."""+ from ..heritage_resolver import HeritageResolver++ resolver = HeritageResolver(self._parsed_files, import_targets)+ total_resolved = 0++ files_with_heritage = [+ (p, pf) for p, pf in self._parsed_files.items() if pf.heritage+ ]+ if progress:+ progress.on_phase_start("graph.heritage", len(files_with_heritage))+ for path, parsed in files_with_heritage:+ resolved = resolver.resolve_file(path, parsed.heritage)+ for rh in resolved:+ if rh.child_id in self._graph and rh.parent_id in self._graph:+ if not self._graph.has_edge(rh.child_id, rh.parent_id):+ self._graph.add_edge(+ rh.child_id,+ rh.parent_id,+ edge_type=rh.edge_type,+ confidence=rh.confidence,+ )+ total_resolved += 1+ else:+ existing = self._graph[rh.child_id][rh.parent_id]+ if rh.confidence > existing.get("confidence", 0):+ existing["confidence"] = rh.confidence+ if progress:+ progress.on_item_done("graph.heritage")++ if progress:+ _phase_done = getattr(progress, "on_phase_done", None)+ if _phase_done is not None:+ _phase_done("graph.heritage")+ log.info("Heritage edges resolved", total=total_resolved)++ def _resolve_member_reads(self, progress: Any | None = None) -> None:+ """Phase 1c: emit ``reads`` edges for C# property / member access.++ Runs after type-use resolution so the dead-code analyser sees+ member access as evidence of reachability. The pass is C#-only+ today (the lever is largest there); the helper module is set+ up to receive other languages via additional strategies.+ """+ from ..languages.csharp_member_reads import (+ build_csharp_type_to_file,+ collect_csharp_source_texts,+ resolve_csharp_member_reads,+ )++ has_csharp = any(+ pf.file_info.language == "csharp" for pf in self._parsed_files.values()+ )+ if not has_csharp:+ return++ phase = "graph.member_reads"+ if progress:+ progress.on_phase_start(phase, None)+ try:+ cs_texts = collect_csharp_source_texts(self._parsed_files)+ type_to_file = build_csharp_type_to_file(self._parsed_files)+ added = resolve_csharp_member_reads(self._graph, cs_texts, type_to_file)+ log.info("member_read_edges", language="csharp", added=added)+ except Exception as exc:+ log.warning("member_reads_failed", error=str(exc))+ finally:+ if progress:+ done = getattr(progress, "on_phase_done", None)+ if callable(done):+ done(phase)++ def _resolve_calls(+ self,+ import_targets: dict[str, set[str]],+ progress: Any | None = None,+ ) -> None:+ """Run three-tier call resolution and add CALLS edges to the graph."""+ from ..call_resolver import CallResolver++ resolver = CallResolver(self._parsed_files, import_targets)+ total_resolved = 0++ files_with_calls = [+ (p, pf) for p, pf in self._parsed_files.items() if pf.calls+ ]+ if progress:+ progress.on_phase_start("graph.calls", len(files_with_calls))+ for path, parsed in files_with_calls:+ resolved = resolver.resolve_file(path, parsed.calls)+ for rc in resolved:+ if rc.caller_id in self._graph and rc.callee_id in self._graph:+ if not self._graph.has_edge(rc.caller_id, rc.callee_id):+ self._graph.add_edge(+ rc.caller_id,+ rc.callee_id,+ edge_type="calls",+ confidence=rc.confidence,+ )+ total_resolved += 1+ else:+ existing = self._graph[rc.caller_id][rc.callee_id]+ if rc.confidence > existing.get("confidence", 0):+ existing["confidence"] = rc.confidence+ if progress:+ progress.on_item_done("graph.calls")++ if progress:+ _phase_done = getattr(progress, "on_phase_done", None)+ if _phase_done is not None:+ _phase_done("graph.calls")+ log.info("Call edges resolved", total=total_resolved)diff --git a/packages/core/src/repowise/core/ingestion/graph/_serialize.py b/packages/core/src/repowise/core/ingestion/graph/_serialize.pynew file mode 100644index 000000000..2f2724470--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/_serialize.py@@ -0,0 +1,108 @@+"""Graph serialisation: node-link JSON + standalone SQLite persistence.++Mixed into :class:`GraphBuilder`. The SQLite ``persist`` path is the+self-contained writer used by the standalone graph export; the main pipeline+persists through ``persistence.crud`` instead.+"""++from __future__ import annotations++import contextlib+import json+from pathlib import Path+from typing import Any++import networkx as nx+import structlog++log = structlog.get_logger(__name__)+++class SerializeMixin:+ """JSON + SQLite serialisation for :class:`GraphBuilder`."""++ def to_json(self) -> dict[str, Any]:+ """Serialize the graph to a JSON-compatible dict (node-link format)."""+ return nx.node_link_data(self.graph())++ async def persist(self, db_path: Path, repo_id: str) -> None:+ """Persist the graph to an SQLite database."""+ import sqlite3++ import aiosqlite++ pr = self.pagerank()+ bc = self.betweenness_centrality()+ scc_map = self._build_scc_map()+ cd = self.community_detection()+ g = self.graph()++ async with aiosqlite.connect(db_path) as db:+ await db.executescript("""+ CREATE TABLE IF NOT EXISTS graph_nodes (+ repo_id TEXT NOT NULL,+ path TEXT NOT NULL,+ language TEXT,+ symbol_count INTEGER,+ has_error INTEGER,+ pagerank REAL,+ betweenness REAL,+ scc_id INTEGER,+ community_id INTEGER DEFAULT 0,+ PRIMARY KEY (repo_id, path)+ );+ CREATE TABLE IF NOT EXISTS graph_edges (+ repo_id TEXT NOT NULL,+ source_path TEXT NOT NULL,+ target_path TEXT NOT NULL,+ imported_names TEXT,+ PRIMARY KEY (repo_id, source_path, target_path)+ );+ """)++ with contextlib.suppress(sqlite3.OperationalError):+ await db.execute(+ "ALTER TABLE graph_nodes ADD COLUMN community_id INTEGER DEFAULT 0"+ )++ node_rows = [+ (+ repo_id,+ path,+ data.get("language", ""),+ data.get("symbol_count", 0),+ int(data.get("has_error", False)),+ pr.get(path, 0.0),+ bc.get(path, 0.0),+ scc_map.get(path, 0),+ cd.get(path, cd.get(data.get("file_path", ""), 0)),+ )+ for path, data in g.nodes(data=True)+ ]+ await db.executemany(+ "INSERT OR REPLACE INTO graph_nodes VALUES (?,?,?,?,?,?,?,?,?)",+ node_rows,+ )++ edge_rows = [+ (+ repo_id,+ src,+ dst,+ json.dumps(data.get("imported_names", [])),+ )+ for src, dst, data in g.edges(data=True)+ ]+ await db.executemany(+ "INSERT OR REPLACE INTO graph_edges VALUES (?,?,?,?)",+ edge_rows,+ )++ await db.commit()++ log.info(+ "Graph persisted",+ db_path=str(db_path),+ nodes=len(node_rows),+ edges=len(edge_rows),+ )diff --git a/packages/core/src/repowise/core/ingestion/graph/_stem.py b/packages/core/src/repowise/core/ingestion/graph/_stem.pynew file mode 100644index 000000000..6a37c5777--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/_stem.py@@ -0,0 +1,64 @@+"""Import-stem resolution helpers for the dependency graph builder."""++from __future__ import annotations++from pathlib import Path++# Path segments that mark a file as low-value for stem-based import resolution.+_LOW_VALUE_PATH_SEGMENTS = frozenset(+ {+ "tests",+ "test",+ "_tests",+ "__tests__",+ "testing",+ "test_apps",+ "testdata",+ "test_data",+ "fixtures",+ "examples",+ "example",+ "samples",+ "sample",+ "scripts",+ "benchmarks",+ "bench",+ "docs",+ "doc",+ }+)+++def _stem_priority(path: str, stem: str) -> tuple[int, int, int, str]:+ """Sort key for choosing among files that share an import stem.++ Lower tuples sort first; callers take ``candidates[0]`` as the resolution.+ """+ path_obj = Path(path)+ parts = path_obj.parts+ if path_obj.name == "__init__.py":+ parent_match = 0+ else:+ parent_dir = parts[-2].lower() if len(parts) >= 2 else ""+ parent_match = 0 if parent_dir == stem else 1+ low_value = 1 if any(seg.lower() in _LOW_VALUE_PATH_SEGMENTS for seg in parts) else 0+ return (parent_match, low_value, len(parts), path)+++def build_stem_map(path_set: set[str]) -> dict[str, list[str]]:+ """Map import-stems to candidate file paths, sorted best-first."""+ buckets: dict[str, list[str]] = {}+ for p in path_set:+ path_obj = Path(p)+ if path_obj.name == "__init__.py":+ parent = path_obj.parent.name+ if not parent:+ continue+ stem = parent.lower()+ else:+ stem = path_obj.stem.lower()+ buckets.setdefault(stem, []).append(p)++ for stem, paths in buckets.items():+ paths.sort(key=lambda candidate: _stem_priority(candidate, stem))+ return bucketsdiff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.pynew file mode 100644index 000000000..14fc94246--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py@@ -0,0 +1,325 @@+"""GraphBuilder — constructs the directed dependency graph from ParsedFiles.++This module holds the structural core (node/edge construction, import+resolution driver, lifecycle). Metrics, edge augmentation, resolution passes,+and serialisation are provided by mixins in sibling modules to keep every+file under the project's 400-line ceiling.+"""++from __future__ import annotations++from pathlib import Path+from typing import Any++import networkx as nx+import structlog++from ..models import ParsedFile+from ..resolvers import ResolverContext, resolve_import+from ..resolvers.go import read_go_module_path, read_go_modules+from ..type_ref_resolution import resolve_type_refs+from ._edges import EdgesMixin+from ._metrics import MetricsMixin+from ._resolvers import ResolveMixin+from ._serialize import SerializeMixin+from ._stem import build_stem_map++log = structlog.get_logger(__name__)+++class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin):+ """Build a dependency graph from a collection of ParsedFile objects.++ Usage::++ builder = GraphBuilder()+ for parsed in parsed_files:+ builder.add_file(parsed)+ graph = builder.build()+ pr = builder.pagerank()+ """++ def __init__(self, repo_path: Path | str | None = None) -> None:+ self._graph: nx.DiGraph = nx.DiGraph()+ self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile+ self._built = False+ self._repo_path: Path | None = Path(repo_path) if repo_path else None+ self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)++ # Community / flow / metric caches (invalidated on build)+ self._community_cache: dict[str, int] | None = None+ self._symbol_community_cache: dict[str, int] | None = None+ self._community_info_cache: dict[int, Any] | None = None+ self._community_algo: str = ""+ self._pagerank_cache: dict[str, float] | None = None+ self._betweenness_cache: dict[str, float] | None = None+ self._in_degree_cache: dict[str, int] | None = None+ self._out_degree_cache: dict[str, int] | None = None+ self._symbol_pagerank_cache: dict[str, float] | None = None+ self._symbol_betweenness_cache: dict[str, float] | None = None+ self._execution_flow_cache: Any | None = None++ def set_tsconfig_resolver(self, resolver: Any) -> None:+ """Attach a :class:`TsconfigResolver` for TS/JS path-alias resolution."""+ self._tsconfig_resolver = resolver++ def _invalidate_metric_caches(self) -> None:+ """Clear every cached metric (called on build + add_file)."""+ self._community_cache = None+ self._symbol_community_cache = None+ self._community_info_cache = None+ self._community_algo = ""+ self._pagerank_cache = None+ self._betweenness_cache = None+ self._in_degree_cache = None+ self._out_degree_cache = None+ self._symbol_pagerank_cache = None+ self._symbol_betweenness_cache = None+ self._execution_flow_cache = None++ def release_graph(self) -> None:+ """Drop the in-memory NetworkX object after metrics are materialized.++ For large-repo scale work: once file-level metrics have been loaded+ from SQL via :meth:`load_metrics_from_sql`, callers that no longer need+ graph traversal (e.g. the fast-mode pipeline, which generates no docs)+ can release the structural graph to free memory. Metric reads continue+ to be served from the loaded caches.++ ``_built`` stays True so a subsequent ``graph()`` call returns the+ empty graph rather than silently (and expensively) rebuilding the+ structure we deliberately dropped.+ """+ self._graph = nx.DiGraph()+ self._built = True++ # ------------------------------------------------------------------+ # Building+ # ------------------------------------------------------------------++ def add_file(self, parsed: ParsedFile) -> None:+ """Register one parsed file and its symbols in the graph."""+ path = parsed.file_info.path+ self._parsed_files[path] = parsed+ self._built = False # invalidate cached metrics++ # --- File node ---+ self._graph.add_node(+ path,+ node_type="file",+ language=parsed.file_info.language,+ symbol_count=len(parsed.symbols),+ has_error=bool(parsed.parse_errors),+ is_test=parsed.file_info.is_test,+ is_entry_point=parsed.file_info.is_entry_point,+ )++ # --- Symbol nodes ---+ for sym in parsed.symbols:+ self._graph.add_node(+ sym.id,+ node_type="symbol",+ kind=sym.kind,+ name=sym.name,+ qualified_name=sym.qualified_name,+ file_path=path,+ start_line=sym.start_line,+ end_line=sym.end_line,+ visibility=sym.visibility,+ is_async=sym.is_async,+ language=sym.language,+ parent_name=sym.parent_name,+ signature=sym.signature,+ decorators=sym.decorators,+ is_exported_symbol=sym.is_exported_symbol,+ )++ # DEFINES edge: file → symbol+ self._graph.add_edge(+ path,+ sym.id,+ edge_type="defines",+ )++ # HAS_METHOD edge: class/struct → method+ if sym.parent_name and sym.kind == "method":+ parent_id = f"{path}::{sym.parent_name}"+ if parent_id in self._graph:+ self._graph.add_edge(+ parent_id,+ sym.id,+ edge_type="has_method",+ )++ # --- Synthetic module-level symbol for top-level calls ---+ module_sym_id = f"{path}::__module__"+ self._graph.add_node(+ module_sym_id,+ node_type="symbol",+ kind="module",+ name="__module__",+ file_path=path,+ start_line=0,+ end_line=0,+ visibility="private",+ language=parsed.file_info.language,+ )+ self._graph.add_edge(path, module_sym_id, edge_type="defines")++ def build(self, progress: Any | None = None) -> nx.DiGraph:+ """Resolve imports and calls, add edges. Returns the finalized graph.++ *progress* is an optional ``ProgressCallback`` (duck-typed). When+ provided, sub-phase events ``graph.imports`` / ``graph.heritage`` /+ ``graph.calls`` are emitted so the CLI can surface per-file progress+ instead of a single opaque "0/1" bar over the whole build.+ """+ self._invalidate_metric_caches()++ # Clear import/call edges but keep structural edges (defines, has_method)+ edges_to_remove = [+ (u, v)+ for u, v, d in self._graph.edges(data=True)+ if d.get("edge_type") not in ("defines", "has_method")+ ]+ self._graph.remove_edges_from(edges_to_remove)++ # Build lookup tables for import resolution+ path_set = set(self._parsed_files.keys())+ stem_map = build_stem_map(path_set)++ # Construct resolver context+ go_modules = read_go_modules(self._repo_path)+ ctx = ResolverContext(+ path_set=path_set,+ stem_map=stem_map,+ graph=self._graph,+ repo_path=self._repo_path,+ tsconfig_resolver=self._tsconfig_resolver,+ go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),+ go_modules=go_modules,+ has_sfc_files=any(p.endswith((".vue", ".svelte", ".astro")) for p in path_set),+ parsed_files=self._parsed_files,+ )++ # --- Phase 1 prelude: language-specific warmups ---+ # Some languages need an expensive one-time index built before any+ # per-file import resolution can run. Doing it here, under a dedicated+ # phase event, keeps the cost out of ``graph.imports`` and surfaces+ # a meaningful progress label instead of a stuck per-file bar.+ from ..graph_warmups import run_warmups++ run_warmups(self._parsed_files, ctx, progress=progress)++ # --- Phase 1: Resolve file-level imports ---+ import_targets: dict[str, set[str]] = {} # file → set of imported files++ # Per-language import-resolution timing — surfaces which language is+ # actually dominating the import loop on multi-language repos+ # (audit #30). Persisted to state.json for after-the-fact analysis.+ import time as _t++ lang_import_time: dict[str, float] = {}++ if progress:+ progress.on_phase_start("graph.imports", len(self._parsed_files))+ for path, parsed in self._parsed_files.items():+ _lang = parsed.file_info.language+ _t0 = _t.monotonic()+ file_imports: set[str] = set()+ for imp in parsed.imports:+ target = resolve_import(imp.module_path, path, parsed.file_info.language, ctx)+ if target:+ imp.resolved_file = target+ file_imports.add(target)+ # Aggregate imported_names on parallel edges+ if self._graph.has_edge(path, target):+ existing = self._graph[path][target].get("imported_names", [])+ merged = list(set(existing + imp.imported_names))+ self._graph[path][target]["imported_names"] = merged+ else:+ self._graph.add_edge(+ path,+ target,+ edge_type="imports",+ imported_names=list(imp.imported_names),+ )+ import_targets[path] = file_imports+ lang_import_time[_lang] = lang_import_time.get(_lang, 0.0) + (_t.monotonic() - _t0)+ if progress:+ progress.on_item_done("graph.imports")+ if progress:+ _phase_done = getattr(progress, "on_phase_done", None)+ if _phase_done is not None:+ _phase_done("graph.imports")+ if lang_import_time:+ log.info(+ "import_resolution_per_language",+ seconds={k: round(v, 2) for k, v in lang_import_time.items()},+ )++ # --- Phase 1b: Resolve non-import type references (e.g. C# ctor params) ---+ # Lives between import resolution and heritage so the type-use+ # edges feed into heritage's import_targets propagation if a+ # future language emits them, and so dead-code reachability+ # sees every edge before any analysis pass runs.+ if progress:+ progress.on_phase_start("graph.type_refs", len(self._parsed_files))+ type_use_counts = resolve_type_refs(self._parsed_files, ctx, self._graph)+ for path in self._parsed_files:+ for _, target, data in self._graph.out_edges(path, data=True):+ # Treat both real ``using`` imports and synthesised+ # type-use edges as import-like for downstream heritage+ # / call resolution. Without this, type-use edges would+ # only contribute to file-reachability — not to+ # ``import_targets`` which gates cross-file call /+ # heritage lookups.+ if data.get("edge_type") in ("imports", "type_use"):+ import_targets.setdefault(path, set()).add(target)+ if progress:+ _phase_done = getattr(progress, "on_phase_done", None)+ if _phase_done is not None:+ _phase_done("graph.type_refs")+ del type_use_counts # used only for logging inside resolve_type_refs++ # --- Phase 1c: Resolve C# member-access reads ---+ # Bind `var x = new T(...); ... x.Prop` style property reads+ # to T's defining file. Cuts the largest single bucket of C#+ # unused_export false positives (audit #23).+ self._resolve_member_reads(progress=progress)++ # --- Phase 2: Resolve heritage (extends/implements) ---+ self._resolve_heritage(import_targets, progress=progress)++ # --- Phase 3: Resolve symbol-level calls ---+ self._resolve_calls(import_targets, progress=progress)++ self._built = True++ # Count edge types for logging+ edge_counts: dict[str, int] = {}+ for _, _, d in self._graph.edges(data=True):+ et = d.get("edge_type", "imports")+ edge_counts[et] = edge_counts.get(et, 0) + 1++ file_nodes = sum(+ 1 for _, d in self._graph.nodes(data=True) if d.get("node_type", "file") == "file"+ )+ symbol_nodes = sum(+ 1 for _, d in self._graph.nodes(data=True) if d.get("node_type") == "symbol"+ )++ log.info(+ "Graph built",+ file_nodes=file_nodes,+ symbol_nodes=symbol_nodes,+ edges=self._graph.number_of_edges(),+ edge_types=edge_counts,+ )+ return self._graph++ def graph(self) -> nx.DiGraph:+ """Return the graph (building it first if necessary)."""+ if not self._built:+ self.build()+ return self._graphdiff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.pyindex ed8af4092..5d974bf2c 100644--- a/packages/core/src/repowise/core/persistence/__init__.py+++ b/packages/core/src/repowise/core/persistence/__init__.py@@ -18,12 +18,11 @@ from repowise.core.providers.embedding.base import Embedder, MockEmbedderfrom .crud import (batch_upsert_graph_edges,+ batch_upsert_graph_metrics,batch_upsert_graph_nodes,batch_upsert_symbols,bulk_upsert_decisions,bulk_upsert_external_systems,- link_graph_nodes_to_external_systems,- list_external_systems,count_chat_messages,create_chat_message,create_conversation,@@ -43,6 +42,7 @@ from .crud import (get_git_metadata,get_git_metadata_bulk,get_graph_edges_for_node,+ get_graph_metrics,get_graph_node,get_graph_nodes_by_ids,get_node_degree_counts,@@ -53,9 +53,11 @@ from .crud import (get_stale_decisions,get_stale_pages,get_top_entry_points,+ link_graph_nodes_to_external_systems,list_chat_messages,list_conversations,list_decisions,+ list_external_systems,list_page_ids,list_pages,load_prior_pages,@@ -99,6 +101,7 @@ from .models import (GenerationJob,GitMetadata,GraphEdge,+ GraphMetric,GraphNode,Page,PageVersion,@@ -132,6 +135,7 @@ __all__ = ["GenerationJob","GitMetadata","GraphEdge",+ "GraphMetric","GraphNode",# vector store"InMemoryVectorStore",@@ -148,25 +152,15 @@ __all__ = ["async_sessionmaker",# crud"batch_upsert_graph_edges",+ "batch_upsert_graph_metrics","batch_upsert_graph_nodes","batch_upsert_symbols",# decision crud"bulk_upsert_decisions",# external systems crud (C4 L1)"bulk_upsert_external_systems",- "link_graph_nodes_to_external_systems",- "list_external_systems",# chat crud"count_chat_messages",- # graph read-side queries- "get_all_file_metrics",- "get_community_members",- "get_cross_community_edges",- "get_graph_edges_for_node",- "get_graph_node",- "get_graph_nodes_by_ids",- "get_node_degree_counts",- "get_top_entry_points","create_chat_message","create_conversation","create_engine",@@ -174,10 +168,14 @@ __all__ = ["delete_conversation","delete_decision","delete_repository",+ # graph read-side queries+ "get_all_file_metrics",# git metadata crud"get_all_git_metadata",+ "get_community_members","get_configured_db_url","get_conversation",+ "get_cross_community_edges","get_db_url",# dead code crud"get_dead_code_findings",@@ -187,6 +185,11 @@ __all__ = ["get_generation_job","get_git_metadata","get_git_metadata_bulk",+ "get_graph_edges_for_node",+ "get_graph_metrics",+ "get_graph_node",+ "get_graph_nodes_by_ids",+ "get_node_degree_counts","get_page","get_page_versions","get_repo_db_path",@@ -195,10 +198,13 @@ __all__ = ["get_session","get_stale_decisions","get_stale_pages",+ "get_top_entry_points","init_db",+ "link_graph_nodes_to_external_systems","list_chat_messages","list_conversations","list_decisions",+ "list_external_systems","list_page_ids","list_pages","load_prior_pages",diff --git a/packages/core/src/repowise/core/persistence/crud.py b/packages/core/src/repowise/core/persistence/crud.pyindex 1f5e387c5..a43213c07 100644--- a/packages/core/src/repowise/core/persistence/crud.py+++ b/packages/core/src/repowise/core/persistence/crud.py@@ -31,6 +31,7 @@ from .models import (GenerationJob,GitMetadata,GraphEdge,+ GraphMetric,GraphNode,HealthFileMetric,HealthFinding,@@ -553,6 +554,69 @@ async def batch_upsert_graph_edges(await session.flush()+async def batch_upsert_graph_metrics(+ session: AsyncSession,+ repository_id: str,+ metrics: dict[str, dict],+) -> None:+ """Materialize the file-level metrics snapshot into ``graph_metrics``.++ *metrics* maps ``node_id`` → a dict with ``pagerank``, ``betweenness``,+ ``community_id``, ``in_degree``, ``out_degree``. Additive to+ ``graph_nodes`` — this is the snapshot read back by+ ``GraphBuilder.load_metrics_from_sql`` on large repos. SELECT-then-write+ for dialect portability (SQLite + Postgres).+ """+ fields = ("pagerank", "betweenness", "community_id", "in_degree", "out_degree")+ for node_id, m in metrics.items():+ result = await session.execute(+ select(GraphMetric).where(+ GraphMetric.repository_id == repository_id,+ GraphMetric.node_id == node_id,+ )+ )+ existing = result.scalar_one_or_none()+ if existing is not None:+ for key in fields:+ if key in m:+ setattr(existing, key, m[key])+ else:+ session.add(+ GraphMetric(+ id=_new_uuid(),+ repository_id=repository_id,+ node_id=node_id,+ pagerank=float(m.get("pagerank", 0.0)),+ betweenness=float(m.get("betweenness", 0.0)),+ community_id=int(m.get("community_id", 0)),+ in_degree=int(m.get("in_degree", 0)),+ out_degree=int(m.get("out_degree", 0)),+ )+ )++ await session.flush()+++async def get_graph_metrics(+ session: AsyncSession,+ repository_id: str,+) -> dict[str, dict]:+ """Read the materialized ``graph_metrics`` snapshot as ``node_id → metrics``."""+ result = await session.execute(+ select(GraphMetric).where(GraphMetric.repository_id == repository_id)+ )+ return {+ row.node_id: {+ "pagerank": row.pagerank,+ "betweenness": row.betweenness,+ "community_id": row.community_id,+ "in_degree": row.in_degree,+ "out_degree": row.out_degree,+ }+ for row in result.scalars().all()+ }++# ---------------------------------------------------------------------------# ExternalSystem CRUD# ---------------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.pyindex ec96e286e..18bf0f0ca 100644--- a/packages/core/src/repowise/core/persistence/models.py+++ b/packages/core/src/repowise/core/persistence/models.py@@ -100,7 +100,7 @@ class Page(Base):page_type: Mapped[str] = mapped_column(String(64), nullable=False)title: Mapped[str] = mapped_column(Text, nullable=False)content: Mapped[str] = mapped_column(Text, nullable=False)- # 1–3 sentence purpose blurb. Always populated (LLM-extracted from content+ # 1-3 sentence purpose blurb. Always populated (LLM-extracted from content# for full mode, deterministic structure summary for index-only mode).# Surfaced by get_context as the default narrative; content is gated# behind include=["full_doc"] to keep MCP responses small.@@ -243,6 +243,34 @@ class GraphEdge(Base):)+class GraphMetric(Base):+ """Materialized file-level graph metrics snapshot (large-repo scale).++ Lets metric reads be served from SQL without recomputing the expensive+ NetworkX centrality kernels on big graphs. Written after the graph is+ built (additive to ``graph_nodes``) and read back into a GraphBuilder via+ ``load_metrics_from_sql``.+ """++ __tablename__ = "graph_metrics"++ id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_new_uuid)+ repository_id: Mapped[str] = mapped_column(+ String(32), ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True+ )+ node_id: Mapped[str] = mapped_column(Text, nullable=False)+ pagerank: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)+ betweenness: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)+ community_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)+ in_degree: Mapped[int] = mapped_column(Integer, nullable=False, default=0)+ out_degree: Mapped[int] = mapped_column(Integer, nullable=False, default=0)+ created_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=True), nullable=False, default=_now_utc+ )++ __table_args__ = (UniqueConstraint("repository_id", "node_id", name="uq_graph_metric"),)++class WebhookEvent(Base):__tablename__ = "webhook_events"diff --git a/packages/core/src/repowise/core/persistence/vector_store.py b/packages/core/src/repowise/core/persistence/vector_store.pydeleted file mode 100644index 6d61559eb..000000000--- a/packages/core/src/repowise/core/persistence/vector_store.py+++ /dev/null@@ -1,536 +0,0 @@-"""Vector store abstraction and implementations for repowise semantic search.--Three implementations are provided:--InMemoryVectorStore- Pure Python, no external dependencies. Cosine similarity search over- an in-memory dict. Suitable for tests and development.--LanceDBVectorStore- Embedded vector database stored in a local directory. Requires the- ``repowise-core[search]`` extra (lancedb>=0.12).--PgVectorStore- Stores embeddings in the ``wiki_pages.embedding`` pgvector column.- Requires the ``repowise-core[pgvector]`` extra and a running PostgreSQL- with the ``vector`` extension enabled.--All stores accept an :class:`Embedder` at construction time and handle-embedding internally — callers pass raw text, not pre-computed vectors.-"""--from __future__ import annotations--import math-from abc import ABC, abstractmethod-from typing import TYPE_CHECKING--from repowise.core.providers.embedding.base import Embedder--from .search import SearchResult--if TYPE_CHECKING:- from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker--__all__ = [- "InMemoryVectorStore",- "LanceDBVectorStore",- "PgVectorStore",- "VectorStore",-]---# ----------------------------------------------------------------------------# Abstract base-# ------------------------------------------------------------------------------class VectorStore(ABC):- """Abstract vector store. All methods are async."""-- @abstractmethod- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:- """Embed *text* and upsert the vector under *page_id*."""- ...-- @abstractmethod- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:- """Embed *query* and return the *limit* nearest pages."""- ...-- @abstractmethod- async def delete(self, page_id: str) -> None:- """Remove the vector for *page_id* from the store."""- ...-- @abstractmethod- async def close(self) -> None:- """Release any resources held by the store."""- ...-- async def list_page_ids(self) -> set[str]:- """Return the set of page IDs currently stored.-- Used by ``repowise doctor --repair`` to detect three-store- inconsistencies. Implementations may override for efficiency.- """- return set() # default: empty (subclasses should override)-- async def get_page_summary_by_path(self, path: str) -> dict | None:- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.-- Used for RAG context injection during doc generation: when generating page B- that imports A, we fetch A's previously-generated summary and feed it to the LLM.- """- return None # default: no-op (subclasses should override)-- async def get_page_summaries_by_paths(- self, paths: list[str]- ) -> dict[str, dict]:- """Batch variant of :meth:`get_page_summary_by_path`.-- Returns a mapping of resolved paths → summary dict for every- input path that produced a non-None result. The default- implementation fires all per-path calls concurrently via- ``asyncio.gather`` so callers don't have to await each one- sequentially — backends that can do a single SQL/index scan- should override this for the obvious efficiency gain.- """- import asyncio as _asyncio-- if not paths:- return {}- coros = [self.get_page_summary_by_path(p) for p in paths]- results = await _asyncio.gather(*coros, return_exceptions=True)- out: dict[str, dict] = {}- for path, result in zip(paths, results, strict=False):- if isinstance(result, dict) and result.get("summary"):- out[path] = result- return out---# ----------------------------------------------------------------------------# InMemoryVectorStore-# ------------------------------------------------------------------------------def _cosine(a: list[float], b: list[float]) -> float:- """Cosine similarity between two vectors (returns 0.0 for zero vectors)."""- dot = sum(x * y for x, y in zip(a, b, strict=False))- norm_a = math.sqrt(sum(x * x for x in a))- norm_b = math.sqrt(sum(x * x for x in b))- denom = norm_a * norm_b- return dot / denom if denom > 0 else 0.0---class InMemoryVectorStore(VectorStore):- """Cosine-similarity vector store backed by a plain Python dict.-- Suitable for unit tests and small-scale development use.- No external dependencies beyond the Embedder.- """-- def __init__(self, embedder: Embedder) -> None:- self._embedder = embedder- # page_id → (vector, metadata)- self._store: dict[str, tuple[list[float], dict]] = {}-- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:- vectors = await self._embedder.embed([text])- self._store[page_id] = (vectors[0], dict(metadata))-- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:- if not self._store:- return []- q_vecs = await self._embedder.embed([query])- q_vec = q_vecs[0]-- scored: list[tuple[float, str, dict]] = []- for pid, (vec, meta) in self._store.items():- score = _cosine(q_vec, vec)- scored.append((score, pid, meta))-- scored.sort(key=lambda t: t[0], reverse=True)-- results = []- for score, pid, meta in scored[:limit]:- content = meta.get("content", "")- results.append(- SearchResult(- page_id=pid,- title=str(meta.get("title", "")),- page_type=str(meta.get("page_type", "")),- target_path=str(meta.get("target_path", "")),- score=score,- snippet=str(content)[:200].rstrip(),- search_type="vector",- )- )- return results-- async def delete(self, page_id: str) -> None:- self._store.pop(page_id, None)-- async def close(self) -> None:- self._store.clear()-- async def list_page_ids(self) -> set[str]:- return set(self._store.keys())-- async def get_page_summary_by_path(self, path: str) -> dict | None:- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.-- Used for RAG context injection during doc generation: when generating page B- that imports A, we fetch A's previously-generated summary and feed it to the LLM.-- Implementation note: reads 'summary' from metadata if present (set by the- generation pipeline), else falls back to the first 500 chars of 'content'.- 'key_exports' reads the 'exports' metadata field if present, else [].- """- for _pid, (_, meta) in self._store.items():- if meta.get("target_path") == path:- summary = meta.get("summary") or str(meta.get("content", ""))[:500]- key_exports = meta.get("exports") or []- return {"summary": summary, "key_exports": list(key_exports)}- return None-- async def get_page_summaries_by_paths(- self, paths: list[str]- ) -> dict[str, dict]:- """Single-pass scan over the store — avoids N full scans when- many paths are queried at once.- """- if not paths:- return {}- wanted = set(paths)- out: dict[str, dict] = {}- for _pid, (_, meta) in self._store.items():- tp = meta.get("target_path")- if tp in wanted and tp not in out:- summary = meta.get("summary") or str(meta.get("content", ""))[:500]- out[str(tp)] = {- "summary": summary,- "key_exports": list(meta.get("exports") or []),- }- if len(out) == len(wanted):- break- return out-- def __len__(self) -> int:- return len(self._store)---# ----------------------------------------------------------------------------# LanceDBVectorStore-# ------------------------------------------------------------------------------class LanceDBVectorStore(VectorStore):- """Vector store backed by LanceDB (embedded, local file storage).-- Requires the ``repowise-core[search]`` extra:- pip install repowise-core[search]-- Data is stored in *db_path* (e.g. ``.repowise/lancedb/``).- The LanceDB table is created lazily on the first call to- :meth:`embed_and_upsert`.- """-- _TABLE_NAME = "wiki_pages"-- def __init__(self, db_path: str, embedder: Embedder, table_name: str | None = None) -> None:- self._db_path = db_path- self._embedder = embedder- self._table_name = table_name or self._TABLE_NAME- self._db = None- self._table = None-- async def _ensure_connected(self) -> None:- if self._db is not None:- return- try:- import lancedb # type: ignore[import]- except ImportError as exc:- raise RuntimeError(- "LanceDB is not installed. Install it with: pip install repowise-core[search]"- ) from exc-- self._db = await lancedb.connect_async(self._db_path)- table_names = await self._db.table_names()- if self._table_name in table_names:- self._table = await self._db.open_table(self._table_name)- else:- self._table = None # will be created on first upsert-- async def _ensure_table(self, sample_vector: list[float]) -> None:- """Create the LanceDB table if it does not exist yet."""- if self._table is not None:- return-- try:- import pyarrow as pa # type: ignore[import]- except ImportError as exc:- raise RuntimeError(- "pyarrow is required for LanceDBVectorStore. "- "It is installed automatically with lancedb."- ) from exc-- dim = len(sample_vector)- schema = pa.schema(- [- pa.field("page_id", pa.string()),- pa.field("vector", pa.list_(pa.float32(), dim)),- pa.field("title", pa.string()),- pa.field("page_type", pa.string()),- pa.field("target_path", pa.string()),- pa.field("content_snippet", pa.string()),- ]- )- self._table = await self._db.create_table( # type: ignore[union-attr]- self._table_name, schema=schema, exist_ok=True- )-- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:- await self._ensure_connected()- vectors = await self._embedder.embed([text])- vector = vectors[0]- await self._ensure_table(vector)-- content = str(metadata.get("content", text))- row = {- "page_id": page_id,- "vector": [float(v) for v in vector],- "title": str(metadata.get("title", "")),- "page_type": str(metadata.get("page_type", "")),- "target_path": str(metadata.get("target_path", "")),- "content_snippet": content[:200],- }-- # merge_insert: upsert by page_id (LanceDB 0.12+)- try:- await (- self._table.merge_insert("page_id")- .when_matched_update_all()- .when_not_matched_insert_all()- .execute([row])- ) # type: ignore[union-attr]- except AttributeError:- # Fallback for older LanceDB versions: delete + add- safe_id = page_id.replace("'", "''")- await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]- await self._table.add([row]) # type: ignore[union-attr]-- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:- await self._ensure_connected()- if self._table is None:- return []-- q_vecs = await self._embedder.embed([query])- q_vec = [float(v) for v in q_vecs[0]]-- raw = (- await self._table.query() # type: ignore[union-attr]- .nearest_to(q_vec)- .limit(limit)- .to_list()- )-- return [- SearchResult(- page_id=r["page_id"],- title=r.get("title", ""),- page_type=r.get("page_type", ""),- target_path=r.get("target_path", ""),- score=float(r.get("_distance", 0.0)),- snippet=r.get("content_snippet", ""),- search_type="vector",- )- for r in raw- ]-- async def delete(self, page_id: str) -> None:- await self._ensure_connected()- if self._table is not None:- safe_id = page_id.replace("'", "''")- await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]-- async def close(self) -> None:- self._table = None- self._db = None-- async def list_page_ids(self) -> set[str]:- await self._ensure_connected()- if self._table is None:- return set()- rows = await self._table.query().select(["page_id"]).to_list() # type: ignore[union-attr]- return {r["page_id"] for r in rows}-- async def get_page_summary_by_path(self, path: str) -> dict | None:- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.-- Used for RAG context injection during doc generation: when generating page B- that imports A, we fetch A's previously-generated summary and feed it to the LLM.-- Implementation note: LanceDB stores up to 200 chars of content in 'content_snippet'.- We use that as the summary. 'key_exports' is not stored in the LanceDB schema, so- we return [] — the caller only uses the text summary for prompt injection.- """- await self._ensure_connected()- if self._table is None:- return None-- safe_path = path.replace("'", "''")- try:- rows = (- await self._table.query() # type: ignore[union-attr]- .where(f"target_path = '{safe_path}'")- .select(["content_snippet"])- .limit(1)- .to_list()- )- except Exception:- return None-- if not rows:- return None-- summary = rows[0].get("content_snippet") or ""- return {"summary": str(summary), "key_exports": []}---# ----------------------------------------------------------------------------# PgVectorStore-# ------------------------------------------------------------------------------class PgVectorStore(VectorStore):- """Vector store that writes embeddings to the ``wiki_pages.embedding`` column.-- Requires:- - PostgreSQL with the ``vector`` extension.- - The Alembic migration ``0001_initial_schema`` has been applied.- - The ``repowise-core[pgvector]`` extra.-- Uses raw SQL to avoid importing ``pgvector.sqlalchemy.Vector`` at module- level (keeps the base package installable without the extra).- """-- def __init__(- self,- session_factory: async_sessionmaker[AsyncSession],- embedder: Embedder,- ) -> None:- self._session_factory = session_factory- self._embedder = embedder-- async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:- vectors = await self._embedder.embed([text])- vector = vectors[0]- # pgvector expects a list encoded as a string like "[0.1, 0.2, ...]"- vec_str = "[" + ",".join(str(v) for v in vector) + "]"-- from sqlalchemy.sql import text as sa_text-- async with self._session_factory() as session:- await session.execute(- sa_text("UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"),- {"emb": vec_str, "pid": page_id},- )- await session.commit()-- async def search(self, query: str, limit: int = 10) -> list[SearchResult]:- q_vecs = await self._embedder.embed([query])- q_vec = q_vecs[0]- vec_str = "[" + ",".join(str(v) for v in q_vec) + "]"-- from sqlalchemy.sql import text as sa_text-- async with self._session_factory() as session:- rows = await session.execute(- sa_text(- "SELECT id, title, content, page_type, target_path, "- " 1 - (embedding <=> CAST(:q AS vector)) AS score "- "FROM wiki_pages "- "WHERE embedding IS NOT NULL "- "ORDER BY embedding <=> CAST(:q AS vector) "- "LIMIT :lim"- ),- {"q": vec_str, "lim": limit},- )- raw = rows.fetchall()-- return [- SearchResult(- page_id=r[0],- title=r[1],- page_type=r[3],- target_path=r[4],- score=float(r[5]),- snippet=str(r[2])[:200].rstrip(),- search_type="vector",- )- for r in raw- ]-- async def delete(self, page_id: str) -> None:- from sqlalchemy.sql import text as sa_text-- async with self._session_factory() as session:- await session.execute(- sa_text("UPDATE wiki_pages SET embedding = NULL WHERE id = :pid"),- {"pid": page_id},- )- await session.commit()-- async def close(self) -> None:- pass # session_factory manages connection lifecycle-- async def list_page_ids(self) -> set[str]:- from sqlalchemy.sql import text as sa_text-- async with self._session_factory() as session:- rows = await session.execute(- sa_text("SELECT id FROM wiki_pages WHERE embedding IS NOT NULL")- )- return {r[0] for r in rows.fetchall()}-- async def get_page_summary_by_path(self, path: str) -> dict | None:- """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.-- Used for RAG context injection during doc generation: when generating page B- that imports A, we fetch A's previously-generated summary and feed it to the LLM.-- Implementation note: reads the 'content' column (first 500 chars) from the- wiki_pages table matched by target_path. 'key_exports' is derived from the- page's exports if stored in a metadata JSON column; otherwise returns [].- """- from sqlalchemy.sql import text as sa_text-- async with self._session_factory() as session:- rows = await session.execute(- sa_text(- "SELECT content, metadata FROM wiki_pages "- "WHERE target_path = :path "- "LIMIT 1"- ),- {"path": path},- )- row = rows.fetchone()-- if row is None:- return None-- content = str(row[0] or "")[:500]- # Extract key_exports from metadata JSON column if present- key_exports: list[str] = []- if row[1] and isinstance(row[1], dict):- key_exports = list(row[1].get("exports", []))- elif row[1] and isinstance(row[1], str):- import json- try:- meta = json.loads(row[1])- key_exports = list(meta.get("exports", []))- except (json.JSONDecodeError, AttributeError):- pass-- return {"summary": content, "key_exports": key_exports}diff --git a/packages/core/src/repowise/core/persistence/vector_store/README.md b/packages/core/src/repowise/core/persistence/vector_store/README.mdnew file mode 100644index 000000000..a9ac9d49a--- /dev/null+++ b/packages/core/src/repowise/core/persistence/vector_store/README.md@@ -0,0 +1,52 @@+# `vector_store`++Pluggable semantic-search backends for repowise wiki pages.++## Purpose++Embed generated wiki pages into vectors and answer nearest-neighbour+queries. Callers pass raw text; each store owns an `Embedder` and handles+embedding internally.++## Public API++Imported as before via `repowise.core.persistence.vector_store`:++- `VectorStore` — abstract base class.+- `InMemoryVectorStore` — pure-Python dict, no extra deps (tests / dev).+- `LanceDBVectorStore` — embedded LanceDB (`repowise-core[search]` extra).+- `PgVectorStore` — `wiki_pages.embedding` pgvector column (`[pgvector]` extra).+- `cosine_similarity(a, b)` — shared helper.++### Methods++- `embed_and_upsert(page_id, text, metadata)` — single-item write.+- `embed_batch(items: list[tuple[id, text, metadata]])` — embed a whole batch+ in one model call. The ABC provides a correct sequential default; each+ concrete store overrides it to make a single `embed()` call plus a bulk+ upsert. Use this on the hot indexing path; keep `embed_and_upsert` for+ incremental single-page updates.+- `search`, `delete`, `close`, `list_page_ids`,+ `get_page_summary_by_path`, `get_page_summaries_by_paths`.++## Internal layout++| Module | Contents |+|--------|----------|+| `_base.py` | `VectorStore` ABC + `cosine_similarity` |+| `in_memory.py` | `InMemoryVectorStore` |+| `lancedb_store.py` | `LanceDBVectorStore` |+| `pgvector_store.py` | `PgVectorStore` |++The package `__init__` re-exports every public name so the historical+`from repowise.core.persistence.vector_store import ...` path is unchanged.++## Extension points++Author a new backend by subclassing `VectorStore` and implementing the four+abstract methods. Override `embed_batch` if your backend can amortise batched+writes; otherwise the default sequential implementation is used.++## Tests++`tests/unit/persistence/test_vector_store.py`.diff --git a/packages/core/src/repowise/core/persistence/vector_store/__init__.py b/packages/core/src/repowise/core/persistence/vector_store/__init__.pynew file mode 100644index 000000000..84716d669--- /dev/null+++ b/packages/core/src/repowise/core/persistence/vector_store/__init__.py@@ -0,0 +1,40 @@+"""Vector store abstraction and implementations for repowise semantic search.++Three implementations are provided:++InMemoryVectorStore+ Pure Python, no external dependencies. Cosine similarity search over+ an in-memory dict. Suitable for tests and development.++LanceDBVectorStore+ Embedded vector database stored in a local directory. Requires the+ ``repowise-core[search]`` extra (lancedb>=0.12).++PgVectorStore+ Stores embeddings in the ``wiki_pages.embedding`` pgvector column.+ Requires the ``repowise-core[pgvector]`` extra and a running PostgreSQL+ with the ``vector`` extension enabled.++All stores accept an :class:`Embedder` at construction time and handle+embedding internally — callers pass raw text, not pre-computed vectors.++Every store implements :meth:`VectorStore.embed_batch`, which embeds a whole+list of pages in a single model call. Large-repo indexing uses the batch path+to avoid one embedding round-trip per page; the single-item+:meth:`VectorStore.embed_and_upsert` path is preserved for incremental updates.+"""++from __future__ import annotations++from ._base import VectorStore, cosine_similarity+from .in_memory import InMemoryVectorStore+from .lancedb_store import LanceDBVectorStore+from .pgvector_store import PgVectorStore++__all__ = [+ "InMemoryVectorStore",+ "LanceDBVectorStore",+ "PgVectorStore",+ "VectorStore",+ "cosine_similarity",+]diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.pynew file mode 100644index 000000000..95507eb2c--- /dev/null+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py@@ -0,0 +1,101 @@+"""Vector store abstract base class and shared helpers.++The concrete implementations live in sibling modules+(:mod:`in_memory`, :mod:`lancedb_store`, :mod:`pgvector_store`) and are+re-exported from the package ``__init__`` so the historical import path+``repowise.core.persistence.vector_store`` keeps working unchanged.+"""++from __future__ import annotations++import math+from abc import ABC, abstractmethod++from ..search import SearchResult++__all__ = ["VectorStore", "cosine_similarity"]+++def cosine_similarity(a: list[float], b: list[float]) -> float:+ """Cosine similarity between two vectors (returns 0.0 for zero vectors)."""+ dot = sum(x * y for x, y in zip(a, b, strict=False))+ norm_a = math.sqrt(sum(x * x for x in a))+ norm_b = math.sqrt(sum(x * x for x in b))+ denom = norm_a * norm_b+ return dot / denom if denom > 0 else 0.0+++class VectorStore(ABC):+ """Abstract vector store. All methods are async."""++ @abstractmethod+ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:+ """Embed *text* and upsert the vector under *page_id*."""+ ...++ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:+ """Embed and upsert many ``(page_id, text, metadata)`` items at once.++ The default implementation processes items sequentially via+ :meth:`embed_and_upsert`, so any backend gets correct behaviour for+ free. Backends that can embed a whole batch in a single model call+ (the common case) override this to amortise the network / GPU+ round-trip — see the bundled stores. Callers may always use this+ path; it never has worse semantics than calling+ :meth:`embed_and_upsert` in a loop.+ """+ for page_id, text, metadata in items:+ await self.embed_and_upsert(page_id, text, metadata)++ @abstractmethod+ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:+ """Embed *query* and return the *limit* nearest pages."""+ ...++ @abstractmethod+ async def delete(self, page_id: str) -> None:+ """Remove the vector for *page_id* from the store."""+ ...++ @abstractmethod+ async def close(self) -> None:+ """Release any resources held by the store."""+ ...++ async def list_page_ids(self) -> set[str]:+ """Return the set of page IDs currently stored.++ Used by ``repowise doctor --repair`` to detect three-store+ inconsistencies. Implementations may override for efficiency.+ """+ return set() # default: empty (subclasses should override)++ async def get_page_summary_by_path(self, path: str) -> dict | None:+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.++ Used for RAG context injection during doc generation: when generating page B+ that imports A, we fetch A's previously-generated summary and feed it to the LLM.+ """+ return None # default: no-op (subclasses should override)++ async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:+ """Batch variant of :meth:`get_page_summary_by_path`.++ Returns a mapping of resolved paths → summary dict for every+ input path that produced a non-None result. The default+ implementation fires all per-path calls concurrently via+ ``asyncio.gather`` so callers don't have to await each one+ sequentially — backends that can do a single SQL/index scan+ should override this for the obvious efficiency gain.+ """+ import asyncio as _asyncio++ if not paths:+ return {}+ coros = [self.get_page_summary_by_path(p) for p in paths]+ results = await _asyncio.gather(*coros, return_exceptions=True)+ out: dict[str, dict] = {}+ for path, result in zip(paths, results, strict=False):+ if isinstance(result, dict) and result.get("summary"):+ out[path] = result+ return outdiff --git a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py b/packages/core/src/repowise/core/persistence/vector_store/in_memory.pynew file mode 100644index 000000000..b97e04633--- /dev/null+++ b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py@@ -0,0 +1,110 @@+"""Pure-Python in-memory vector store (tests + small-scale dev)."""++from __future__ import annotations++from repowise.core.providers.embedding.base import Embedder++from ..search import SearchResult+from ._base import VectorStore, cosine_similarity++__all__ = ["InMemoryVectorStore"]+++class InMemoryVectorStore(VectorStore):+ """Cosine-similarity vector store backed by a plain Python dict.++ Suitable for unit tests and small-scale development use.+ No external dependencies beyond the Embedder.+ """++ def __init__(self, embedder: Embedder) -> None:+ self._embedder = embedder+ # page_id → (vector, metadata)+ self._store: dict[str, tuple[list[float], dict]] = {}++ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:+ vectors = await self._embedder.embed([text])+ self._store[page_id] = (vectors[0], dict(metadata))++ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:+ if not items:+ return+ texts = [text for _, text, _ in items]+ vectors = await self._embedder.embed(texts)+ for (page_id, _text, metadata), vector in zip(items, vectors, strict=True):+ self._store[page_id] = (vector, dict(metadata))++ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:+ if not self._store:+ return []+ q_vecs = await self._embedder.embed([query])+ q_vec = q_vecs[0]++ scored: list[tuple[float, str, dict]] = []+ for pid, (vec, meta) in self._store.items():+ score = cosine_similarity(q_vec, vec)+ scored.append((score, pid, meta))++ scored.sort(key=lambda t: t[0], reverse=True)++ results = []+ for score, pid, meta in scored[:limit]:+ content = meta.get("content", "")+ results.append(+ SearchResult(+ page_id=pid,+ title=str(meta.get("title", "")),+ page_type=str(meta.get("page_type", "")),+ target_path=str(meta.get("target_path", "")),+ score=score,+ snippet=str(content)[:200].rstrip(),+ search_type="vector",+ )+ )+ return results++ async def delete(self, page_id: str) -> None:+ self._store.pop(page_id, None)++ async def close(self) -> None:+ self._store.clear()++ async def list_page_ids(self) -> set[str]:+ return set(self._store.keys())++ async def get_page_summary_by_path(self, path: str) -> dict | None:+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.++ Reads 'summary' from metadata if present (set by the generation+ pipeline), else falls back to the first 500 chars of 'content'.+ 'key_exports' reads the 'exports' metadata field if present, else [].+ """+ for _pid, (_, meta) in self._store.items():+ if meta.get("target_path") == path:+ summary = meta.get("summary") or str(meta.get("content", ""))[:500]+ key_exports = meta.get("exports") or []+ return {"summary": summary, "key_exports": list(key_exports)}+ return None++ async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:+ """Single-pass scan over the store — avoids N full scans when+ many paths are queried at once.+ """+ if not paths:+ return {}+ wanted = set(paths)+ out: dict[str, dict] = {}+ for _pid, (_, meta) in self._store.items():+ tp = meta.get("target_path")+ if tp in wanted and tp not in out:+ summary = meta.get("summary") or str(meta.get("content", ""))[:500]+ out[str(tp)] = {+ "summary": summary,+ "key_exports": list(meta.get("exports") or []),+ }+ if len(out) == len(wanted):+ break+ return out++ def __len__(self) -> int:+ return len(self._store)diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.pynew file mode 100644index 000000000..3d131088f--- /dev/null+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py@@ -0,0 +1,201 @@+"""LanceDB-backed vector store (embedded, local file storage)."""++from __future__ import annotations++from repowise.core.providers.embedding.base import Embedder++from ..search import SearchResult+from ._base import VectorStore++__all__ = ["LanceDBVectorStore"]+++class LanceDBVectorStore(VectorStore):+ """Vector store backed by LanceDB (embedded, local file storage).++ Requires the ``repowise-core[search]`` extra:+ pip install repowise-core[search]++ Data is stored in *db_path* (e.g. ``.repowise/lancedb/``).+ The LanceDB table is created lazily on the first call to+ :meth:`embed_and_upsert`.+ """++ _TABLE_NAME = "wiki_pages"++ def __init__(+ self, db_path: str, embedder: Embedder, table_name: str | None = None+ ) -> None:+ self._db_path = db_path+ self._embedder = embedder+ self._table_name = table_name or self._TABLE_NAME+ self._db = None+ self._table = None++ async def _ensure_connected(self) -> None:+ if self._db is not None:+ return+ try:+ import lancedb # type: ignore[import]+ except ImportError as exc:+ raise RuntimeError(+ "LanceDB is not installed. Install it with: pip install repowise-core[search]"+ ) from exc++ self._db = await lancedb.connect_async(self._db_path)+ table_names = await self._db.table_names()+ if self._table_name in table_names:+ self._table = await self._db.open_table(self._table_name)+ else:+ self._table = None # will be created on first upsert++ async def _ensure_table(self, sample_vector: list[float]) -> None:+ """Create the LanceDB table if it does not exist yet."""+ if self._table is not None:+ return++ try:+ import pyarrow as pa # type: ignore[import]+ except ImportError as exc:+ raise RuntimeError(+ "pyarrow is required for LanceDBVectorStore. "+ "It is installed automatically with lancedb."+ ) from exc++ dim = len(sample_vector)+ schema = pa.schema(+ [+ pa.field("page_id", pa.string()),+ pa.field("vector", pa.list_(pa.float32(), dim)),+ pa.field("title", pa.string()),+ pa.field("page_type", pa.string()),+ pa.field("target_path", pa.string()),+ pa.field("content_snippet", pa.string()),+ ]+ )+ self._table = await self._db.create_table( # type: ignore[union-attr]+ self._table_name, schema=schema, exist_ok=True+ )++ @staticmethod+ def _row(page_id: str, vector: list[float], metadata: dict) -> dict:+ content = str(metadata.get("content", ""))+ return {+ "page_id": page_id,+ "vector": [float(v) for v in vector],+ "title": str(metadata.get("title", "")),+ "page_type": str(metadata.get("page_type", "")),+ "target_path": str(metadata.get("target_path", "")),+ "content_snippet": content[:200],+ }++ async def _upsert_rows(self, rows: list[dict]) -> None:+ # merge_insert: upsert by page_id (LanceDB 0.12+)+ try:+ await (+ self._table.merge_insert("page_id") # type: ignore[union-attr]+ .when_matched_update_all()+ .when_not_matched_insert_all()+ .execute(rows)+ )+ except AttributeError:+ # Fallback for older LanceDB versions: delete + add+ for row in rows:+ safe_id = str(row["page_id"]).replace("'", "''")+ await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]+ await self._table.add(rows) # type: ignore[union-attr]++ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:+ await self._ensure_connected()+ vectors = await self._embedder.embed([text])+ vector = vectors[0]+ await self._ensure_table(vector)+ meta = {"content": text, **metadata}+ await self._upsert_rows([self._row(page_id, vector, meta)])++ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:+ if not items:+ return+ await self._ensure_connected()+ texts = [text for _, text, _ in items]+ vectors = await self._embedder.embed(texts)+ await self._ensure_table(vectors[0])+ rows = [+ self._row(page_id, vector, {"content": text, **metadata})+ for (page_id, text, metadata), vector in zip(items, vectors, strict=True)+ ]+ await self._upsert_rows(rows)++ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:+ await self._ensure_connected()+ if self._table is None:+ return []++ q_vecs = await self._embedder.embed([query])+ q_vec = [float(v) for v in q_vecs[0]]++ raw = (+ await self._table.query() # type: ignore[union-attr]+ .nearest_to(q_vec)+ .limit(limit)+ .to_list()+ )++ return [+ SearchResult(+ page_id=r["page_id"],+ title=r.get("title", ""),+ page_type=r.get("page_type", ""),+ target_path=r.get("target_path", ""),+ score=float(r.get("_distance", 0.0)),+ snippet=r.get("content_snippet", ""),+ search_type="vector",+ )+ for r in raw+ ]++ async def delete(self, page_id: str) -> None:+ await self._ensure_connected()+ if self._table is not None:+ safe_id = page_id.replace("'", "''")+ await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]++ async def close(self) -> None:+ self._table = None+ self._db = None++ async def list_page_ids(self) -> set[str]:+ await self._ensure_connected()+ if self._table is None:+ return set()+ rows = await self._table.query().select(["page_id"]).to_list() # type: ignore[union-attr]+ return {r["page_id"] for r in rows}++ async def get_page_summary_by_path(self, path: str) -> dict | None:+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.++ LanceDB stores up to 200 chars of content in 'content_snippet'; we use+ that as the summary. 'key_exports' is not stored in the schema, so we+ return [] — the caller only uses the text summary for prompt injection.+ """+ await self._ensure_connected()+ if self._table is None:+ return None++ safe_path = path.replace("'", "''")+ try:+ rows = (+ await self._table.query() # type: ignore[union-attr]+ .where(f"target_path = '{safe_path}'")+ .select(["content_snippet"])+ .limit(1)+ .to_list()+ )+ except Exception:+ return None++ if not rows:+ return None++ summary = rows[0].get("content_snippet") or ""+ return {"summary": str(summary), "key_exports": []}diff --git a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.pynew file mode 100644index 000000000..4a5f7f40e--- /dev/null+++ b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py@@ -0,0 +1,169 @@+"""PostgreSQL/pgvector-backed vector store."""++from __future__ import annotations++from typing import TYPE_CHECKING++from repowise.core.providers.embedding.base import Embedder++from ..search import SearchResult+from ._base import VectorStore++if TYPE_CHECKING:+ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++__all__ = ["PgVectorStore"]+++def _encode(vector: list[float]) -> str:+ """Encode a vector as the pgvector literal ``"[0.1,0.2,...]"``."""+ return "[" + ",".join(str(v) for v in vector) + "]"+++class PgVectorStore(VectorStore):+ """Vector store that writes embeddings to the ``wiki_pages.embedding`` column.++ Requires:+ - PostgreSQL with the ``vector`` extension.+ - The Alembic migration ``0001_initial_schema`` has been applied.+ - The ``repowise-core[pgvector]`` extra.++ Uses raw SQL to avoid importing ``pgvector.sqlalchemy.Vector`` at module+ level (keeps the base package installable without the extra).+ """++ def __init__(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ embedder: Embedder,+ ) -> None:+ self._session_factory = session_factory+ self._embedder = embedder++ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict) -> None:+ vectors = await self._embedder.embed([text])+ vec_str = _encode(vectors[0])++ from sqlalchemy.sql import text as sa_text++ async with self._session_factory() as session:+ await session.execute(+ sa_text(+ "UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"+ ),+ {"emb": vec_str, "pid": page_id},+ )+ await session.commit()++ async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:+ if not items:+ return+ texts = [text for _, text, _ in items]+ vectors = await self._embedder.embed(texts)+ params = [+ {"emb": _encode(vector), "pid": page_id}+ for (page_id, _text, _meta), vector in zip(items, vectors, strict=True)+ ]++ from sqlalchemy.sql import text as sa_text++ stmt = sa_text(+ "UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid"+ )+ async with self._session_factory() as session:+ for p in params:+ await session.execute(stmt, p)+ await session.commit()++ async def search(self, query: str, limit: int = 10) -> list[SearchResult]:+ q_vecs = await self._embedder.embed([query])+ vec_str = _encode(q_vecs[0])++ from sqlalchemy.sql import text as sa_text++ async with self._session_factory() as session:+ rows = await session.execute(+ sa_text(+ "SELECT id, title, content, page_type, target_path, "+ " 1 - (embedding <=> CAST(:q AS vector)) AS score "+ "FROM wiki_pages "+ "WHERE embedding IS NOT NULL "+ "ORDER BY embedding <=> CAST(:q AS vector) "+ "LIMIT :lim"+ ),+ {"q": vec_str, "lim": limit},+ )+ raw = rows.fetchall()++ return [+ SearchResult(+ page_id=r[0],+ title=r[1],+ page_type=r[3],+ target_path=r[4],+ score=float(r[5]),+ snippet=str(r[2])[:200].rstrip(),+ search_type="vector",+ )+ for r in raw+ ]++ async def delete(self, page_id: str) -> None:+ from sqlalchemy.sql import text as sa_text++ async with self._session_factory() as session:+ await session.execute(+ sa_text("UPDATE wiki_pages SET embedding = NULL WHERE id = :pid"),+ {"pid": page_id},+ )+ await session.commit()++ async def close(self) -> None:+ pass # session_factory manages connection lifecycle++ async def list_page_ids(self) -> set[str]:+ from sqlalchemy.sql import text as sa_text++ async with self._session_factory() as session:+ rows = await session.execute(+ sa_text("SELECT id FROM wiki_pages WHERE embedding IS NOT NULL")+ )+ return {r[0] for r in rows.fetchall()}++ async def get_page_summary_by_path(self, path: str) -> dict | None:+ """Return {'summary': str, 'key_exports': list[str]} for a previously-indexed page, or None.++ Reads the 'content' column (first 500 chars) from the wiki_pages table+ matched by target_path. 'key_exports' is derived from the page's+ ``exports`` if stored in a metadata JSON column; otherwise returns [].+ """+ from sqlalchemy.sql import text as sa_text++ async with self._session_factory() as session:+ rows = await session.execute(+ sa_text(+ "SELECT content, metadata FROM wiki_pages "+ "WHERE target_path = :path "+ "LIMIT 1"+ ),+ {"path": path},+ )+ row = rows.fetchone()++ if row is None:+ return None++ content = str(row[0] or "")[:500]+ key_exports: list[str] = []+ if row[1] and isinstance(row[1], dict):+ key_exports = list(row[1].get("exports", []))+ elif row[1] and isinstance(row[1], str):+ import json++ try:+ meta = json.loads(row[1])+ key_exports = list(meta.get("exports", []))+ except (json.JSONDecodeError, AttributeError):+ pass++ return {"summary": content, "key_exports": key_exports}diff --git a/packages/core/src/repowise/core/pipeline/checkpoint.py b/packages/core/src/repowise/core/pipeline/checkpoint.pynew file mode 100644index 000000000..8236fd207--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/checkpoint.py@@ -0,0 +1,194 @@+"""Pipeline checkpoint/resume integration over the Phase-1 ``JobStore`` seam.++On a large repo a crashed ``repowise init`` should not start from scratch. This+module records each pipeline phase's lifecycle to a :class:`JobStore` so a+re-run can detect which phases already completed and skip them.++Design+------+Phase boundaries are observed through the existing ``pipeline_hooks`` registry+(``HookProgressCallback`` fires ``PRE``/``POST`` hooks around every+``on_phase_start`` / ``on_phase_done``). Those hooks are **synchronous** but+fire inside the running event loop, while ``JobStore`` writes are **async**. To+bridge the two without losing ordering, the hooks push lightweight events onto+an ``asyncio.Queue`` that a single background consumer drains in order —+guaranteeing a phase's ``create_job`` happens before its completion update.++The whole mechanism is **opt-in**: it only activates when a ``JobStore`` is+supplied to :class:`PhaseCheckpointer`. When absent, the orchestrator runs+exactly as before — no new writes, no behaviour change.+"""++from __future__ import annotations++import asyncio+import enum+from typing import TYPE_CHECKING++import structlog++from repowise.core.registry import HookPhase, pipeline_hooks++if TYPE_CHECKING:+ from repowise.core.persistence._interfaces.job_store import JobStore+ from repowise.core.registry import PipelineHookRegistry++logger = structlog.get_logger(__name__)++__all__ = [+ "DEFAULT_CHECKPOINT_PHASES",+ "PhaseCheckpointer",+ "find_completed_phases",+]++# Major phases worth checkpointing. These names match the strings the+# orchestrator passes to ``progress.on_phase_start`` / ``on_phase_done``.+DEFAULT_CHECKPOINT_PHASES: tuple[str, ...] = (+ "parse",+ "git",+ "co_change",+ "generation",+)+++class _EventKind(enum.Enum):+ START = "start"+ DONE = "done"+ STOP = "stop" # sentinel to shut the consumer down+++async def find_completed_phases(+ job_store: JobStore, repository_id: str+) -> set[str]:+ """Return the set of phases already COMPLETED for *repository_id*.++ Used by the resume path to skip work that a prior run finished.+ """+ from repowise.core.persistence._interfaces.job_store import JobState++ jobs = await job_store.list_jobs(repository_id=repository_id)+ return {j.phase for j in jobs if j.state == JobState.COMPLETED}+++class PhaseCheckpointer:+ """Records phase lifecycle to a ``JobStore``, driven by pipeline hooks.++ Use as an async context manager around a pipeline run::++ async with PhaseCheckpointer(job_store, repo_id) as cp:+ await run_pipeline(..., )++ The consumer task started on ``__aenter__`` drains queued phase events; on+ ``__aexit__`` it is signalled to stop and awaited so all writes land before+ control returns.+ """++ def __init__(+ self,+ job_store: JobStore,+ repository_id: str,+ *,+ phases: tuple[str, ...] = DEFAULT_CHECKPOINT_PHASES,+ registry: PipelineHookRegistry | None = None,+ ) -> None:+ self._job_store = job_store+ self._repository_id = repository_id+ self._phases = phases+ self._registry = registry if registry is not None else pipeline_hooks+ self._queue: asyncio.Queue[tuple[_EventKind, str]] = asyncio.Queue()+ self._job_ids: dict[str, str] = {}+ self._consumer: asyncio.Task[None] | None = None+ self._registered: list[tuple[str, HookPhase, object]] = []++ # -- hook callbacks (sync, fired inside the loop) ----------------------++ def _enqueue_start(self, phase: str) -> None:+ self._queue.put_nowait((_EventKind.START, phase))++ def _enqueue_done(self, phase: str) -> None:+ self._queue.put_nowait((_EventKind.DONE, phase))++ def register(self) -> None:+ """Register pre/post hooks for every tracked phase."""+ for phase in self._phases:+ start_cb = self._make_cb(self._enqueue_start)+ done_cb = self._make_cb(self._enqueue_done)+ self._registry.register(phase, start_cb, when=HookPhase.PRE)+ self._registry.register(phase, done_cb, when=HookPhase.POST)+ self._registered.append((phase, HookPhase.PRE, start_cb))+ self._registered.append((phase, HookPhase.POST, done_cb))++ @staticmethod+ def _make_cb(fn: object):+ # pipeline_hooks expects Callable[[str], None]; bind through a plain+ # function so each registration is a distinct, removable object.+ def _cb(phase: str) -> None:+ fn(phase) # type: ignore[operator]++ return _cb++ def unregister(self) -> None:+ """Best-effort removal of this checkpointer's hooks from the registry."""+ hooks = getattr(self._registry, "_hooks", None)+ if not isinstance(hooks, dict):+ return+ for phase, when, cb in self._registered:+ bucket = hooks.get((phase, when))+ if bucket and cb in bucket:+ bucket.remove(cb)+ self._registered.clear()++ # -- async consumer ----------------------------------------------------++ async def _consume(self) -> None:+ from repowise.core.persistence._interfaces.job_store import JobState++ while True:+ kind, phase = await self._queue.get()+ try:+ if kind is _EventKind.STOP:+ return+ if kind is _EventKind.START:+ job = await self._job_store.create_job(+ repository_id=self._repository_id, phase=phase+ )+ self._job_ids[phase] = job.id+ await self._job_store.update_state(job.id, JobState.RUNNING)+ elif kind is _EventKind.DONE:+ job_id = self._job_ids.get(phase)+ if job_id is not None:+ await self._job_store.update_state(+ job_id, JobState.COMPLETED+ )+ except Exception as exc: # checkpointing must never break a run+ logger.warning(+ "checkpoint_write_failed", phase=phase, kind=kind.value, error=str(exc)+ )+ finally:+ self._queue.task_done()++ async def start(self) -> None:+ """Register hooks and start the background consumer."""+ self.register()+ self._consumer = asyncio.create_task(self._consume())++ async def aclose(self) -> None:+ """Drain queued events, stop the consumer, and remove hooks.++ Call on the success path. If the pipeline raises instead, leaving this+ uncalled is intentional: in-flight phases stay in ``RUNNING`` so a+ re-run's ``find_resumable`` can detect the interrupted work.+ """+ await self._queue.join()+ self._queue.put_nowait((_EventKind.STOP, ""))+ if self._consumer is not None:+ await self._consumer+ self._consumer = None+ self.unregister()++ async def __aenter__(self) -> PhaseCheckpointer:+ await self.start()+ return self++ async def __aexit__(self, *exc: object) -> None:+ await self.aclose()diff --git a/packages/core/src/repowise/core/pipeline/modes.py b/packages/core/src/repowise/core/pipeline/modes.pynew file mode 100644index 000000000..7b55fd8f5--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/modes.py@@ -0,0 +1,58 @@+"""Orchestrator execution modes.++``OrchestratorMode`` selects how much of the pipeline runs. It is a thin policy+object over the existing :func:`run_pipeline` — not a fork of it — so the two+modes share one code path and differ only in the knobs they flip.++- ``STANDARD`` (default): the historical behaviour. FULL git indexing, all+ analysis, and (when requested) LLM doc generation.+- ``FAST``: graph + ESSENTIAL git only, no LLM doc generation. Intended for a+ quick first index of a very large repo; the deferred git signals can be+ backfilled later (see ``git_indexer.backfill``).+"""++from __future__ import annotations++import enum++from repowise.core.ingestion.git_indexer import GitIndexTier++__all__ = ["OrchestratorMode"]+++class OrchestratorMode(enum.StrEnum):+ """Pipeline execution depth."""++ STANDARD = "standard"+ FAST = "fast"++ @property+ def git_tier(self) -> GitIndexTier:+ """Git-indexing tier this mode runs."""+ return (+ GitIndexTier.ESSENTIAL if self is OrchestratorMode.FAST else GitIndexTier.FULL+ )++ @property+ def allows_doc_generation(self) -> bool:+ """Whether LLM doc generation may run in this mode.++ FAST never generates docs (and never makes LLM calls), regardless of+ the ``generate_docs`` flag passed to the pipeline.+ """+ return self is not OrchestratorMode.FAST++ @property+ def sql_backed_metrics(self) -> bool:+ """Whether file-level metrics should be served from SQL post-build.++ When ``True`` the pipeline materializes the metrics snapshot to+ ``graph_metrics`` and may drop the in-memory NetworkX object (via+ ``GraphBuilder.release_graph``), serving subsequent metric reads from+ the loaded snapshot — the large-repo scaling path.++ Default off in ``STANDARD`` (doc generation traverses the live graph)+ and on in ``FAST`` (which generates no docs, so nothing needs the+ structural graph after build + persistence).+ """+ return self is OrchestratorMode.FASTdiff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 564dea1d9..8f6bbb92a 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -24,6 +24,8 @@ from typing import Anyimport structlog+from repowise.core.ingestion.git_indexer import GitIndexTier+from repowise.core.pipeline.modes import OrchestratorModefrom repowise.core.pipeline.progress import ProgressCallbackfrom repowise.core.registry import HookProgressCallback@@ -214,6 +216,8 @@ async def run_pipeline(concurrency: int = 5,test_run: bool = False,resume: bool = False,+ mode: OrchestratorMode = OrchestratorMode.STANDARD,+ job_store: Any | None = None,progress: ProgressCallback | None = None,cost_tracker: Any | None = None,generation_config: Any | None = None,@@ -266,10 +270,28 @@ async def run_pipeline(commit_depth = max(1, min(commit_depth, 5000))+ # Mode policy: FAST forces ESSENTIAL git indexing and disables doc+ # generation (and therefore all LLM calls). STANDARD preserves the+ # caller's flags exactly. This is the single switch point — the rest of+ # the pipeline reads ``generate_docs`` / the git tier, not ``mode``.+ git_tier = mode.git_tier+ generate_docs = generate_docs and mode.allows_doc_generation+# Wrap the incoming progress callback so registered pipeline hooks fire# around each phase transition. Zero-op when no hooks are registered.progress = HookProgressCallback(progress)+ # Optional crash-resume checkpointing. When a JobStore is supplied, a+ # checkpointer records each major phase's lifecycle (via the same hook+ # seam) so a re-run can detect completed phases. No-op otherwise — the+ # default (None) preserves existing behaviour exactly.+ _checkpointer = None+ if job_store is not None:+ from repowise.core.pipeline.checkpoint import PhaseCheckpointer++ _checkpointer = PhaseCheckpointer(job_store, str(repo_path))+ await _checkpointer.start()+# Attach cost tracker to provider if suppliedif cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):llm_client._cost_tracker = cost_tracker@@ -288,6 +310,7 @@ async def run_pipeline(repo_path,commit_depth=commit_depth,follow_renames=follow_renames,+ tier=git_tier,progress=progress,)@@ -473,6 +496,12 @@ async def run_pipeline(_phase_done(progress, "graph.flows")# ---- Build result -------------------------------------------------------+ # Flush checkpoints on the success path; on an exception above this point+ # the checkpointer is intentionally not closed so interrupted phases stay+ # RUNNING for a resumable re-run.+ if _checkpointer is not None:+ await _checkpointer.aclose()+elapsed = time.monotonic() - startlanguages = {fi.language for fi in file_infos if hasattr(fi, "language") and fi.language}symbol_count = sum(len(pf.symbols) for pf in parsed_files)@@ -793,6 +822,7 @@ async def _run_git_indexing(*,commit_depth: int,follow_renames: bool,+ tier: GitIndexTier = GitIndexTier.FULL,progress: ProgressCallback | None,) -> tuple[Any | None, list[dict], dict[str, dict]]:"""Run git history indexing.@@ -806,6 +836,7 @@ async def _run_git_indexing(repo_path,commit_limit=commit_depth,follow_renames=follow_renames,+ tier=tier,)def _on_start(total: int) -> None:diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 0456351dc..751b4c8a6 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -27,7 +27,10 @@ async def persist_graph_nodes(update path can refresh ``graph_nodes`` (including symbol-levelPageRank / betweenness) without constructing a full ``PipelineResult``."""- from repowise.core.persistence import batch_upsert_graph_nodes+ from repowise.core.persistence import (+ batch_upsert_graph_metrics,+ batch_upsert_graph_nodes,+ )graph = graph_builder.graph()pr = graph_builder.pagerank()@@ -96,6 +99,16 @@ async def persist_graph_nodes(if nodes:await batch_upsert_graph_nodes(session, repo_id, nodes)+ # Materialize the file-level metrics snapshot (graph_metrics) so large+ # repos can serve metric reads from SQL without recomputing the NetworkX+ # centrality kernels. Additive to graph_nodes; never changes node rows.+ try:+ await batch_upsert_graph_metrics(+ session, repo_id, graph_builder.file_metrics_snapshot()+ )+ except Exception as exc: # materialization is non-load-bearing+ logger.warning("graph_metrics_materialize_skipped", error=str(exc))+async def persist_pipeline_result(result: Any,diff --git a/scripts/benchmark_large_repo.py b/scripts/benchmark_large_repo.pynew file mode 100644index 000000000..03dbd9188--- /dev/null+++ b/scripts/benchmark_large_repo.py@@ -0,0 +1,181 @@+#!/usr/bin/env python3+"""Benchmark the repowise pipeline against a synthetic large repository.++Generates a throwaway git repo with a configurable number of source files+(default 5,000) wired together with cross-module imports, then runs the+indexing pipeline and records per-phase wall-clock to+``bench/results/<git-sha>.json`` (gitignored).++The 5k size exercises the Phase-2 scale work (tiered git indexing, ESSENTIAL+``--mode fast`` path, batched embedding, checkpointing). A 30k dry-run is a+later milestone.++Usage::++ python scripts/benchmark_large_repo.py # 5k files, fast mode+ python scripts/benchmark_large_repo.py --files 2000+ python scripts/benchmark_large_repo.py --mode standard --keep+"""++from __future__ import annotations++import argparse+import asyncio+import json+import subprocess+import sys+import tempfile+import time+from datetime import UTC, datetime+from pathlib import Path++REPO_ROOT = Path(__file__).resolve().parent.parent+++def _git(repo: Path, *args: str) -> str:+ return subprocess.run(+ ["git", *args],+ cwd=repo,+ capture_output=True,+ text=True,+ check=True,+ ).stdout.strip()+++def generate_synthetic_repo(root: Path, n_files: int, n_commits: int) -> int:+ """Create *n_files* Python modules under *root* with cross-imports and a+ git history of *n_commits* commits. Returns the number of files written.++ Modules are grouped into packages of 100 and each imports a symbol from a+ lower-numbered module so the dependency graph has real edges (and PageRank+ has something to rank).+ """+ root.mkdir(parents=True, exist_ok=True)+ _git(root, "init", "-q")+ _git(root, "config", "user.email", "[email protected]")+ _git(root, "config", "user.name", "bench")++ files_per_pkg = 100+ written = 0+ for i in range(n_files):+ pkg = root / f"pkg_{i // files_per_pkg:03d}"+ pkg.mkdir(exist_ok=True)+ (pkg / "__init__.py").touch()+ mod = pkg / f"mod_{i:05d}.py"+ # Import from an earlier module to create a graph edge.+ import_line = ""+ if i > 0:+ dep = i - 1+ import_line = (+ f"from pkg_{dep // files_per_pkg:03d}.mod_{dep:05d} import value_{dep}\n"+ )+ mod.write_text(+ f'"""Synthetic module {i}."""\n'+ f"{import_line}\n"+ f"value_{i} = {i}\n\n\n"+ f"def compute_{i}(x: int) -> int:\n"+ f' """Return x scaled by module ordinal {i}."""\n'+ f" return x * value_{i}\n"+ )+ written += 1++ # Spread the files across n_commits commits so git indexing has history.+ _git(root, "add", "-A")+ _git(root, "commit", "-q", "-m", "feat: initial synthetic tree")+ commits = max(1, n_commits - 1)+ step = max(1, n_files // commits)+ for c in range(commits):+ # Touch a slice of files to create churn + co-change-able commits.+ touched = []+ for i in range(c * step, min((c + 1) * step, n_files)):+ mod = root / f"pkg_{i // files_per_pkg:03d}" / f"mod_{i:05d}.py"+ with mod.open("a") as fh:+ fh.write(f"\n# touch {c}\n")+ touched.append(str(mod.relative_to(root)))+ if touched:+ _git(root, "add", *touched)+ _git(root, "commit", "-q", "-m", f"refactor: tweak batch {c}")+ return written+++async def _run(repo: Path, mode_name: str) -> tuple[dict[str, float], float, object]:+ from repowise.core.pipeline import PhaseTimingRecorder, run_pipeline+ from repowise.core.pipeline.modes import OrchestratorMode++ mode = OrchestratorMode(mode_name)+ recorder = PhaseTimingRecorder(None)+ start = time.monotonic()+ result = await run_pipeline(+ repo,+ generate_docs=False,+ mode=mode,+ progress=recorder,+ )+ total = time.monotonic() - start+ return recorder.timings, total, result+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--files", type=int, default=5000, help="Number of source files.")+ parser.add_argument("--commits", type=int, default=50, help="Number of git commits.")+ parser.add_argument(+ "--mode", choices=["fast", "standard"], default="fast", help="Pipeline mode."+ )+ parser.add_argument(+ "--keep", action="store_true", help="Keep the generated repo (print its path)."+ )+ parser.add_argument(+ "--output-dir",+ type=Path,+ default=REPO_ROOT / "bench" / "results",+ help="Where to write the result JSON (gitignored).",+ )+ args = parser.parse_args(argv)++ tmp = Path(tempfile.mkdtemp(prefix="repowise-bench-"))+ repo = tmp / "synthetic_repo"+ print(f"Generating {args.files} files across {args.commits} commits in {repo} ...")+ gen_start = time.monotonic()+ n = generate_synthetic_repo(repo, args.files, args.commits)+ gen_secs = time.monotonic() - gen_start+ print(f"Generated {n} files in {gen_secs:.1f}s. Running pipeline (mode={args.mode}) ...")++ timings, total, result = asyncio.run(_run(repo, args.mode))++ try:+ sha = _git(REPO_ROOT, "rev-parse", "--short", "HEAD")+ except Exception:+ sha = "unknown"++ record = {+ "git_sha": sha,+ "timestamp": datetime.now(UTC).isoformat(),+ "mode": args.mode,+ "files_requested": args.files,+ "files_generated": n,+ "commits": args.commits,+ "generation_seconds": round(gen_secs, 2),+ "pipeline_total_seconds": round(total, 2),+ "phase_timings": timings,+ "files_indexed": getattr(result, "file_count", None),+ "symbol_count": getattr(result, "symbol_count", None),+ }++ args.output_dir.mkdir(parents=True, exist_ok=True)+ out = args.output_dir / f"{sha}.json"+ out.write_text(json.dumps(record, indent=2))+ print(f"\nWrote {out}")+ print(json.dumps(record, indent=2))++ if args.keep:+ print(f"\nKept synthetic repo at {repo}")+ else:+ import shutil++ shutil.rmtree(tmp, ignore_errors=True)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tests/integration/persistence/test_embed_batch_pg.py b/tests/integration/persistence/test_embed_batch_pg.pynew file mode 100644index 000000000..a049cb236--- /dev/null+++ b/tests/integration/persistence/test_embed_batch_pg.py@@ -0,0 +1,97 @@+"""PgVectorStore.embed_batch against a real pgvector-enabled PostgreSQL.++Skipped automatically when ``testcontainers`` is not installed or Docker is+not running. Exercises the batched embed path (one embedder call + per-row+UPDATE of ``wiki_pages.embedding``) on the dialect production uses.+"""++from __future__ import annotations++from datetime import UTC, datetime++import pytest+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine+from sqlalchemy.sql import text as sa_text++from repowise.core.persistence.database import init_db+from repowise.core.persistence.models import Page+from repowise.core.persistence.stores import SqlIndexStore+from repowise.core.persistence.vector_store import PgVectorStore+from repowise.core.providers.embedding.base import MockEmbedder++testcontainers = pytest.importorskip(+ "testcontainers.postgres",+ reason="testcontainers not installed",+)++[email protected](scope="module")+def pgvector_container():+ try:+ with testcontainers.PostgresContainer("pgvector/pgvector:pg16") as pg:+ yield pg+ except Exception as exc: # pragma: no cover - skip when Docker / image missing+ pytest.skip(f"pgvector container unavailable: {exc}")+++async def pg_factory(pgvector_container):+ raw_url = pgvector_container.get_connection_url()+ url = raw_url.replace("postgresql+psycopg2://", "postgresql+asyncpg://").replace(+ "postgresql://", "postgresql+asyncpg://"+ )+ engine = create_async_engine(url, echo=False)+ try:+ await init_db(engine)+ # The pgvector ``embedding`` column is added by the Alembic migration in+ # production; create_all does not, so add it here to match.+ async with engine.begin() as conn:+ await conn.execute(sa_text("CREATE EXTENSION IF NOT EXISTS vector"))+ await conn.execute(+ sa_text(+ "ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS "+ f"embedding vector({MockEmbedder.dimensions})"+ )+ )+ yield async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)+ finally:+ await engine.dispose()+++async def test_embed_batch_writes_embeddings_postgres(pg_factory):+ async with pg_factory() as session:+ repo = await SqlIndexStore(session).upsert_repository(+ name="pg-embed", local_path="/tmp/pg-embed"+ )+ now = datetime.now(UTC)+ for i in range(3):+ session.add(+ Page(+ id=f"file_page:m{i}.py",+ repository_id=repo.id,+ page_type="file_page",+ title=f"File m{i}",+ content=f"content {i}",+ summary="",+ target_path=f"m{i}.py",+ source_hash="h",+ model_name="mock-model-1",+ provider_name="mock",+ created_at=now,+ updated_at=now,+ )+ )+ await session.commit()++ store = PgVectorStore(pg_factory, MockEmbedder())+ assert await store.list_page_ids() == set() # no embeddings yet++ items = [(f"file_page:m{i}.py", f"content {i}", {"page_type": "file_page"}) for i in range(3)]+ await store.embed_batch(items)++ assert await store.list_page_ids() == {f"file_page:m{i}.py" for i in range(3)}++ # Empty batch is a no-op.+ await store.embed_batch([])+ assert len(await store.list_page_ids()) == 3diff --git a/tests/integration/persistence/test_graph_metrics_pg.py b/tests/integration/persistence/test_graph_metrics_pg.pynew file mode 100644index 000000000..a7b5ba80b--- /dev/null+++ b/tests/integration/persistence/test_graph_metrics_pg.py@@ -0,0 +1,123 @@+"""graph_metrics materialization round-trip against a real PostgreSQL container.++Skipped automatically when ``testcontainers`` is not installed or Docker is+not running. Proves the materialized snapshot round-trips through Postgres and+that a GraphBuilder hydrated from it reproduces the NetworkX metrics.+"""++from __future__ import annotations++from datetime import datetime++import pytest+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine++from repowise.core.ingestion.graph import GraphBuilder+from repowise.core.ingestion.models import FileInfo, Import, ParsedFile+from repowise.core.persistence import batch_upsert_graph_metrics, get_graph_metrics+from repowise.core.persistence.database import init_db+from repowise.core.persistence.stores import SqlIndexStore++testcontainers = pytest.importorskip(+ "testcontainers.postgres",+ reason="testcontainers not installed",+)++[email protected](scope="module")+def postgres_container():+ try:+ with testcontainers.PostgresContainer("postgres:16-alpine") as pg:+ yield pg+ except Exception as exc: # pragma: no cover - skip when Docker missing+ pytest.skip(f"PostgreSQL container unavailable: {exc}")+++async def pg_session(postgres_container):+ raw_url = postgres_container.get_connection_url()+ url = raw_url.replace("postgresql+psycopg2://", "postgresql+asyncpg://").replace(+ "postgresql://", "postgresql+asyncpg://"+ )+ engine = create_async_engine(url, echo=False)+ try:+ await init_db(engine)+ factory = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)+ async with factory() as session:+ yield session+ finally:+ await engine.dispose()+++def _build_sample() -> GraphBuilder:+ def fi(path: str) -> FileInfo:+ return FileInfo(+ path=path,+ abs_path=f"/repo/{path}",+ language="python",+ size_bytes=100,+ git_hash="",+ last_modified=datetime.now(),+ is_test=False,+ is_config=False,+ is_api_contract=False,+ is_entry_point=False,+ )++ def imp(mp: str) -> Import:+ return Import(+ raw_statement=f"import {mp}",+ module_path=mp,+ imported_names=[],+ is_relative=False,+ resolved_file=None,+ )++ def parsed(path: str, imports: list[Import]) -> ParsedFile:+ return ParsedFile(+ file_info=fi(path),+ symbols=[],+ imports=imports,+ exports=[],+ docstring=None,+ parse_errors=[],+ content_hash="",+ )++ b = GraphBuilder()+ b.add_file(parsed("a.py", [imp("b"), imp("c")]))+ b.add_file(parsed("b.py", [imp("c")]))+ b.add_file(parsed("c.py", []))+ b.build()+ return b+++async def test_graph_metrics_roundtrip_postgres(pg_session: AsyncSession):+ idx = SqlIndexStore(pg_session)+ repo = await idx.upsert_repository(name="pg-metrics", local_path="/tmp/pg-metrics")++ builder = _build_sample()+ snapshot = builder.file_metrics_snapshot()++ await batch_upsert_graph_metrics(pg_session, repo.id, snapshot)+ read_back = await get_graph_metrics(pg_session, repo.id)++ assert set(read_back) == set(snapshot)+ for node, m in snapshot.items():+ assert read_back[node]["pagerank"] == pytest.approx(m["pagerank"])+ assert read_back[node]["betweenness"] == pytest.approx(m["betweenness"])+ assert read_back[node]["community_id"] == m["community_id"]+ assert read_back[node]["in_degree"] == m["in_degree"]+ assert read_back[node]["out_degree"] == m["out_degree"]++ # A builder hydrated from the SQL snapshot reproduces the NetworkX metrics.+ hydrated = _build_sample()+ hydrated.load_metrics_from_sql(read_back)+ nx_pr = builder.pagerank()+ for node, score in hydrated.pagerank().items():+ assert score == pytest.approx(nx_pr[node])++ # Upsert again → idempotent (no duplicate rows).+ await batch_upsert_graph_metrics(pg_session, repo.id, snapshot)+ assert len(await get_graph_metrics(pg_session, repo.id)) == len(snapshot)diff --git a/tests/integration/test_generation_pipeline.py b/tests/integration/test_generation_pipeline.pyindex 58f2217b1..45d77e29a 100644--- a/tests/integration/test_generation_pipeline.py+++ b/tests/integration/test_generation_pipeline.py@@ -81,6 +81,13 @@ class _SlowVectorStore:finally:self.active_embeds -= 1+ async def embed_batch(self, items: list[tuple[str, str, dict[str, Any]]]) -> None:+ # Mirror the generator's batched embed path: a level's pages are+ # embedded together after their LLM calls have all returned. The real+ # stores issue one batched embed call, so this never fans out.+ for pid, text, meta in items:+ await self.embed_and_upsert(pid, text, meta)+async def list_page_ids(self) -> set[str]:return set()diff --git a/tests/integration/test_pipeline_fast_checkpoint.py b/tests/integration/test_pipeline_fast_checkpoint.pynew file mode 100644index 000000000..711ef46d5--- /dev/null+++ b/tests/integration/test_pipeline_fast_checkpoint.py@@ -0,0 +1,87 @@+"""Integration tests for --mode fast and pipeline checkpointing.++Runs the real pipeline against the sample repo (no LLM, no external services)+to prove:+ * FAST mode skips doc generation and the co-change walk (ESSENTIAL git tier);+ * a supplied JobStore receives a COMPLETED record for each major phase;+ * an interrupted phase (left RUNNING) is excluded from the resume set.+"""++from __future__ import annotations++from datetime import UTC, datetime+from pathlib import Path++from repowise.core.persistence._interfaces.job_store import JobRecord, JobState+from repowise.core.pipeline import run_pipeline+from repowise.core.pipeline.checkpoint import find_completed_phases+from repowise.core.pipeline.modes import OrchestratorMode+++class _FakeJobStore:+ def __init__(self) -> None:+ self.jobs: dict[str, JobRecord] = {}+ self._seq = 0++ async def create_job(self, *, repository_id, phase, metadata=None) -> JobRecord:+ self._seq += 1+ jid = f"job{self._seq}"+ now = datetime.now(UTC)+ rec = JobRecord(+ jid, repository_id, phase, JobState.PENDING, None, now, now, None, metadata or {}+ )+ self.jobs[jid] = rec+ return rec++ async def update_state(self, job_id, state, *, cursor=None, error=None) -> JobRecord:+ old = self.jobs[job_id]+ rec = JobRecord(+ old.id, old.repository_id, old.phase, state, cursor or old.cursor,+ old.started_at, datetime.now(UTC), error, old.metadata,+ )+ self.jobs[job_id] = rec+ return rec++ async def list_jobs(self, *, repository_id=None, phase=None, state=None, limit=100):+ return list(self.jobs.values())+++async def test_fast_mode_skips_generation_and_co_change(sample_repo_path: Path) -> None:+ store = _FakeJobStore()+ result = await run_pipeline(+ sample_repo_path,+ generate_docs=True, # FAST must override this to False (no LLM client passed)+ mode=OrchestratorMode.FAST,+ job_store=store,+ )++ # No LLM generation happened.+ assert result.generated_pages is None+ # ESSENTIAL git tier: per-file metadata exists but no co-change partners.+ assert result.git_metadata_list, "expected git metadata for sample repo"+ for meta in result.git_metadata_list:+ assert meta.get("co_change_partners_json", "[]") == "[]"+++async def test_checkpoints_recorded_for_completed_phases(sample_repo_path: Path) -> None:+ store = _FakeJobStore()+ await run_pipeline(+ sample_repo_path,+ mode=OrchestratorMode.FAST,+ job_store=store,+ )++ completed = await find_completed_phases(store, str(Path(sample_repo_path).resolve()))+ # parse + git always run; both should be recorded COMPLETED.+ assert "parse" in completed+ assert "git" in completed+ # Every recorded job reached a terminal COMPLETED state on the success path.+ assert all(j.state == JobState.COMPLETED for j in store.jobs.values())+++async def test_interrupted_phase_excluded_from_resume_set() -> None:+ store = _FakeJobStore()+ j = await store.create_job(repository_id="r", phase="generation")+ await store.update_state(j.id, JobState.RUNNING) # simulate a crash mid-phase+ completed = await find_completed_phases(store, "r")+ assert "generation" not in completeddiff --git a/tests/unit/cli/test_fast_mode_offer.py b/tests/unit/cli/test_fast_mode_offer.pynew file mode 100644index 000000000..4927ded91--- /dev/null+++ b/tests/unit/cli/test_fast_mode_offer.py@@ -0,0 +1,48 @@+"""Tests for the large-repo fast-mode offer helpers in cli.ui."""++from __future__ import annotations++from repowise.cli.ui import (+ LARGE_REPO_FILE_THRESHOLD,+ RepoScanInfo,+ build_contextual_next_steps,+ should_offer_fast_mode,+)+++def test_should_offer_fast_mode_none():+ assert should_offer_fast_mode(None) is False+++def test_should_offer_fast_mode_small_repo():+ assert should_offer_fast_mode(RepoScanInfo(total_files=100)) is False+++def test_should_offer_fast_mode_at_threshold():+ # Threshold is strict ">" — exactly at the boundary does not offer.+ assert should_offer_fast_mode(RepoScanInfo(total_files=LARGE_REPO_FILE_THRESHOLD)) is False+++def test_should_offer_fast_mode_large_repo():+ assert should_offer_fast_mode(RepoScanInfo(total_files=LARGE_REPO_FILE_THRESHOLD + 1)) is True+++def test_next_steps_fast_mode_leads_with_full_upgrade():+ steps = build_contextual_next_steps(index_only=True, fast_mode=True)+ cmds = [c for c, _ in steps]+ descs = " ".join(d for _, d in steps).lower()+ # Fast mode tells the user how to get the full result.+ assert cmds[0] == "repowise init"+ assert "full" in descs and "docs" in descs+++def test_next_steps_index_only_without_fast():+ steps = build_contextual_next_steps(index_only=True, fast_mode=False)+ cmds = [c for c, _ in steps]+ assert any("--provider" in c for c in cmds) # the existing generate hint+++def test_next_steps_full_mode():+ steps = build_contextual_next_steps(index_only=False)+ cmds = [c for c, _ in steps]+ assert any("search" in c for c in cmds)diff --git a/tests/unit/generation/test_page_generator_tiering.py b/tests/unit/generation/test_page_generator_tiering.pynew file mode 100644index 000000000..14207fb03--- /dev/null+++ b/tests/unit/generation/test_page_generator_tiering.py@@ -0,0 +1,92 @@+"""Tests for tiered doc generation (page_generator/tiering.py + tier-2 path)."""++from __future__ import annotations++from repowise.core.generation.context_assembler import ContextAssembler+from repowise.core.generation.models import GeneratedPage, GenerationConfig+from repowise.core.generation.page_generator import PageGenerator+from repowise.core.generation.page_generator.tiering import partition_file_tiers+from repowise.core.providers.llm.mock import MockProvider++# ---------------------------------------------------------------------------+# partition_file_tiers — pure partition logic+# ---------------------------------------------------------------------------++_PR = {"a.py": 0.5, "b.py": 0.3, "c.py": 0.1, "d.py": 0.05}+_SELECTED = set(_PR)+++def test_partition_none_puts_everything_in_tier1():+ """Default (None) reproduces current selection: all pages tier-1."""+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, None)+ assert tier1 == _SELECTED+ assert tier2 == set()+++def test_partition_n_geq_len_is_all_tier1():+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 99)+ assert tier1 == _SELECTED+ assert tier2 == set()+++def test_partition_top_n_by_pagerank():+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 2)+ # Highest two PageRank scores → tier-1.+ assert tier1 == {"a.py", "b.py"}+ assert tier2 == {"c.py", "d.py"}+++def test_partition_zero_is_all_tier2():+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 0)+ assert tier1 == set()+ assert tier2 == _SELECTED+++def test_partition_is_a_partition():+ tier1, tier2 = partition_file_tiers(_SELECTED, _PR, 3)+ assert tier1 | tier2 == _SELECTED+ assert tier1 & tier2 == set()+++def test_partition_tie_break_is_deterministic():+ pr = {"x.py": 0.1, "y.py": 0.1, "z.py": 0.1}+ t1a, _ = partition_file_tiers(set(pr), pr, 1)+ t1b, _ = partition_file_tiers(set(pr), pr, 1)+ assert t1a == t1b # stable path-based tie-break+++# ---------------------------------------------------------------------------+# Tier-2 deterministic render — no LLM call+# ---------------------------------------------------------------------------+++async def test_tier2_render_makes_no_provider_call(+ sample_config, sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes+):+ provider = MockProvider()+ assembler = ContextAssembler(sample_config)+ gen = PageGenerator(provider, assembler, sample_config)++ ctx = assembler.assemble_file_page(+ sample_parsed_file,+ sample_graph,+ graph_metrics["pagerank"],+ graph_metrics["betweenness"],+ graph_metrics["community"],+ sample_source_bytes,+ )+ page = await gen._generate_file_page_tier2(sample_parsed_file, ctx)++ assert isinstance(page, GeneratedPage)+ assert page.page_type == "file_page"+ assert page.provider_name == "template"+ assert page.metadata["doc_tier"] == 2+ assert page.input_tokens == 0 and page.output_tokens == 0+ # Deterministic structure, no model invocation.+ assert provider.call_count == 0+ assert "## Overview" in page.content+ assert "## Public API" in page.content+++def test_generation_config_tier1_top_n_defaults_none():+ assert GenerationConfig().tier1_top_n is Nonediff --git a/tests/unit/ingestion/test_git_indexer_tiers.py b/tests/unit/ingestion/test_git_indexer_tiers.pynew file mode 100644index 000000000..e88fa664c--- /dev/null+++ b/tests/unit/ingestion/test_git_indexer_tiers.py@@ -0,0 +1,206 @@+"""Unit tests for the tiered git indexer (ESSENTIAL vs FULL) and backfill.++These exercise the tier gating directly on the module-level functions and the+``GitIndexer`` shims, plus the resumable backfill worker against a fake+JobStore — no real git repo required.+"""++from __future__ import annotations++import time+from unittest.mock import MagicMock++import pytest++from repowise.core.ingestion.git_indexer import (+ BACKFILL_PHASE,+ GitIndexer,+ GitIndexTier,+ backfill_full_tier,+ index_file,+)+from repowise.core.ingestion.git_indexer.records import _CommitRec++# ---------------------------------------------------------------------------+# Tier flags+# ---------------------------------------------------------------------------+++def test_full_tier_includes_expensive_signals() -> None:+ assert GitIndexTier.FULL.includes_blame is True+ assert GitIndexTier.FULL.includes_co_change is True+++def test_essential_tier_skips_expensive_signals() -> None:+ assert GitIndexTier.ESSENTIAL.includes_blame is False+ assert GitIndexTier.ESSENTIAL.includes_co_change is False+++def test_indexer_defaults_to_full_tier() -> None:+ assert GitIndexer("/tmp/repo").tier is GitIndexTier.FULL+++# ---------------------------------------------------------------------------+# index_file: include_blame gating+# ---------------------------------------------------------------------------+++def _commits() -> list[_CommitRec]:+ now = int(time.time())+ return [+ _CommitRec("a" * 40, "Alice", "[email protected]", now, False, "feat: add widget", 10, 2),+ _CommitRec("b" * 40, "Bob", "[email protected]", now - 86400, False, "fix bug", 4, 1),+ ]+++def test_index_file_essential_skips_blame(tmp_path) -> None:+ """With include_blame=False, get_blame_ownership must never be consulted —+ ownership falls back to the top commit author.+ """+ repo = MagicMock()+ # If blame were called it would raise, proving it must not be reached.+ repo.blame.side_effect = AssertionError("blame must not run in ESSENTIAL tier")+ (tmp_path / "a.py").write_text("x = 1\n")++ meta = index_file(+ repo,+ "a.py",+ repo_path=tmp_path,+ commit_limit=500,+ follow_renames=False,+ include_blame=False,+ precomputed_commits=_commits(),+ )+ assert meta["commit_count_total"] == 2+ assert meta["primary_owner_name"] == "Alice" # commit-author fallback+ repo.blame.assert_not_called()+++def test_index_file_full_consults_blame(tmp_path) -> None:+ """With include_blame=True, blame ownership overrides the commit author."""+ repo = MagicMock()+ blame_commit = MagicMock()+ blame_commit.author.name = "Carol"+ blame_commit.author.email = "[email protected]"+ repo.blame.return_value = [(blame_commit, ["line1", "line2", "line3"])]+ (tmp_path / "a.py").write_text("x = 1\n")++ meta = index_file(+ repo,+ "a.py",+ repo_path=tmp_path,+ commit_limit=500,+ follow_renames=False,+ include_blame=True,+ precomputed_commits=_commits(),+ )+ assert meta["primary_owner_name"] == "Carol" # blame wins+ repo.blame.assert_called_once()+++# ---------------------------------------------------------------------------+# index_repo: ESSENTIAL skips the co-change walk+# ---------------------------------------------------------------------------+++async def test_index_repo_essential_skips_co_change(tmp_path) -> None:+ """ESSENTIAL tier must not invoke compute_co_changes and must leave+ co_change_partners empty, while still producing per-file metadata.+ """+ import git as gitpython++ repo = gitpython.Repo.init(tmp_path)+ f = tmp_path / "a.py"+ f.write_text("x = 1\n")+ repo.index.add(["a.py"])+ repo.index.commit("feat: add a")+ g = tmp_path / "b.py"+ g.write_text("y = 2\n")+ repo.index.add(["b.py"])+ repo.index.commit("feat: add b")++ co_change_seen = {"called": False}+ import repowise.core.ingestion.git_indexer.indexer as indexer_mod++ orig = indexer_mod.compute_co_changes++ def _spy(*args, **kwargs): # pragma: no cover - should not run+ co_change_seen["called"] = True+ return orig(*args, **kwargs)++ indexer_mod.compute_co_changes = _spy+ try:+ idx = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)+ summary, results = await idx.index_repo("repo1")+ finally:+ indexer_mod.compute_co_changes = orig++ assert co_change_seen["called"] is False+ assert summary.files_indexed >= 1+ for meta in results:+ assert meta.get("co_change_partners_json", "[]") == "[]"+++# ---------------------------------------------------------------------------+# Backfill worker (resumable via JobStore)+# ---------------------------------------------------------------------------+++class _FakeJobStore:+ """Minimal in-memory JobStore capturing the lifecycle for assertions."""++ def __init__(self) -> None:+ self.created: list[dict] = []+ self.states: list[tuple[str, str]] = []+ self._seq = 0++ async def create_job(self, *, repository_id, phase, metadata=None):+ from repowise.core.persistence._interfaces.job_store import JobRecord, JobState++ self._seq += 1+ jid = f"job{self._seq}"+ self.created.append({"id": jid, "phase": phase, "metadata": metadata})+ from datetime import UTC, datetime++ now = datetime.now(UTC)+ return JobRecord(jid, repository_id, phase, JobState.PENDING, None, now, now, None, metadata or {})++ async def update_state(self, job_id, state, *, cursor=None, error=None):+ self.states.append((job_id, state.value))+ return None+++async def test_backfill_runs_full_and_records_job(tmp_path) -> None:+ import git as gitpython++ repo = gitpython.Repo.init(tmp_path)+ (tmp_path / "a.py").write_text("x = 1\n")+ repo.index.add(["a.py"])+ repo.index.commit("feat: add a")++ idx = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)+ store = _FakeJobStore()++ summary, _results = await backfill_full_tier(idx, "repo1", job_store=store)++ # The job was recorded under the backfill phase and reached COMPLETED.+ assert store.created and store.created[0]["phase"] == BACKFILL_PHASE+ assert ("job1", "running") in store.states+ assert ("job1", "completed") in store.states+ # The indexer's tier is restored after the backfill.+ assert idx.tier is GitIndexTier.ESSENTIAL+ assert summary.files_indexed >= 1+++async def test_backfill_marks_failed_on_error(tmp_path, monkeypatch) -> None:+ idx = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)+ store = _FakeJobStore()++ async def _boom(_repo_id):+ raise RuntimeError("kaboom")++ monkeypatch.setattr(idx, "index_repo", _boom)+ with pytest.raises(RuntimeError):+ await backfill_full_tier(idx, "repo1", job_store=store)+ assert ("job1", "failed") in store.states+ assert idx.tier is GitIndexTier.ESSENTIAL # restored even on failurediff --git a/tests/unit/ingestion/test_graph_metrics_sql.py b/tests/unit/ingestion/test_graph_metrics_sql.pynew file mode 100644index 000000000..4b6fcc6cd--- /dev/null+++ b/tests/unit/ingestion/test_graph_metrics_sql.py@@ -0,0 +1,122 @@+"""SQL-backed graph metrics: snapshot correctness + read routing.++Proves that the materialized ``graph_metrics`` snapshot reproduces the+NetworkX-computed file-level metrics, and that ``load_metrics_from_sql``+routes subsequent reads through the snapshot (so the in-memory NetworkX+object can be dropped on large repos).+"""++from __future__ import annotations++from datetime import datetime++from repowise.core.ingestion.graph import GraphBuilder+from repowise.core.ingestion.models import FileInfo, Import, ParsedFile+from repowise.core.pipeline.modes import OrchestratorMode+++def _fi(path: str) -> FileInfo:+ return FileInfo(+ path=path,+ abs_path=f"/repo/{path}",+ language="python",+ size_bytes=100,+ git_hash="",+ last_modified=datetime.now(),+ is_test=False,+ is_config=False,+ is_api_contract=False,+ is_entry_point=False,+ )+++def _imp(module_path: str) -> Import:+ return Import(+ raw_statement=f"import {module_path}",+ module_path=module_path,+ imported_names=[],+ is_relative=False,+ resolved_file=None,+ )+++def _parsed(path: str, imports: list[Import] | None = None) -> ParsedFile:+ return ParsedFile(+ file_info=_fi(path),+ symbols=[],+ imports=imports or [],+ exports=[],+ docstring=None,+ parse_errors=[],+ content_hash="",+ )+++def _build_sample() -> GraphBuilder:+ """a imports b and c; b imports c — a small connected file graph."""+ b = GraphBuilder()+ b.add_file(_parsed("a.py", [_imp("b"), _imp("c")]))+ b.add_file(_parsed("b.py", [_imp("c")]))+ b.add_file(_parsed("c.py"))+ b.build()+ return b+++def test_snapshot_matches_networkx():+ b = _build_sample()+ nx_pr = b.pagerank()+ nx_bc = b.betweenness_centrality()+ nx_cd = b.community_detection()+ nx_in = b.in_degree()+ nx_out = b.out_degree()++ snap = b.file_metrics_snapshot()++ # Every file-subgraph node is represented and matches the live NetworkX+ # values byte-for-byte.+ assert set(snap) == set(nx_pr)+ for node, m in snap.items():+ assert m["pagerank"] == nx_pr[node]+ assert m["betweenness"] == nx_bc[node]+ assert m["community_id"] == nx_cd.get(node, 0)+ assert m["in_degree"] == nx_in[node]+ assert m["out_degree"] == nx_out[node]+++def test_load_from_sql_routes_reads_without_recompute():+ source = _build_sample()+ snap = source.file_metrics_snapshot()+ nx_pr = source.pagerank()++ # A fresh builder hydrated purely from the snapshot.+ hydrated = _build_sample()+ hydrated.load_metrics_from_sql(snap)++ assert hydrated.pagerank() == {n: snap[n]["pagerank"] for n in snap}+ assert hydrated.pagerank() == nx_pr # equals the original NetworkX result+ assert hydrated.in_degree() == {n: snap[n]["in_degree"] for n in snap}++ # Dropping the structural graph must not break metric reads — they are+ # served from the loaded snapshot, proving the read path is SQL-routed.+ hydrated.release_graph()+ assert hydrated.graph().number_of_nodes() == 0+ assert hydrated.pagerank() == nx_pr+ assert hydrated.betweenness_centrality() == source.betweenness_centrality()+ assert hydrated.community_detection() == {n: snap[n]["community_id"] for n in snap}+++def test_in_out_degree_reflect_import_edges():+ b = _build_sample()+ ind = b.in_degree()+ outd = b.out_degree()+ # c.py is imported by a and b → in-degree 2, no out-edges.+ assert ind["c.py"] == 2+ assert outd["c.py"] == 0+ # a.py imports two files, imported by none.+ assert outd["a.py"] == 2+ assert ind["a.py"] == 0+++def test_mode_sql_backed_metrics_flag():+ assert OrchestratorMode.FAST.sql_backed_metrics is True+ assert OrchestratorMode.STANDARD.sql_backed_metrics is Falsediff --git a/tests/unit/persistence/test_models.py b/tests/unit/persistence/test_models.pyindex 2829110bb..bf1d37282 100644--- a/tests/unit/persistence/test_models.py+++ b/tests/unit/persistence/test_models.py@@ -313,5 +313,6 @@ def test_base_includes_all_models():"health_snapshots","coverage_files","pipeline_jobs",+ "graph_metrics",}assert expected == table_namesdiff --git a/tests/unit/persistence/test_vector_store.py b/tests/unit/persistence/test_vector_store.pyindex 17f121fd9..17a8620f2 100644--- a/tests/unit/persistence/test_vector_store.py+++ b/tests/unit/persistence/test_vector_store.py@@ -224,6 +224,95 @@ async def test_in_memory_store_close_clears(in_memory_vector_store):assert len(in_memory_vector_store) == 0+# ---------------------------------------------------------------------------+# embed_batch+# ---------------------------------------------------------------------------+++async def test_in_memory_embed_batch_empty_is_noop(in_memory_vector_store):+ await in_memory_vector_store.embed_batch([])+ assert len(in_memory_vector_store) == 0+++async def test_in_memory_embed_batch_matches_single_path(mock_embedder):+ """embed_batch must produce the same stored vectors as N embed_and_upsert calls."""+ from repowise.core.persistence.vector_store import InMemoryVectorStore++ items = [+ (+ f"p{i}",+ f"content number {i}",+ {+ "title": f"Page {i}",+ "page_type": "file_page",+ "target_path": f"f{i}.py",+ "content": f"content number {i}",+ },+ )+ for i in range(5)+ ]++ batched = InMemoryVectorStore(mock_embedder)+ await batched.embed_batch(items)++ single = InMemoryVectorStore(mock_embedder)+ for page_id, text, meta in items:+ await single.embed_and_upsert(page_id, text, meta)++ assert len(batched) == len(single) == 5+ # Identical query results prove vectors + metadata landed identically.+ bq = await batched.search("content number 2", limit=5)+ sq = await single.search("content number 2", limit=5)+ assert [(r.page_id, r.title) for r in bq] == [(r.page_id, r.title) for r in sq]+++async def test_in_memory_embed_batch_searchable(in_memory_vector_store):+ await in_memory_vector_store.embed_batch(+ [+ (+ "p1",+ "Python decorator pattern",+ {"title": "Dec", "page_type": "file_page", "target_path": "a.py", "content": "Python decorator pattern"},+ ),+ (+ "p2",+ "Rust ownership model",+ {"title": "Rust", "page_type": "file_page", "target_path": "b.py", "content": "Rust ownership model"},+ ),+ ]+ )+ assert len(in_memory_vector_store) == 2+ results = await in_memory_vector_store.search("Python decorator pattern", limit=2)+ assert results[0].page_id == "p1"+++async def test_base_embed_batch_default_uses_single_path(mock_embedder):+ """A backend that does NOT override embed_batch falls back to the ABC's+ sequential default, which must still upsert every item correctly.+ """+ from repowise.core.persistence.vector_store import VectorStore++ class _CountingStore(VectorStore):+ def __init__(self) -> None:+ self.calls: list[str] = []++ async def embed_and_upsert(self, page_id, text, metadata):+ self.calls.append(page_id)++ async def search(self, query, limit=10):+ return []++ async def delete(self, page_id):+ return None++ async def close(self):+ return None++ store = _CountingStore()+ await store.embed_batch([("a", "x", {}), ("b", "y", {})])+ assert store.calls == ["a", "b"]++# ---------------------------------------------------------------------------# LanceDB (optional — skipped if not installed)# ---------------------------------------------------------------------------@@ -246,3 +335,22 @@ async def test_lancedb_vector_store_basic(tmp_path, mock_embedder):assert results[0].page_id == "p1"finally:await store.close()+++async def test_lancedb_embed_batch(tmp_path, mock_embedder):+ lancedb = pytest.importorskip("lancedb") # noqa: F841+ from repowise.core.persistence.vector_store import LanceDBVectorStore++ store = LanceDBVectorStore(str(tmp_path / "lance"), mock_embedder)+ try:+ await store.embed_batch(+ [+ ("p1", "Python async generators", {"target_path": "a.py"}),+ ("p2", "Rust ownership model", {"target_path": "b.py"}),+ ]+ )+ ids = await store.list_page_ids()+ assert ids == {"p1", "p2"}+ finally:+ await store.close()diff --git a/tests/unit/pipeline/test_checkpoint.py b/tests/unit/pipeline/test_checkpoint.pynew file mode 100644index 000000000..0d7d88f78--- /dev/null+++ b/tests/unit/pipeline/test_checkpoint.py@@ -0,0 +1,129 @@+"""Unit tests for pipeline checkpoint/resume.++Exercise the real ``pipeline_hooks`` → queue → ``JobStore`` seam with a fresh+registry (no global-state leakage) and an in-memory fake JobStore.+"""++from __future__ import annotations++from datetime import UTC, datetime++import pytest++from repowise.core.persistence._interfaces.job_store import JobRecord, JobState+from repowise.core.pipeline.checkpoint import (+ DEFAULT_CHECKPOINT_PHASES,+ PhaseCheckpointer,+ find_completed_phases,+)+from repowise.core.registry import HookPhase, HookProgressCallback, PipelineHookRegistry+++class FakeJobStore:+ """In-memory JobStore tracking create/update calls for assertions."""++ def __init__(self) -> None:+ self.jobs: dict[str, JobRecord] = {}+ self.events: list[tuple[str, str]] = [] # (op, phase-or-state)+ self._seq = 0++ async def create_job(self, *, repository_id, phase, metadata=None) -> JobRecord:+ self._seq += 1+ jid = f"job{self._seq}"+ now = datetime.now(UTC)+ rec = JobRecord(+ jid, repository_id, phase, JobState.PENDING, None, now, now, None, metadata or {}+ )+ self.jobs[jid] = rec+ self.events.append(("create", phase))+ return rec++ async def update_state(self, job_id, state, *, cursor=None, error=None) -> JobRecord:+ old = self.jobs[job_id]+ rec = JobRecord(+ old.id, old.repository_id, old.phase, state, cursor or old.cursor,+ old.started_at, datetime.now(UTC), error, old.metadata,+ )+ self.jobs[job_id] = rec+ self.events.append(("state", f"{old.phase}:{state.value}"))+ return rec++ async def list_jobs(self, *, repository_id=None, phase=None, state=None, limit=100):+ return [+ j+ for j in self.jobs.values()+ if (repository_id is None or j.repository_id == repository_id)+ ]+++async def _fire_phase(cb: HookProgressCallback, phase: str) -> None:+ cb.on_phase_start(phase, None)+ cb.on_phase_done(phase)+++async def test_checkpointer_records_phase_lifecycle() -> None:+ registry = PipelineHookRegistry()+ store = FakeJobStore()+ cp = PhaseCheckpointer(store, "repo-1", phases=("parse", "git"), registry=registry)+ cb = HookProgressCallback(None, registry)++ async with cp:+ await _fire_phase(cb, "parse")+ await _fire_phase(cb, "git")++ # Two jobs created (parse, git), each RUNNING then COMPLETED, in order.+ assert store.events == [+ ("create", "parse"),+ ("state", "parse:running"),+ ("state", "parse:completed"),+ ("create", "git"),+ ("state", "git:running"),+ ("state", "git:completed"),+ ]+++async def test_checkpointer_ignores_untracked_phases() -> None:+ registry = PipelineHookRegistry()+ store = FakeJobStore()+ cp = PhaseCheckpointer(store, "repo-1", phases=("parse",), registry=registry)+ cb = HookProgressCallback(None, registry)++ async with cp:+ await _fire_phase(cb, "external_systems") # not tracked+ await _fire_phase(cb, "parse")++ assert [e for e in store.events if e[0] == "create"] == [("create", "parse")]+++async def test_checkpointer_unregisters_hooks_on_close() -> None:+ registry = PipelineHookRegistry()+ store = FakeJobStore()+ cp = PhaseCheckpointer(store, "repo-1", phases=("parse",), registry=registry)++ async with cp:+ assert registry.hooks_for("parse", HookPhase.PRE)+ # After close, the checkpointer's hooks are gone — no leakage into the+ # shared registry for the next run.+ assert not registry.hooks_for("parse", HookPhase.PRE)+ assert not registry.hooks_for("parse", HookPhase.POST)+++async def test_find_completed_phases() -> None:+ store = FakeJobStore()+ j1 = await store.create_job(repository_id="r", phase="parse")+ await store.update_state(j1.id, JobState.COMPLETED)+ j2 = await store.create_job(repository_id="r", phase="git")+ await store.update_state(j2.id, JobState.RUNNING)++ done = await find_completed_phases(store, "r")+ assert done == {"parse"}+++def test_default_phases_are_sane() -> None:+ assert "parse" in DEFAULT_CHECKPOINT_PHASES+ assert "generation" in DEFAULT_CHECKPOINT_PHASES++[email protected]("phase", DEFAULT_CHECKPOINT_PHASES)+def test_default_phase_names_are_strings(phase: str) -> None:+ assert isinstance(phase, str) and phasediff --git a/tests/unit/pipeline/test_modes.py b/tests/unit/pipeline/test_modes.pynew file mode 100644index 000000000..b08aa32fe--- /dev/null+++ b/tests/unit/pipeline/test_modes.py@@ -0,0 +1,22 @@+"""Unit tests for OrchestratorMode policy."""++from __future__ import annotations++from repowise.core.ingestion.git_indexer import GitIndexTier+from repowise.core.pipeline.modes import OrchestratorMode+++def test_standard_mode_full_git_and_docs() -> None:+ assert OrchestratorMode.STANDARD.git_tier is GitIndexTier.FULL+ assert OrchestratorMode.STANDARD.allows_doc_generation is True+++def test_fast_mode_essential_git_no_docs() -> None:+ assert OrchestratorMode.FAST.git_tier is GitIndexTier.ESSENTIAL+ assert OrchestratorMode.FAST.allows_doc_generation is False+++def test_mode_is_str_enum() -> None:+ # StrEnum members compare equal to their string value — handy for CLI glue.+ assert OrchestratorMode.FAST == "fast"+ assert OrchestratorMode("standard") is OrchestratorMode.STANDARD