Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

Doc generation upgrade: enforced budget, faster runs, wiki interlinking (#208)

Raghav Chamadiya60f8a7ddbe03
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd.py b/packages/cli/src/repowise/cli/commands/init_cmd.py
index 66e662c94..dc166c4f2 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd.py
@@ -257,6 +257,8 @@ def _run_workspace_generation(
skip_infra: bool,
test_run: bool,
reasoning: str = "auto",
+ onboarding: bool = True,
+ coverage_pct: float | None = None,
) -> list[Any]:
"""Run LLM generation for a single repo in the workspace init flow.
@@ -317,19 +319,79 @@ def _run_workspace_generation(
except ImportError:
vector_store = InMemoryVectorStore(embedder_impl)
- # Cost estimate
+ # Coverage chooser — interactive when TTY, falls back to the
+ # ``coverage`` flag (or the default of 20%) for non-TTY / CI runs.
+ # Computes per-option counts + costs from the live ingestion data
+ # so the table never lies about what generation will produce.
+ from repowise.cli.cost_estimator import compute_coverage_options
+ from repowise.cli.coverage_select import interactive_coverage_select
+
gen_config = GenerationConfig(
max_concurrency=concurrency,
reasoning=resolve_reasoning(reasoning),
+ enable_onboarding=onboarding,
+ )
+ use_interactive_coverage = (
+ sys.stdin.isatty() and coverage_pct is None and not yes
)
- plans = build_generation_plan(
- result.parsed_files, result.graph_builder, gen_config, skip_tests, skip_infra
+ if use_interactive_coverage:
+ options = compute_coverage_options(
+ parsed_files=result.parsed_files,
+ graph_builder=result.graph_builder,
+ base_config=gen_config,
+ provider_name=provider.provider_name,
+ model_name=provider.model_name,
+ repo_path=repo_path,
+ skip_tests=skip_tests,
+ skip_infra=skip_infra,
+ )
+ chosen = interactive_coverage_select(console, options)
+ chosen_pct = chosen.pct
+ plans = chosen.plans
+ est = chosen.estimate
+ else:
+ chosen_pct = coverage_pct if coverage_pct is not None else gen_config.coverage_pct
+ from dataclasses import replace as _replace
+
+ gen_config_for_plan = _replace(
+ gen_config, coverage_pct=chosen_pct, max_pages_pct=chosen_pct
+ )
+ plans = build_generation_plan(
+ result.parsed_files,
+ result.graph_builder,
+ gen_config_for_plan,
+ skip_tests,
+ skip_infra,
+ )
+ est = estimate_cost(
+ plans,
+ provider.provider_name,
+ provider.model_name,
+ repo_path=repo_path,
+ )
+
+ # Bake the chosen coverage into the gen_config that runs generation,
+ # so the page generator's selection layer honors the user's pick.
+ from dataclasses import replace as _replace_cfg
+
+ gen_config = _replace_cfg(
+ gen_config, coverage_pct=chosen_pct, max_pages_pct=chosen_pct
)
- est = estimate_cost(plans, provider.provider_name, provider.model_name)
+
+ if est.cost_range is not None:
+ cost_str = (
+ f"${est.cost_range.low:.2f} - ${est.cost_range.high:.2f} USD "
+ f"(median ${est.estimated_cost_usd:.2f})"
+ )
+ if est.is_calibrated:
+ cost_str += " [calibrated]"
+ else:
+ cost_str = f"${est.estimated_cost_usd:.2f} USD"
console.print(
- f" Estimated tokens: ~{est.estimated_input_tokens + est.estimated_output_tokens:,} "
- f"(${est.estimated_cost_usd:.2f} USD, {est.total_pages} pages)"
+ f" Coverage: {int(chosen_pct * 100)}% / "
+ f"~{est.estimated_input_tokens + est.estimated_output_tokens:,} tokens "
+ f"({cost_str}, {est.total_pages} pages)"
)
if (
@@ -430,6 +492,8 @@ def _workspace_init(
dry_run: bool = False,
resume: bool = False,
force: bool = False,
+ onboarding: bool = True,
+ coverage_pct: float | None = None,
) -> None:
"""Multi-repo workspace initialization.
@@ -539,6 +603,11 @@ def _workspace_init(
if not index_only:
try:
provider = resolve_provider(provider_name, model, primary_repo.path)
+ # Re-resolve the embedder now that interactive_provider_select
+ # may have set the provider's API key in os.environ. Without
+ # this, full-mode runs would display "mock" forever because
+ # the initial resolution happened before the key was available.
+ embedder_name_resolved = _resolve_embedder(embedder_name)
console.print(
f" Provider: [cyan]{provider.provider_name}[/cyan] / "
f"Model: [cyan]{provider.model_name}[/cyan]"
@@ -656,6 +725,8 @@ def _workspace_init(
skip_infra=skip_infra,
test_run=test_run,
reasoning=resolved_reasoning,
+ onboarding=onboarding,
+ coverage_pct=coverage_pct,
)
result.generated_pages = generated_pages
total_pages += len(generated_pages)
@@ -916,6 +987,29 @@ def _workspace_init(
default=False,
help="In multi-repo mode, index all detected repos without prompting.",
)
+ "--onboarding/--no-onboarding",
+ "onboarding",
+ default=True,
+ help=(
+ "Generate the curated Onboarding collection (Project Overview, "
+ "Architecture Guide, Getting Started, Codebase Map, Key Concepts, "
+ "How It Works, Development Guide, Active Landscape). Default: on. "
+ "Slots with insufficient signal are skipped automatically."
+ ),
+)
+ "--coverage",
+ "coverage_pct",
+ type=float,
+ default=None,
+ metavar="PCT",
+ help=(
+ "Documentation coverage as a fraction of repo files (e.g. 0.10, 0.20, "
+ "0.50). Bypasses the interactive coverage chooser. Default when "
+ "interactive: prompt; otherwise 0.20."
+ ),
+)
def init_command(
path: str | None,
provider_name: str | None,
@@ -937,6 +1031,8 @@ def init_command(
no_claude_md: bool,
include_submodules: bool,
init_all: bool,
+ onboarding: bool,
+ coverage_pct: float | None,
) -> None:
"""Generate wiki documentation for a codebase.
@@ -995,6 +1091,8 @@ def init_command(
dry_run=dry_run,
resume=resume,
force=force,
+ onboarding=onboarding,
+ coverage_pct=coverage_pct,
)
return
@@ -1126,6 +1224,11 @@ def init_command(
provider_name, model = _ips(console, model)
provider = resolve_provider(provider_name, model, repo_path)
+ # resolve_provider / interactive_provider_select may have just set
+ # the API key in os.environ. Re-resolve the embedder so the
+ # display (and the embed path below) honors the key the user just
+ # pasted, rather than the pre-prompt "mock" fallback.
+ embedder_name_resolved = _resolve_embedder(embedder_name)
if not is_interactive:
console.print(f"[bold]repowise init[/bold] — {repo_path}")
console.print(
@@ -1266,17 +1369,60 @@ def init_command(
f"Generating wiki pages with {provider.provider_name} / {provider.model_name}",
)
- # Cost estimation
+ # Cost estimation + coverage selection. The coverage chooser
+ # is rendered interactively when stdin is a TTY and no explicit
+ # ``--coverage`` flag was passed; otherwise the configured
+ # percentage drives a single non-interactive estimate.
+ from dataclasses import replace as _replace_cfg
+ from repowise.cli.cost_estimator import compute_coverage_options
+ from repowise.cli.coverage_select import interactive_coverage_select
from repowise.core.generation import GenerationConfig
+
gen_config = GenerationConfig(
max_concurrency=concurrency,
language=language,
reasoning=resolved_reasoning,
+ enable_onboarding=onboarding,
)
- plans = build_generation_plan(
- result.parsed_files, result.graph_builder, gen_config, skip_tests, skip_infra
+ if sys.stdin.isatty() and coverage_pct is None and not yes:
+ options = compute_coverage_options(
+ parsed_files=result.parsed_files,
+ graph_builder=result.graph_builder,
+ base_config=gen_config,
+ provider_name=provider.provider_name,
+ model_name=provider.model_name,
+ repo_path=repo_path,
+ skip_tests=skip_tests,
+ skip_infra=skip_infra,
+ )
+ chosen = interactive_coverage_select(console, options)
+ chosen_pct = chosen.pct
+ plans = chosen.plans
+ est = chosen.estimate
+ else:
+ chosen_pct = (
+ coverage_pct if coverage_pct is not None else gen_config.coverage_pct
+ )
+ gen_config_for_plan = _replace_cfg(
+ gen_config, coverage_pct=chosen_pct, max_pages_pct=chosen_pct
+ )
+ plans = build_generation_plan(
+ result.parsed_files,
+ result.graph_builder,
+ gen_config_for_plan,
+ skip_tests,
+ skip_infra,
+ )
+ est = estimate_cost(
+ plans,
+ provider.provider_name,
+ provider.model_name,
+ repo_path=repo_path,
+ )
+
+ gen_config = _replace_cfg(
+ gen_config, coverage_pct=chosen_pct, max_pages_pct=chosen_pct
)
- est = estimate_cost(plans, provider.provider_name, provider.model_name)
table = Table(title="Generation Plan", border_style=BRAND)
table.add_column("Page Type", style="cyan")
@@ -1295,10 +1441,31 @@ def init_command(
lang_parts = [f"{lang} {pct:.0%}" for lang, pct in lang_items]
console.print(f" Languages: {', '.join(lang_parts)}")
+ if est.cost_range is not None:
+ cost_str = (
+ f"${est.cost_range.low:.2f} - ${est.cost_range.high:.2f} USD "
+ f"(median ${est.estimated_cost_usd:.2f})"
+ )
+ if est.is_calibrated:
+ cost_str += " [calibrated]"
+ else:
+ cost_str = f"${est.estimated_cost_usd:.2f} USD"
+
console.print(
- f" Estimated tokens: ~{est.estimated_input_tokens + est.estimated_output_tokens:,} "
- f"(${est.estimated_cost_usd:.2f} USD)"
+ f" Coverage: {int(chosen_pct * 100)}% / "
+ f"~{est.estimated_input_tokens + est.estimated_output_tokens:,} tokens "
+ f"({cost_str})"
)
+ if onboarding:
+ console.print(
+ " [cyan]Onboarding collection:[/cyan] "
+ "[dim]up to 8 curated pages — Project Overview, Architecture Guide, "
+ "Getting Started, Codebase Map, Key Concepts, How It Works, "
+ "Development Guide, Active Landscape "
+ "(slots without enough signal are skipped).[/dim]"
+ )
+ else:
+ console.print(" [dim]Onboarding collection: disabled (--no-onboarding).[/dim]")
console.print()
if dry_run:
@@ -1449,6 +1616,23 @@ def init_command(
run_async(_persist_result(result, repo_path))
console.print(" [green]✓[/green] Database updated")
+ # Persist the onboarding choice so subsequent `repowise update` runs
+ # honor it without re-passing the flag. Default True is omitted to keep
+ # config files tidy — only the override is recorded.
+ if not onboarding:
+ cfg = load_config(repo_path)
+ cfg["enable_onboarding"] = False
+ try:
+ import yaml # type: ignore[import-untyped]
+
+ cfg_path = repo_path / ".repowise" / "config.yaml"
+ cfg_path.write_text(
+ yaml.dump(cfg, default_flow_style=False, sort_keys=False),
+ encoding="utf-8",
+ )
+ except ImportError:
+ pass
+
# ---- Post-run: config, state, MCP, editor project files ----
if commit_limit is not None:
cfg = load_config(repo_path)
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd.py b/packages/cli/src/repowise/cli/commands/update_cmd.py
index c9df4c316..f1f95eceb 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd.py
@@ -360,10 +360,15 @@ def update_command(
cfg = load_config(repo_path)
language = cfg.get("language", "en")
+ # Config-driven (saved by `repowise init`); CLI override not surfaced
+ # on update yet — defaults to on to keep the onboarding collection
+ # fresh as the codebase evolves.
+ enable_onboarding_cfg = bool(cfg.get("enable_onboarding", True))
config = GenerationConfig(
max_concurrency=concurrency,
language=language,
reasoning=resolve_reasoning(reasoning, cfg),
+ enable_onboarding=enable_onboarding_cfg,
)
# Read exclude patterns from config (set during init or via web UI)
@@ -593,9 +598,37 @@ def update_command(
affected_parsed = [pf for pf in parsed_files if pf.file_info.path in regen_set]
affected_source = {p: s for p, s in source_map.items() if p in regen_set}
+ # Load prior pages so the generator can skip the LLM call for any affected
+ # file whose freshly rendered prompt still hashes to the persisted value.
+ async def _load_prior() -> dict[str, object]:
+ from repowise.cli.helpers import get_db_url_for_repo
+ from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ load_prior_pages,
+ upsert_repository,
+ )
+
+ url = get_db_url_for_repo(repo_path)
+ engine = create_engine(url)
+ sf = create_session_factory(engine)
+ async with get_session(sf) as session:
+ repo = await upsert_repository(
+ session, name=repo_path.name, local_path=str(repo_path)
+ )
+ return await load_prior_pages(session, repo.id)
+
+ try:
+ prior_pages = run_async(_load_prior())
+ except Exception:
+ prior_pages = {}
+
# Generate affected pages
assembler = ContextAssembler(config)
- generator = PageGenerator(provider, assembler, config, language=config.language)
+ generator = PageGenerator(
+ provider, assembler, config, language=config.language, prior_pages=prior_pages
+ )
repo_name = repo_path.name
generated_pages = run_async(
diff --git a/packages/cli/src/repowise/cli/cost_estimator.py b/packages/cli/src/repowise/cli/cost_estimator.py
deleted file mode 100644
index 38a7f38e3..000000000
--- a/packages/cli/src/repowise/cli/cost_estimator.py
+++ /dev/null
@@ -1,279 +0,0 @@
-"""Pre-generation cost estimation — mirrors page_generator.generate_all() selection logic."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
-
-_INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()
-_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})
-_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()
-
-
-def _is_infra_file(parsed: Any) -> bool:
- lang = parsed.file_info.language
- if lang in _INFRA_LANGUAGES:
- return True
- name = Path(parsed.file_info.path).name
- return name in _INFRA_FILENAMES
-
-
-def _is_significant_file(
- parsed: Any,
- pagerank: dict[str, float],
- betweenness: dict[str, float],
- config: Any,
- pr_threshold: float,
-) -> bool:
- if len(parsed.symbols) < config.file_page_min_symbols:
- return False
- path = parsed.file_info.path
- return (
- parsed.file_info.is_entry_point
- or pagerank.get(path, 0.0) >= pr_threshold
- or betweenness.get(path, 0.0) > 0.0
- )
-
-
-# ---------------------------------------------------------------------------
-# Plan and estimate data types
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class PageTypePlan:
- """Count of pages to generate for a given page type."""
-
- page_type: str
- count: int
- level: int
-
-
-@dataclass
-class CostEstimate:
- """Estimated cost for a generation run."""
-
- plans: list[PageTypePlan] = field(default_factory=list)
- total_pages: int = 0
- estimated_input_tokens: int = 0
- estimated_output_tokens: int = 0
- estimated_cost_usd: float = 0.0
- provider_name: str = ""
- model_name: str = ""
-
-
-# Token heuristics per page type (input, output)
-_TOKEN_HEURISTICS: dict[str, tuple[int, int]] = {
- "api_contract": (3000, 2000),
- "symbol_spotlight": (2000, 1500),
- "file_page": (4000, 2500),
- "scc_page": (3000, 2000),
- "module_page": (4000, 2500),
- "cross_package": (3000, 2000),
- "repo_overview": (5000, 3000),
- "architecture_diagram": (4000, 2500),
- "infra_page": (2000, 1500),
-}
-
-# Cost per 1K tokens (input, output).
-# Exact model names are checked first; prefix fallbacks catch unknown variants.
-_COST_TABLE_EXACT: dict[str, tuple[float, float]] = {
- # OpenAI GPT-5.4 family (prices per MTok → divide by 1000 for per-1K)
- "gpt-5.4": (0.0025, 0.015), # $2.50/$15 per MTok
- "gpt-5.4-mini": (0.00075, 0.0045), # $0.75/$4.50 per MTok
- "gpt-5.4-nano": (0.0002, 0.00125), # $0.20/$1.25 per MTok
- # Gemini family
- "gemini-3.1-pro-preview": (0.002, 0.012), # $2/$12 per MTok
- "gemini-3-flash-preview": (0.0005, 0.003), # $0.50/$3 per MTok
- "gemini-3.1-flash-lite-preview": (0.00025, 0.0015), # $0.25/$1.50 per MTok
- # Anthropic Claude 4.x family
- "claude-opus-4-6": (0.005, 0.025), # $5/$25 per MTok
- "claude-sonnet-4-6": (0.003, 0.015), # $3/$15 per MTok
- "claude-haiku-4-5": (0.001, 0.005), # $1/$5 per MTok
-}
-
-# Prefix fallbacks for unknown model variants
-_COST_TABLE_PREFIX: dict[str, tuple[float, float]] = {
- "gpt-5.4-nano": (0.0002, 0.00125),
- "gpt-5.4-mini": (0.00075, 0.0045),
- "gpt-5.4": (0.0025, 0.015),
- "claude-opus": (0.005, 0.025),
- "claude-sonnet": (0.003, 0.015),
- "claude-haiku": (0.001, 0.005),
- "claude": (0.003, 0.015),
- "gemini": (0.00025, 0.0015),
- "llama": (0.0, 0.0),
- "mock": (0.0, 0.0),
-}
-
-
-def _lookup_cost(model_name: str) -> tuple[float, float]:
- """Return (input_rate, output_rate) per 1K tokens for a model."""
- lower = model_name.lower()
- # Exact match first
- if lower in _COST_TABLE_EXACT:
- return _COST_TABLE_EXACT[lower]
- # Longest matching prefix wins
- best_prefix = ""
- best_rates = (0.0, 0.0)
- for prefix, rates in _COST_TABLE_PREFIX.items():
- if lower.startswith(prefix) and len(prefix) > len(best_prefix):
- best_prefix = prefix
- best_rates = rates
- return best_rates
-
-
-def build_generation_plan(
- parsed_files: list[Any],
- graph_builder: Any,
- config: Any,
- skip_tests: bool = False,
- skip_infra: bool = False,
-) -> list[PageTypePlan]:
- """Replicate the page selection logic from ``generate_all()`` without calling the LLM.
-
- Returns a list of :class:`PageTypePlan` entries — one per page type.
- """
- graph = graph_builder.graph()
- pagerank = graph_builder.pagerank()
- betweenness = graph_builder.betweenness_centrality()
- sccs = graph_builder.strongly_connected_components()
-
- plans: list[PageTypePlan] = []
-
- # Optionally filter
- files = parsed_files
- if skip_tests:
- files = [p for p in files if not p.file_info.is_test]
-
- code_files = [
- p
- for p in files
- if not p.file_info.is_api_contract
- and not _is_infra_file(p)
- and p.file_info.language in _CODE_LANGUAGES
- ]
-
- # Budget calculation — mirrors page_generator.generate_all() lines 544-564
- budget = max(50, int(len(files) * config.max_pages_pct))
-
- # Fixed overhead pages (always generated)
- api_count = sum(1 for p in files if p.file_info.is_api_contract)
- scc_count = sum(1 for scc in sccs if len(scc) > 1)
- modules: set[str] = set()
- for p in code_files:
- parts = Path(p.file_info.path).parts
- modules.add(parts[0] if len(parts) > 1 else "root")
- module_count = len(modules)
- fixed_overhead = api_count + scc_count + module_count + 2 # +2 = repo_overview + arch_diagram
-
- remaining = max(0, budget - fixed_overhead)
-
- # File page budget (priority over symbol_spotlight)
- code_pr_scores = sorted(
- [pagerank.get(p.file_info.path, 0.0) for p in code_files],
- reverse=True,
- )
- n_file_uncapped = (
- max(1, int(len(code_pr_scores) * config.file_page_top_percentile)) if code_pr_scores else 0
- )
- n_file_cap = min(n_file_uncapped, remaining)
- pr_threshold = code_pr_scores[n_file_cap - 1] if code_pr_scores and n_file_cap > 0 else 0.0
-
- # Actual file_page count: ALL files passing _is_significant_file
- # (betweenness > 0 and entry_point are independent of the PageRank threshold)
- file_page_count = sum(
- 1
- for p in code_files
- if _is_significant_file(p, pagerank, betweenness, config, pr_threshold)
- )
-
- # Symbol spotlight budget
- sym_budget = max(0, remaining - n_file_cap)
- all_public_symbols = [
- (sym, p) for p in files for sym in p.symbols if sym.visibility == "public"
- ]
- n_sym_uncapped = (
- max(1, int(len(all_public_symbols) * config.top_symbol_percentile))
- if all_public_symbols
- else 0
- )
- n_sym_cap = min(n_sym_uncapped, sym_budget)
-
- # Infra page count
- infra_count = 0
- if not skip_infra:
- infra_count = sum(1 for p in files if _is_infra_file(p))
-
- # Cross-package count (monorepo only — estimate from inter-module edges)
- cross_package_count = 0
- try:
- # Check if monorepo structure exists
- if len(modules) > 1:
- # Count distinct cross-module import pairs
- seen_pairs: set[tuple[str, str]] = set()
- for u, v in graph.edges():
- u_parts = Path(u).parts
- v_parts = Path(v).parts
- u_mod = u_parts[0] if len(u_parts) > 1 else "root"
- v_mod = v_parts[0] if len(v_parts) > 1 else "root"
- if u_mod != v_mod:
- pair = (u_mod, v_mod)
- if pair not in seen_pairs:
- seen_pairs.add(pair)
- cross_package_count = len(seen_pairs)
- except Exception:
- pass
-
- # Build plan list
- if api_count:
- plans.append(PageTypePlan("api_contract", api_count, 0))
- if n_sym_cap:
- plans.append(PageTypePlan("symbol_spotlight", n_sym_cap, 1))
- if file_page_count:
- plans.append(PageTypePlan("file_page", file_page_count, 2))
- if scc_count:
- plans.append(PageTypePlan("scc_page", scc_count, 3))
- if module_count:
- plans.append(PageTypePlan("module_page", module_count, 4))
- if cross_package_count:
- plans.append(PageTypePlan("cross_package", cross_package_count, 5))
- plans.append(PageTypePlan("repo_overview", 1, 6))
- plans.append(PageTypePlan("architecture_diagram", 1, 6))
- if infra_count:
- plans.append(PageTypePlan("infra_page", infra_count, 7))
-
- return plans
-
-
-def estimate_cost(
- plans: list[PageTypePlan],
- provider_name: str,
- model_name: str,
-) -> CostEstimate:
- """Estimate token counts and USD cost from a generation plan."""
- total_pages = sum(p.count for p in plans)
- total_input = 0
- total_output = 0
-
- for plan in plans:
- inp, out = _TOKEN_HEURISTICS.get(plan.page_type, (3000, 2000))
- total_input += inp * plan.count
- total_output += out * plan.count
-
- input_rate, output_rate = _lookup_cost(model_name)
-
- cost = (total_input / 1000) * input_rate + (total_output / 1000) * output_rate
-
- return CostEstimate(
- plans=plans,
- total_pages=total_pages,
- estimated_input_tokens=total_input,
- estimated_output_tokens=total_output,
- estimated_cost_usd=cost,
- provider_name=provider_name,
- model_name=model_name,
- )
diff --git a/packages/cli/src/repowise/cli/cost_estimator/__init__.py b/packages/cli/src/repowise/cli/cost_estimator/__init__.py
new file mode 100644
index 000000000..161003616
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/__init__.py
@@ -0,0 +1,37 @@
+"""Cost estimation for the repowise generation pipeline.
+
+Public API kept stable for existing callers:
+
+- :func:`build_generation_plan` — given parsed_files + graph + config,
+ returns a list of :class:`PageTypePlan`. Internally delegates to
+ :func:`repowise.core.generation.select_pages` so the estimator and
+ the actual generator never disagree.
+- :func:`estimate_cost` — given plans + model, returns a
+ :class:`CostEstimate` with a median + low/high range. Optionally
+ calibrated against ``.repowise/db.sqlite`` telemetry.
+- :func:`_lookup_cost` — retained for the provider unit tests.
+"""
+
+from .coverage import (
+ DEFAULT_COVERAGE_OPTIONS,
+ RECOMMENDED_COVERAGE,
+ CoverageOption,
+ compute_coverage_options,
+)
+from .estimator import estimate_cost
+from .plans import build_generation_plan
+from .pricing import _lookup_cost
+from .types import CostEstimate, CostRange, PageTypePlan
+
+__all__ = [
+ "CostEstimate",
+ "CostRange",
+ "CoverageOption",
+ "DEFAULT_COVERAGE_OPTIONS",
+ "PageTypePlan",
+ "RECOMMENDED_COVERAGE",
+ "_lookup_cost",
+ "build_generation_plan",
+ "compute_coverage_options",
+ "estimate_cost",
+]
diff --git a/packages/cli/src/repowise/cli/cost_estimator/calibration.py b/packages/cli/src/repowise/cli/cost_estimator/calibration.py
new file mode 100644
index 000000000..60acb80f9
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/calibration.py
@@ -0,0 +1,55 @@
+"""Telemetry-based calibration from prior generation runs.
+
+When ``.repowise/db.sqlite`` exists and has ``wiki_pages`` rows for the
+target repo, we read actual ``input_tokens`` / ``output_tokens`` per
+page_type and use the per-type average instead of the static
+heuristics. This makes the **second** run of ``repowise init`` /
+``repowise update`` on the same repo highly accurate.
+
+The function is intentionally tolerant of missing tables and schema
+drift — any error short-circuits to ``{}`` so the heuristic path runs.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+
+def load_telemetry_averages(repo_path: Path | str) -> dict[str, tuple[float, float]]:
+ """Return ``{page_type: (avg_input, avg_output)}`` from prior runs.
+
+ Returns an empty dict when no telemetry is available — the caller
+ falls back to static heuristics.
+ """
+ db_path = Path(repo_path) / ".repowise" / "db.sqlite"
+ if not db_path.exists():
+ return {}
+
+ try:
+ with sqlite3.connect(str(db_path)) as conn:
+ cur = conn.execute(
+ """
+ SELECT page_type, AVG(input_tokens), AVG(output_tokens), COUNT(*)
+ FROM wiki_pages
+ WHERE input_tokens > 0 AND output_tokens > 0
+ GROUP BY page_type
+ """
+ )
+ rows = cur.fetchall()
+ except sqlite3.Error as exc:
+ log.debug("cost_estimator.telemetry_load_failed", error=str(exc))
+ return {}
+
+ averages: dict[str, tuple[float, float]] = {}
+ for page_type, avg_in, avg_out, n in rows:
+ # Require at least 3 historical samples per page_type to trust
+ # the average — fewer than that is dominated by outliers.
+ if n is None or n < 3:
+ continue
+ averages[str(page_type)] = (float(avg_in or 0.0), float(avg_out or 0.0))
+ return averages
diff --git a/packages/cli/src/repowise/cli/cost_estimator/coverage.py b/packages/cli/src/repowise/cli/cost_estimator/coverage.py
new file mode 100644
index 000000000..fa046ab1b
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/coverage.py
@@ -0,0 +1,85 @@
+"""Coverage-option enumeration for the init UI.
+
+The init flow asks the user to pick how thoroughly to document the
+repo. This module computes the per-coverage page counts + cost
+estimates that the UI renders, by calling :func:`build_generation_plan`
+once per offered percentage.
+
+Cheap — no LLM calls, just the deterministic selection function.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .estimator import estimate_cost
+from .plans import build_generation_plan
+from .types import CostEstimate, PageTypePlan
+
+
+# Coverage percentages presented in the init UI. The 20% slot is the
+# recommended default — fewer than 10% loses key concepts; more than
+# 50% starts paying for diminishing-return file pages.
+DEFAULT_COVERAGE_OPTIONS: tuple[float, ...] = (0.10, 0.15, 0.20, 0.30, 0.40, 0.50)
+RECOMMENDED_COVERAGE: float = 0.20
+
+
+@dataclass
+class CoverageOption:
+ """One row in the init-flow coverage table."""
+
+ pct: float
+ plans: list[PageTypePlan]
+ estimate: CostEstimate
+ is_recommended: bool
+
+ def page_count_for(self, page_type: str) -> int:
+ for plan in self.plans:
+ if plan.page_type == page_type:
+ return plan.count
+ return 0
+
+
+def compute_coverage_options(
+ *,
+ parsed_files: list[Any],
+ graph_builder: Any,
+ base_config: Any,
+ provider_name: str,
+ model_name: str,
+ repo_path: Path | str | None = None,
+ skip_tests: bool = False,
+ skip_infra: bool = False,
+ percentages: tuple[float, ...] = DEFAULT_COVERAGE_OPTIONS,
+ recommended: float = RECOMMENDED_COVERAGE,
+) -> list[CoverageOption]:
+ """Return one :class:`CoverageOption` per requested percentage.
+
+ ``base_config`` is cloned per percentage so this never mutates the
+ caller's object.
+ """
+ from dataclasses import replace
+
+ options: list[CoverageOption] = []
+ for pct in percentages:
+ cfg = replace(base_config, coverage_pct=pct, max_pages_pct=pct)
+ plans = build_generation_plan(
+ parsed_files, graph_builder, cfg, skip_tests, skip_infra
+ )
+ est = estimate_cost(
+ plans,
+ provider_name,
+ model_name,
+ repo_path=repo_path,
+ )
+ options.append(
+ CoverageOption(
+ pct=pct,
+ plans=plans,
+ estimate=est,
+ is_recommended=abs(pct - recommended) < 1e-9,
+ )
+ )
+ return options
diff --git a/packages/cli/src/repowise/cli/cost_estimator/estimator.py b/packages/cli/src/repowise/cli/cost_estimator/estimator.py
new file mode 100644
index 000000000..9165c2c24
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/estimator.py
@@ -0,0 +1,78 @@
+"""Cost orchestration — combines plans + heuristics/telemetry + pricing."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .calibration import load_telemetry_averages
+from .heuristics import HEURISTIC_VARIANCE, heuristic_tokens
+from .pricing import _lookup_cost
+from .types import CostEstimate, CostRange, PageTypePlan
+
+
+def _tokens_per_page(
+ page_type: str,
+ *,
+ telemetry: dict[str, tuple[float, float]],
+) -> tuple[float, float]:
+ """Return ``(input, output)`` tokens for *page_type*.
+
+ Prefers telemetry averages when available — those reflect *this
+ repo's* actual prompt sizes. Falls back to heuristics for fresh
+ repos and unknown page types.
+ """
+ if page_type in telemetry:
+ return telemetry[page_type]
+ return heuristic_tokens(page_type)
+
+
+def estimate_cost(
+ plans: list[PageTypePlan],
+ provider_name: str,
+ model_name: str,
+ *,
+ repo_path: Path | str | None = None,
+) -> CostEstimate:
+ """Estimate token counts and USD cost from a generation plan.
+
+ When ``repo_path`` points to a repo with prior generation telemetry
+ in ``.repowise/db.sqlite``, the estimate is calibrated against the
+ actual averages from that repo. Otherwise the static heuristics in
+ :mod:`heuristics` are used and a wider variance bracket is reported.
+ """
+ telemetry: dict[str, tuple[float, float]] = {}
+ if repo_path is not None:
+ telemetry = load_telemetry_averages(repo_path)
+ is_calibrated = bool(telemetry)
+
+ total_pages = sum(p.count for p in plans)
+ total_input = 0.0
+ total_output = 0.0
+
+ for plan in plans:
+ inp, out = _tokens_per_page(plan.page_type, telemetry=telemetry)
+ total_input += inp * plan.count
+ total_output += out * plan.count
+
+ input_rate, output_rate = _lookup_cost(model_name)
+ median_cost = (total_input / 1000) * input_rate + (total_output / 1000) * output_rate
+
+ # Tighter variance when telemetry calibrated us; wider for cold-start.
+ variance = 0.10 if is_calibrated else HEURISTIC_VARIANCE
+ cost_range = CostRange(
+ low=median_cost * (1.0 - variance),
+ median=median_cost,
+ high=median_cost * (1.0 + variance),
+ )
+
+ return CostEstimate(
+ plans=plans,
+ total_pages=total_pages,
+ estimated_input_tokens=int(total_input),
+ estimated_output_tokens=int(total_output),
+ estimated_cost_usd=median_cost,
+ provider_name=provider_name,
+ model_name=model_name,
+ cost_range=cost_range,
+ is_calibrated=is_calibrated,
+ )
diff --git a/packages/cli/src/repowise/cli/cost_estimator/heuristics.py b/packages/cli/src/repowise/cli/cost_estimator/heuristics.py
new file mode 100644
index 000000000..2b2e1c770
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/heuristics.py
@@ -0,0 +1,40 @@
+"""Per-page-type token heuristics.
+
+These default averages were recalibrated after observing a real run on
+the repowise repo where the prior estimate (4000/2500 per ``file_page``)
+was ~5-8× under-estimating actual input tokens. Real pages include the
+full Jinja2-rendered context: file path + symbol signatures + imports +
+exports + source snippet + dep summaries + RAG context + git metadata.
+
+The estimator prefers SQLite telemetry when available
+(see :mod:`calibration`); these heuristics are the cold-start fallback.
+"""
+
+from __future__ import annotations
+
+
+# (input_tokens, output_tokens) per page. Heuristics that bias slightly
+# high — better to over-estimate cost than surprise the user with a
+# bill that's an order of magnitude over the quote.
+_TOKEN_HEURISTICS: dict[str, tuple[int, int]] = {
+ "api_contract": (8_000, 3_000),
+ "symbol_spotlight": (8_000, 2_500),
+ "file_page": (25_000, 5_000),
+ "scc_page": (12_000, 4_000),
+ "module_page": (45_000, 6_000),
+ "repo_overview": (30_000, 5_000),
+ "architecture_diagram": (25_000, 5_000),
+ "infra_page": (4_000, 2_000),
+ "onboarding": (15_000, 3_500),
+}
+
+
+# Variance bracket used to derive the low/high range. Plus/minus 25%
+# captures roughly one standard deviation of observed variance in mixed
+# real-world generation runs.
+HEURISTIC_VARIANCE = 0.25
+
+
+def heuristic_tokens(page_type: str) -> tuple[int, int]:
+ """Return ``(input, output)`` tokens for *page_type*; safe default."""
+ return _TOKEN_HEURISTICS.get(page_type, (10_000, 3_000))
diff --git a/packages/cli/src/repowise/cli/cost_estimator/plans.py b/packages/cli/src/repowise/cli/cost_estimator/plans.py
new file mode 100644
index 000000000..8e74aeeee
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/plans.py
@@ -0,0 +1,109 @@
+"""Build a generation plan from ingestion output.
+
+Thin wrapper over :func:`repowise.core.generation.select_pages`. The
+cost estimator and the page generator therefore *share the same
+selection logic* — the pre-run estimate cannot drift from the actual
+run.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from repowise.core.generation import GENERATION_LEVELS
+from repowise.core.generation.selection import SelectionInputs, select_pages
+
+from .types import PageTypePlan
+
+
+def build_generation_plan(
+ parsed_files: list[Any],
+ graph_builder: Any,
+ config: Any,
+ skip_tests: bool = False,
+ skip_infra: bool = False,
+) -> list[PageTypePlan]:
+ """Return the per-page-type plan that ``generate_all`` will execute.
+
+ Parameters
+ ----------
+ parsed_files:
+ Output of the ingestion phase. Test/infra files are filtered
+ here when ``skip_tests`` / ``skip_infra`` are set, matching the
+ upstream pipeline's behavior.
+ graph_builder:
+ Finalized GraphBuilder (build() already called).
+ config:
+ GenerationConfig — its ``coverage_pct`` drives the budget.
+ """
+ files = parsed_files
+ if skip_tests:
+ files = [p for p in files if not p.file_info.is_test]
+ if skip_infra:
+ from repowise.core.generation.selection.selector import _is_infra_file
+
+ files = [p for p in files if not _is_infra_file(p)]
+
+ pagerank = graph_builder.pagerank()
+ betweenness = graph_builder.betweenness_centrality()
+ community = graph_builder.community_detection()
+ sccs = list(graph_builder.strongly_connected_components())
+ try:
+ community_info_map = graph_builder.community_info() or {}
+ except Exception:
+ community_info_map = {}
+
+ selection = select_pages(
+ SelectionInputs(
+ parsed_files=files,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ community_info=community_info_map,
+ sccs=sccs,
+ git_meta_map=None,
+ config=config,
+ )
+ )
+
+ counts = selection.counts()
+
+ plans: list[PageTypePlan] = []
+ # Preserve the level-ordered output the prior implementation
+ # exposed — callers like the init UI render plans in this order.
+ for page_type in (
+ "api_contract",
+ "symbol_spotlight",
+ "file_page",
+ "scc_page",
+ "module_page",
+ "repo_overview",
+ "architecture_diagram",
+ "infra_page",
+ ):
+ count = counts.get(page_type, 0)
+ if count:
+ plans.append(
+ PageTypePlan(
+ page_type=page_type,
+ count=count,
+ level=GENERATION_LEVELS[page_type],
+ )
+ )
+
+ # Onboarding count comes from the registry (curated, gated at
+ # generation time); the budget allocator does not touch it.
+ if getattr(config, "enable_onboarding", True):
+ from repowise.core.generation.onboarding import iter_specs as _iter_onboarding
+
+ onboarding_count = len(_iter_onboarding())
+ if onboarding_count:
+ plans.append(
+ PageTypePlan(
+ page_type="onboarding",
+ count=onboarding_count,
+ level=GENERATION_LEVELS["onboarding"],
+ )
+ )
+
+ return plans
diff --git a/packages/cli/src/repowise/cli/cost_estimator/pricing.py b/packages/cli/src/repowise/cli/cost_estimator/pricing.py
new file mode 100644
index 000000000..8c461d4a4
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/pricing.py
@@ -0,0 +1,52 @@
+"""Per-model token pricing.
+
+Rates are USD per 1K tokens (input, output). Exact model names win
+first; longest-prefix fallback catches unknown variants.
+"""
+
+from __future__ import annotations
+
+
+# Exact-match rates. Per-MTok pricing divided by 1000.
+_COST_TABLE_EXACT: dict[str, tuple[float, float]] = {
+ # OpenAI GPT-5.4 family
+ "gpt-5.4": (0.0025, 0.015), # $2.50 / $15 per MTok
+ "gpt-5.4-mini": (0.00075, 0.0045), # $0.75 / $4.50 per MTok
+ "gpt-5.4-nano": (0.0002, 0.00125), # $0.20 / $1.25 per MTok
+ # Gemini family
+ "gemini-3.1-pro-preview": (0.002, 0.012), # $2 / $12 per MTok
+ "gemini-3-flash-preview": (0.0005, 0.003), # $0.50 / $3 per MTok
+ "gemini-3.1-flash-lite-preview": (0.00025, 0.0015), # $0.25 / $1.50 per MTok
+ # Anthropic Claude 4.x family
+ "claude-opus-4-6": (0.005, 0.025), # $5 / $25 per MTok
+ "claude-sonnet-4-6": (0.003, 0.015), # $3 / $15 per MTok
+ "claude-haiku-4-5": (0.001, 0.005), # $1 / $5 per MTok
+}
+
+# Prefix fallbacks for unknown variants.
+_COST_TABLE_PREFIX: dict[str, tuple[float, float]] = {
+ "gpt-5.4-nano": (0.0002, 0.00125),
+ "gpt-5.4-mini": (0.00075, 0.0045),
+ "gpt-5.4": (0.0025, 0.015),
+ "claude-opus": (0.005, 0.025),
+ "claude-sonnet": (0.003, 0.015),
+ "claude-haiku": (0.001, 0.005),
+ "claude": (0.003, 0.015),
+ "gemini": (0.00025, 0.0015),
+ "llama": (0.0, 0.0),
+ "mock": (0.0, 0.0),
+}
+
+
+def _lookup_cost(model_name: str) -> tuple[float, float]:
+ """Return ``(input_rate, output_rate)`` per 1K tokens for *model_name*."""
+ lower = model_name.lower()
+ if lower in _COST_TABLE_EXACT:
+ return _COST_TABLE_EXACT[lower]
+ best_prefix = ""
+ best_rates = (0.0, 0.0)
+ for prefix, rates in _COST_TABLE_PREFIX.items():
+ if lower.startswith(prefix) and len(prefix) > len(best_prefix):
+ best_prefix = prefix
+ best_rates = rates
+ return best_rates
diff --git a/packages/cli/src/repowise/cli/cost_estimator/types.py b/packages/cli/src/repowise/cli/cost_estimator/types.py
new file mode 100644
index 000000000..2f84b5232
--- /dev/null
+++ b/packages/cli/src/repowise/cli/cost_estimator/types.py
@@ -0,0 +1,48 @@
+"""Public dataclasses for the cost estimator."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class PageTypePlan:
+ """Count of pages to generate for a given page type."""
+
+ page_type: str
+ count: int
+ level: int
+
+
+@dataclass
+class CostRange:
+ """Low / median / high cost estimate.
+
+ The median is the best single point estimate. ``low`` and ``high``
+ bracket the typical variance observed in real runs (roughly ±25%
+ when only heuristics are used; tighter when telemetry calibration
+ is available).
+ """
+
+ low: float
+ median: float
+ high: float
+
+
+@dataclass
+class CostEstimate:
+ """Estimated cost for a generation run."""
+
+ plans: list[PageTypePlan] = field(default_factory=list)
+ total_pages: int = 0
+ estimated_input_tokens: int = 0
+ estimated_output_tokens: int = 0
+ estimated_cost_usd: float = 0.0
+ provider_name: str = ""
+ model_name: str = ""
+ # Range output — median equals ``estimated_cost_usd`` for backwards
+ # compatibility with existing callers.
+ cost_range: CostRange | None = None
+ # True when the estimate was calibrated against historical
+ # ``wiki_pages`` rows for this repo (tighter than heuristics alone).
+ is_calibrated: bool = False
diff --git a/packages/cli/src/repowise/cli/coverage_select.py b/packages/cli/src/repowise/cli/coverage_select.py
new file mode 100644
index 000000000..edcc63ae3
--- /dev/null
+++ b/packages/cli/src/repowise/cli/coverage_select.py
@@ -0,0 +1,104 @@
+"""Interactive coverage-level chooser for ``repowise init``.
+
+After ingestion completes but before the LLM run, the user picks how
+thoroughly to document the repo. Counts are computed from the actual
+ingestion result — not hard-coded heuristics — so the displayed
+"pages at this level" always matches what generation will emit.
+"""
+
+from __future__ import annotations
+
+from rich.console import Console
+from rich.prompt import Prompt
+from rich.table import Table
+
+from repowise.cli.cost_estimator import CoverageOption
+from repowise.cli.ui import BRAND, BRAND_STYLE
+
+
+# Columns shown in the coverage table. ``onboarding`` is constant
+# across percentages (curated slots) but we include it so the user
+# sees the full picture.
+_DISPLAY_COLUMNS: tuple[tuple[str, str], ...] = (
+ ("file_page", "File"),
+ ("symbol_spotlight", "Sym"),
+ ("module_page", "Mod"),
+ ("api_contract", "API"),
+ ("infra_page", "Infra"),
+ ("scc_page", "SCC"),
+ ("onboarding", "Onb"),
+)
+
+
+def _format_pct(option: CoverageOption) -> str:
+ label = f"{int(option.pct * 100)}%"
+ if option.is_recommended:
+ label += " (rec)"
+ return label
+
+
+def _format_cost(option: CoverageOption) -> str:
+ est = option.estimate
+ if est.cost_range is None:
+ return f"${est.estimated_cost_usd:.2f}"
+ return f"${est.cost_range.low:.2f}-${est.cost_range.high:.2f}"
+
+
+def render_coverage_table(options: list[CoverageOption]) -> Table:
+ """Build the Rich table shown to the user."""
+ table = Table(
+ title="[bold]Documentation coverage[/bold]",
+ title_style="",
+ border_style=BRAND,
+ padding=(0, 1),
+ )
+ table.add_column("#", style=BRAND_STYLE, width=4)
+ table.add_column("Coverage", style="bold")
+ table.add_column("Pages", justify="right")
+ for _, header in _DISPLAY_COLUMNS:
+ table.add_column(header, justify="right")
+ table.add_column("Est. cost", justify="right")
+
+ for idx, opt in enumerate(options, start=1):
+ row = [
+ f"[{idx}]",
+ _format_pct(opt),
+ str(opt.estimate.total_pages),
+ ]
+ for page_type, _ in _DISPLAY_COLUMNS:
+ row.append(str(opt.page_count_for(page_type)))
+ row.append(_format_cost(opt))
+ table.add_row(*row)
+
+ return table
+
+
+def interactive_coverage_select(
+ console: Console,
+ options: list[CoverageOption],
+) -> CoverageOption:
+ """Render the coverage table and prompt the user for a choice.
+
+ Returns the selected :class:`CoverageOption`. The default is the
+ one whose ``is_recommended`` flag is set, falling back to the
+ middle of the list when none is flagged.
+ """
+ if not options:
+ msg = "No coverage options provided"
+ raise ValueError(msg)
+
+ console.print(render_coverage_table(options))
+
+ # Pick the default = recommended option's index, else the median.
+ default_idx = next(
+ (i for i, o in enumerate(options, start=1) if o.is_recommended),
+ len(options) // 2 + 1,
+ )
+
+ choice = Prompt.ask(
+ " Select coverage",
+ choices=[str(i) for i in range(1, len(options) + 1)],
+ default=str(default_idx),
+ console=console,
+ )
+ return options[int(choice) - 1]
diff --git a/packages/cli/src/repowise/cli/ui.py b/packages/cli/src/repowise/cli/ui.py
index 7a7ef2684..01f228294 100644
--- a/packages/cli/src/repowise/cli/ui.py
+++ b/packages/cli/src/repowise/cli/ui.py
@@ -278,7 +278,7 @@ def print_phase_header(
_PROVIDER_DEFAULTS: dict[str, str] = {
"gemini": "gemini-3.1-flash-lite-preview",
- "openai": "gpt-4.1",
+ "openai": "gpt-5.4-nano",
"anthropic": "claude-sonnet-4-6",
"deepseek": "deepseek-v4-flash",
"ollama": "llama3.2",
@@ -1160,6 +1160,7 @@ _PHASE_LABELS: dict[str, str] = {
"dead_code": "Detecting dead code...",
"decisions": "Extracting decisions...",
"generation": "Generating pages...",
+ "onboarding": "Curating onboarding docs...",
}
diff --git a/packages/core/src/repowise/core/generation/__init__.py b/packages/core/src/repowise/core/generation/__init__.py
index d0b4701b9..db00b92d2 100644
--- a/packages/core/src/repowise/core/generation/__init__.py
+++ b/packages/core/src/repowise/core/generation/__init__.py
@@ -11,7 +11,6 @@ from .context_assembler import (
ApiContractContext,
ArchitectureDiagramContext,
ContextAssembler,
- DiffSummaryContext,
FilePageContext,
InfraPageContext,
ModulePageContext,
@@ -44,11 +43,36 @@ from .models import (
compute_source_hash,
decay_confidence,
)
+from .api_contract_detector import detect_code_api_contracts
+from .interlinking import (
+ LinkIndex,
+ WikiLink,
+ attach_wiki_links_and_backlinks,
+ resolve_wiki_links,
+)
from .page_generator import SYSTEM_PROMPTS, PageGenerator
+from .selection import (
+ BucketAllocation,
+ ModuleGroup,
+ Selection,
+ SelectionInputs,
+ select_pages,
+ summarize_selection,
+)
__all__ = [
+ "BucketAllocation",
"GENERATION_LEVELS",
+ "LinkIndex",
+ "ModuleGroup",
"SYSTEM_PROMPTS",
+ "Selection",
+ "SelectionInputs",
+ "WikiLink",
+ "attach_wiki_links_and_backlinks",
+ "resolve_wiki_links",
+ "select_pages",
+ "summarize_selection",
"ApiContractContext",
"ArchitectureDiagramContext",
"Checkpoint",
@@ -57,7 +81,6 @@ __all__ = [
"ContextAssembler",
"DeadCodeConfig",
"DecisionSummary",
- "DiffSummaryContext",
"EditorFileData",
"EditorFileDataFetcher",
"FilePageContext",
@@ -82,4 +105,5 @@ __all__ = [
"compute_page_id",
"compute_source_hash",
"decay_confidence",
+ "detect_code_api_contracts",
]
diff --git a/packages/core/src/repowise/core/generation/api_contract_detector.py b/packages/core/src/repowise/core/generation/api_contract_detector.py
new file mode 100644
index 000000000..733caf0d2
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/api_contract_detector.py
@@ -0,0 +1,98 @@
+"""Cross-language API-contract detection for ParsedFile objects.
+
+Traverser sets ``FileInfo.is_api_contract = True`` for OpenAPI/Swagger/proto/
+GraphQL files purely from filename/extension. That misses framework-defined
+HTTP surfaces (FastAPI routers, ASP.NET controllers, etc.) where the contract
+is expressed in code, not in a schema file.
+
+This module runs after parsing and flips ``is_api_contract`` for those code
+files based on small per-language heuristics that read only the parsed
+``Symbol``/``Import`` data — no source re-read, no LLM call.
+
+Adding a new framework: write a ``Detector`` callable and register it in
+``_DETECTORS`` keyed by ``LanguageTag``. Keep the heuristic conservative —
+false positives push junk through the api_contract template, false negatives
+just leave files in their default file_page path.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from repowise.core.ingestion.models import ParsedFile
+
+Detector = Callable[[ParsedFile], bool]
+
+
+def _python_is_fastapi_router(parsed: ParsedFile) -> bool:
+ # The parser sometimes resolves "from fastapi import APIRouter" with
+ # module_path = "fastapi" and imported_names = ["APIRouter"], and
+ # sometimes with module_path = "fastapi.APIRouter". Cover both.
+ imports_fastapi = any(
+ imp.module_path == "fastapi"
+ or imp.module_path.startswith("fastapi.")
+ for imp in parsed.imports
+ )
+ if not imports_fastapi:
+ return False
+ fastapi_names = {"APIRouter", "FastAPI"}
+ for imp in parsed.imports:
+ if set(imp.imported_names) & fastapi_names:
+ return True
+ # Decorator form: any symbol decorated with @router.get / @app.post / etc.
+ for sym in parsed.symbols:
+ for dec in sym.decorators:
+ head = dec.lstrip("@").split("(", 1)[0]
+ if "." in head and head.rsplit(".", 1)[1] in {
+ "get", "post", "put", "patch", "delete", "head", "options",
+ }:
+ return True
+ return False
+
+
+_ASPNET_CONTROLLER_BASES = frozenset({"ControllerBase", "Controller", "ApiController"})
+_ASPNET_ATTRIBUTES = frozenset({"ApiController", "Route", "HttpGet", "HttpPost", "HttpPut", "HttpDelete", "HttpPatch"})
+
+
+def _csharp_is_aspnet_controller(parsed: ParsedFile) -> bool:
+ for sym in parsed.symbols:
+ if sym.kind != "class":
+ continue
+ # Inheritance encoded in signature as ": Base" / ", Interface" in C#.
+ sig = sym.signature or ""
+ if any(base in sig for base in _ASPNET_CONTROLLER_BASES):
+ return True
+ for dec in sym.decorators:
+ attr = dec.lstrip("@").lstrip("[").split("(", 1)[0].rstrip("]")
+ if attr in _ASPNET_ATTRIBUTES:
+ return True
+ return False
+
+
+_DETECTORS: dict[str, Detector] = {
+ "python": _python_is_fastapi_router,
+ "csharp": _csharp_is_aspnet_controller,
+}
+
+
+def detect_code_api_contracts(parsed_files: list[ParsedFile]) -> int:
+ """Flip ``is_api_contract`` on parsed files that define an HTTP API surface.
+
+ Returns the number of files newly flagged. Files already flagged by the
+ traverser (OpenAPI/proto/GraphQL) are left untouched.
+ """
+ flipped = 0
+ for pf in parsed_files:
+ if pf.file_info.is_api_contract:
+ continue
+ detector = _DETECTORS.get(pf.file_info.language)
+ if detector is None:
+ continue
+ try:
+ if detector(pf):
+ pf.file_info.is_api_contract = True
+ flipped += 1
+ except Exception:
+ # Defensive: a malformed ParsedFile must never crash generation.
+ continue
+ return flipped
diff --git a/packages/core/src/repowise/core/generation/context_assembler.py b/packages/core/src/repowise/core/generation/context_assembler.py
index f879c3112..7da41dae1 100644
--- a/packages/core/src/repowise/core/generation/context_assembler.py
+++ b/packages/core/src/repowise/core/generation/context_assembler.py
@@ -67,6 +67,11 @@ class FilePageContext:
heritage: list[dict] = field(default_factory=list)
community_label: str = ""
community_cohesion: float = 0.0
+ # Architectural decisions touching this file (extracted by
+ # DecisionExtractor — inline WHY/DECISION markers, README mining,
+ # git archaeology). Kept short on purpose; the module-page renders
+ # the full list.
+ decision_records: list[dict] = field(default_factory=list)
@dataclass
@@ -100,6 +105,13 @@ class ModulePageContext:
community_label: str = ""
community_cohesion: float = 0.0
key_classes: list[dict] = field(default_factory=list)
+ # Phase 2 enrichment: surfaced when available, gracefully degrades.
+ decision_records: list[dict] = field(default_factory=list)
+ dead_code_findings: list[dict] = field(default_factory=list)
+ external_systems: list[dict] = field(default_factory=list)
+ # Top files inside the module by PageRank, for the "key files" section.
+ key_files: list[dict] = field(default_factory=list)
+ top_owners: list[dict] = field(default_factory=list)
@dataclass
@@ -136,6 +148,9 @@ class RepoOverviewContext:
# Graph intelligence enrichment
communities: list[dict] = field(default_factory=list)
execution_flows: list[dict] = field(default_factory=list)
+ # Phase 2: third-party dependencies + headline architectural decisions
+ external_systems: list[dict] = field(default_factory=list)
+ decision_records: list[dict] = field(default_factory=list)
@dataclass
@@ -164,32 +179,6 @@ class InfraPageContext:
targets: list[str]
-@dataclass
-class DiffSummaryContext:
- from_ref: str
- to_ref: str
- added_files: list[str]
- deleted_files: list[str]
- modified_files: list[str]
- symbol_diffs: list[Any]
- affected_page_ids: list[str]
- trigger_commit_sha: str | None = None
- trigger_commit_message: str | None = None
- trigger_commit_author: str | None = None
- diff_text: str | None = None
-
-
-@dataclass
-class CrossPackageContext:
- source_package: str
- target_package: str
- coupling_strength: int # number of boundary files
- used_symbols: list[str] # public symbols from target used by source
- boundary_files: list[str] # files in source that import from target
- source_pagerank_mean: float
- target_pagerank_mean: float
-
-
# ---------------------------------------------------------------------------
# ContextAssembler
# ---------------------------------------------------------------------------
@@ -244,6 +233,7 @@ class ContextAssembler:
git_meta: dict | None = None,
dead_code_findings: list[dict] | None = None,
page_summaries: dict[str, str] | None = None,
+ decision_records: list[dict] | None = None,
) -> FilePageContext:
"""Assemble context for the file_page template."""
path = parsed.file_info.path
@@ -349,6 +339,7 @@ class ContextAssembler:
heritage=heritage_entries,
community_label=community_label,
community_cohesion=community_cohesion,
+ decision_records=decision_records or [],
)
# ------------------------------------------------------------------
@@ -406,6 +397,12 @@ class ContextAssembler:
file_contexts: list[FilePageContext],
graph: Any, # nx.DiGraph
page_summaries: dict[str, str] | None = None,
+ git_meta_map: dict[str, dict] | None = None,
+ decision_records: list[dict] | None = None,
+ dead_code_findings: list[dict] | None = None,
+ external_systems: list[dict] | None = None,
+ community_label: str | None = None,
+ community_cohesion: float | None = None,
) -> ModulePageContext:
"""Assemble context for the module_page template."""
total_symbols = sum(len(fc.symbols) for fc in file_contexts)
@@ -436,16 +433,47 @@ class ContextAssembler:
if fp in page_summaries:
file_summaries[fp] = page_summaries[fp][:200]
- # Community info: pick the dominant community label across files
- community_label = ""
- community_cohesion = 0.0
- labels = [fc.community_label for fc in file_contexts if fc.community_label]
- if labels:
+ # Community info: prefer caller-supplied label (community-grouped
+ # module pages already know their label), else derive the dominant
+ # label across files.
+ if community_label is None:
+ community_label = ""
+ labels = [fc.community_label for fc in file_contexts if fc.community_label]
+ if labels:
+ from collections import Counter
+ community_label = Counter(labels).most_common(1)[0][0]
+ if community_cohesion is None:
+ community_cohesion = 0.0
+ cohesions = [fc.community_cohesion for fc in file_contexts if fc.community_cohesion > 0]
+ if cohesions:
+ community_cohesion = sum(cohesions) / len(cohesions)
+
+ # Key files inside the module by PageRank, with a short summary.
+ ranked = sorted(file_contexts, key=lambda fc: fc.pagerank_score, reverse=True)[:10]
+ key_files = [
+ {
+ "path": fc.file_path,
+ "pagerank": round(fc.pagerank_score, 4),
+ "summary": (file_summaries.get(fc.file_path) or "").strip()[:200],
+ "is_entry_point": fc.is_entry_point,
+ }
+ for fc in ranked
+ ]
+
+ # Aggregate ownership from git metadata: who maintains the most
+ # files in this module.
+ top_owners: list[dict] = []
+ if git_meta_map:
from collections import Counter
- community_label = Counter(labels).most_common(1)[0][0]
- cohesions = [fc.community_cohesion for fc in file_contexts if fc.community_cohesion > 0]
- if cohesions:
- community_cohesion = sum(cohesions) / len(cohesions)
+ owner_counts: Counter[str] = Counter()
+ for fp in files:
+ meta = git_meta_map.get(fp)
+ if meta and meta.get("primary_owner_name"):
+ owner_counts[meta["primary_owner_name"]] += 1
+ top_owners = [
+ {"name": name, "file_count": count}
+ for name, count in owner_counts.most_common(3)
+ ]
# Key classes: collect classes with heritage info from file contexts
key_classes: list[dict] = []
@@ -468,6 +496,11 @@ class ContextAssembler:
community_label=community_label,
community_cohesion=community_cohesion,
key_classes=key_classes,
+ decision_records=decision_records or [],
+ dead_code_findings=dead_code_findings or [],
+ external_systems=external_systems or [],
+ key_files=key_files,
+ top_owners=top_owners,
)
# ------------------------------------------------------------------
@@ -518,6 +551,8 @@ class ContextAssembler:
sccs: list[Any], # list[frozenset[str]]
community: dict[str, int],
graph_builder: Any | None = None,
+ external_systems: list[dict] | None = None,
+ decision_records: list[dict] | None = None,
) -> RepoOverviewContext:
"""Assemble context for the repo_overview template."""
# Top files sorted by PageRank descending
@@ -572,6 +607,8 @@ class ContextAssembler:
circular_dependency_count=circular_count,
communities=communities_list,
execution_flows=execution_flows_list,
+ external_systems=external_systems or [],
+ decision_records=decision_records or [],
)
# ------------------------------------------------------------------
@@ -678,88 +715,6 @@ class ContextAssembler:
targets=targets,
)
- # ------------------------------------------------------------------
- # Diff summary
- # ------------------------------------------------------------------
-
- def assemble_diff_summary(
- self,
- file_diffs: list[Any], # list[FileDiff]
- affected_pages: Any, # AffectedPages
- from_ref: str,
- to_ref: str,
- ) -> DiffSummaryContext:
- """Assemble context for the diff_summary template."""
- added_files = [d.path for d in file_diffs if d.status == "added"]
- deleted_files = [d.path for d in file_diffs if d.status == "deleted"]
- modified_files = [d.path for d in file_diffs if d.status == "modified"]
-
- symbol_diffs = [d.symbol_diff for d in file_diffs if d.symbol_diff is not None]
-
- affected_page_ids = list(affected_pages.regenerate) if affected_pages else []
-
- # Extract trigger commit from first diff that has it
- trigger_sha = None
- trigger_msg = None
- trigger_author = None
- diff_text = None
- for d in file_diffs:
- if getattr(d, "trigger_commit_sha", None):
- trigger_sha = d.trigger_commit_sha
- trigger_msg = d.trigger_commit_message
- trigger_author = d.trigger_commit_author
- diff_text = d.diff_text
- break
-
- return DiffSummaryContext(
- from_ref=from_ref,
- to_ref=to_ref,
- added_files=added_files,
- deleted_files=deleted_files,
- modified_files=modified_files,
- symbol_diffs=symbol_diffs,
- affected_page_ids=affected_page_ids,
- trigger_commit_sha=trigger_sha,
- trigger_commit_message=trigger_msg,
- trigger_commit_author=trigger_author,
- diff_text=diff_text,
- )
-
- # ------------------------------------------------------------------
- # Cross-package context (Phase 9 C2)
- # ------------------------------------------------------------------
-
- def assemble_cross_package(
- self,
- source_pkg: str,
- target_pkg: str,
- source_fcs: list[FilePageContext],
- target_fcs: list[FilePageContext],
- graph: Any,
- ) -> CrossPackageContext:
- """Assemble context for cross-package dependency page."""
- target_paths = {fc.file_path for fc in target_fcs}
- boundary = [fc for fc in source_fcs if any(d in target_paths for d in fc.dependencies)]
- used_syms = sorted(
- {
- sym["name"]
- for fc in target_fcs
- for sym in fc.symbols
- if sym.get("visibility") == "public"
- }
- )[:20]
- return CrossPackageContext(
- source_package=source_pkg,
- target_package=target_pkg,
- coupling_strength=len(boundary),
- used_symbols=used_syms,
- boundary_files=[fc.file_path for fc in boundary],
- source_pagerank_mean=sum(fc.pagerank_score for fc in source_fcs)
- / max(len(source_fcs), 1),
- target_pagerank_mean=sum(fc.pagerank_score for fc in target_fcs)
- / max(len(target_fcs), 1),
- )
-
# ------------------------------------------------------------------
# Structural summary for large files (Phase 9 C1)
# ------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/interlinking.py b/packages/core/src/repowise/core/generation/interlinking.py
new file mode 100644
index 000000000..41f9fd077
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/interlinking.py
@@ -0,0 +1,256 @@
+"""Post-generation interlinking — resolve refs to wiki page IDs.
+
+After every page is generated, this module scans the markdown for
+backtick-quoted references to files / symbols and resolves them to
+``page_id``s of *other* generated pages. Resolved hits land in
+``page.metadata["wiki_links"]``; the reverse index is materialized as
+``page.metadata["backlinks"]`` on each target. The frontend MDX
+preprocessor reads these maps to render anchor tags.
+
+No LLM call — pure regex + dict lookup. Runs in roughly
+``O(N_pages * avg_refs_per_page)`` and is therefore cheap enough to
+run on every generation.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from .models import GeneratedPage, compute_page_id
+
+log = structlog.get_logger(__name__)
+
+
+# Backtick-wrapped identifier-ish strings: file paths, symbol names,
+# qualified names. The same expression used by the hallucination
+# validator — we now repurpose it as a *resolver* instead of a *warner*.
+_BACKTICK_REF_RE = re.compile(r"(?<!`)` *([A-Za-z_/.][\w./\-]*?) *`(?!`)")
+
+# Markdown code fence — content inside is verbatim source, not refs.
+_FENCE_RE = re.compile(r"```.*?```", re.DOTALL)
+
+
+# ---------------------------------------------------------------------------
+# Link index
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LinkIndex:
+ """Lookup tables that map a ref string to a target page_id.
+
+ Built once per generation run; queried per page.
+ """
+
+ by_path: dict[str, str] = field(default_factory=dict)
+ by_basename: dict[str, str] = field(default_factory=dict)
+ by_symbol_qname: dict[str, str] = field(default_factory=dict)
+ by_target: dict[str, str] = field(default_factory=dict)
+
+ @classmethod
+ def build(
+ cls,
+ pages: list[GeneratedPage],
+ parsed_files: list[Any] | None = None,
+ ) -> LinkIndex:
+ """Compose lookup tables from generated pages + parsed files."""
+ idx = cls()
+
+ # Path / target → page_id. ``target_path`` is the canonical key
+ # for non-symbol pages; file_page targets are file paths.
+ for page in pages:
+ if not page.target_path:
+ continue
+ idx.by_target[page.target_path] = page.page_id
+ if page.page_type in {"file_page", "api_contract", "infra_page"}:
+ idx.by_path[page.target_path] = page.page_id
+ base = Path(page.target_path).name
+ # Don't overwrite — first wins to avoid ambiguous basename collisions.
+ idx.by_basename.setdefault(base, page.page_id)
+ elif page.page_type == "symbol_spotlight":
+ # target_path is "file::name" — map the qualified form
+ if "::" in page.target_path:
+ idx.by_symbol_qname.setdefault(
+ page.target_path, page.page_id
+ )
+
+ # Augment with symbol qualified_names from the parser even when
+ # the symbol itself didn't get a spotlight — the link resolves
+ # to its host file_page instead.
+ if parsed_files:
+ for pf in parsed_files:
+ host_page = idx.by_path.get(pf.file_info.path)
+ if not host_page:
+ continue
+ for sym in pf.symbols:
+ qname = getattr(sym, "qualified_name", "") or sym.name
+ idx.by_symbol_qname.setdefault(qname, host_page)
+ # Last-segment match for `Foo.bar` → `bar`
+ if "." in qname:
+ idx.by_symbol_qname.setdefault(
+ qname.rsplit(".", 1)[-1], host_page
+ )
+
+ return idx
+
+ def resolve(self, ref: str) -> str | None:
+ """Resolve a single ref to a ``page_id``; ``None`` if unmatched."""
+ if not ref:
+ return None
+ # Direct page-target hit (highest confidence).
+ if ref in self.by_target:
+ return self.by_target[ref]
+ # File path hit.
+ if ref in self.by_path:
+ return self.by_path[ref]
+ # Basename file hit (e.g. "wiki.py" → packages/.../wiki.py).
+ if "/" not in ref and "." in ref and ref in self.by_basename:
+ return self.by_basename[ref]
+ # Symbol qualified-name hit.
+ if ref in self.by_symbol_qname:
+ return self.by_symbol_qname[ref]
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Resolution
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class WikiLink:
+ anchor: str # the user-visible text matched by the ref
+ target_page_id: str
+ kind: str # "file" | "symbol" | "page"
+
+
+def _strip_code_fences(content: str) -> str:
+ """Drop fenced code blocks so source examples don't generate refs."""
+ return _FENCE_RE.sub("", content)
+
+
+def resolve_wiki_links(
+ page: GeneratedPage,
+ index: LinkIndex,
+ *,
+ max_links_per_page: int = 50,
+) -> list[WikiLink]:
+ """Return the set of resolved wiki links for *page*.
+
+ Deduplicates by ``target_page_id`` — a page that mentions
+ ``foo/bar.py`` ten times yields one link, not ten.
+ """
+ text = _strip_code_fences(page.content or "")
+ seen_targets: set[str] = set()
+ links: list[WikiLink] = []
+ for raw_ref in _BACKTICK_REF_RE.findall(text):
+ ref = raw_ref.strip()
+ target = index.resolve(ref)
+ if target is None or target == page.page_id:
+ continue
+ if target in seen_targets:
+ continue
+ seen_targets.add(target)
+ kind = "file" if "/" in ref or ref.endswith(
+ (".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".rs", ".java")
+ ) else "symbol"
+ links.append(WikiLink(anchor=ref, target_page_id=target, kind=kind))
+ if len(links) >= max_links_per_page:
+ break
+ return links
+
+
+# ---------------------------------------------------------------------------
+# Top-level orchestration
+# ---------------------------------------------------------------------------
+
+
+def attach_wiki_links_and_backlinks(
+ pages: list[GeneratedPage],
+ parsed_files: list[Any] | None = None,
+) -> None:
+ """Populate ``metadata['wiki_links']`` and ``metadata['backlinks']``.
+
+ Mutates each :class:`GeneratedPage` in place. Idempotent and safe
+ to call on a partially-populated page set — pages with no resolved
+ refs simply get empty lists.
+ """
+ if not pages:
+ return
+
+ index = LinkIndex.build(pages, parsed_files)
+ by_id: dict[str, GeneratedPage] = {p.page_id: p for p in pages}
+
+ # First pass — resolve forward links per page.
+ for page in pages:
+ wiki_links = resolve_wiki_links(page, index)
+ page.metadata["wiki_links"] = [
+ {"anchor": w.anchor, "target_page_id": w.target_page_id, "kind": w.kind}
+ for w in wiki_links
+ ]
+
+ # Second pass — build the reverse index. ``backlinks`` on each target
+ # lists the source pages that linked to it, capped at a reasonable
+ # number for UI rendering.
+ backlink_map: dict[str, list[dict]] = {pid: [] for pid in by_id}
+ for page in pages:
+ forward = page.metadata.get("wiki_links") or []
+ for link in forward:
+ target_id = link.get("target_page_id")
+ if not target_id or target_id not in backlink_map:
+ continue
+ backlink_map[target_id].append(
+ {
+ "source_page_id": page.page_id,
+ "source_title": page.title,
+ "source_page_type": page.page_type,
+ "anchor": link.get("anchor", ""),
+ }
+ )
+
+ _BACKLINK_CAP = 25
+ for page_id, sources in backlink_map.items():
+ if not sources:
+ by_id[page_id].metadata["backlinks"] = []
+ continue
+ # Dedup by source_page_id — a single source linking via 5
+ # different anchors counts once.
+ seen: set[str] = set()
+ unique: list[dict] = []
+ for entry in sources:
+ sid = entry["source_page_id"]
+ if sid in seen:
+ continue
+ seen.add(sid)
+ unique.append(entry)
+ if len(unique) >= _BACKLINK_CAP:
+ break
+ by_id[page_id].metadata["backlinks"] = unique
+
+ log.info(
+ "wiki_links.resolved",
+ pages=len(pages),
+ total_forward_links=sum(
+ len(p.metadata.get("wiki_links") or []) for p in pages
+ ),
+ pages_with_backlinks=sum(
+ 1 for p in pages if p.metadata.get("backlinks")
+ ),
+ )
+
+
+__all__ = [
+ "LinkIndex",
+ "WikiLink",
+ "attach_wiki_links_and_backlinks",
+ "resolve_wiki_links",
+]
+
+
+# Re-export for type hints
+_ = compute_page_id
diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.py
index 0b50458b4..7372f7964 100644
--- a/packages/core/src/repowise/core/generation/models.py
+++ b/packages/core/src/repowise/core/generation/models.py
@@ -26,25 +26,26 @@ PageType = Literal[
"file_page",
"scc_page",
"module_page",
- "cross_package",
"repo_overview",
"architecture_diagram",
"infra_page",
- "diff_summary",
+ # Phase 3: onboarding collection (subkind in metadata).
+ "onboarding",
]
-# Maps PageType → generation level (0 = first, 7 = last)
+# Maps PageType → generation level (0 = first, 8 = last).
+# Onboarding runs last so it can reference module/file pages already in the
+# wiki and so its prompts see the freshest signal bundle.
GENERATION_LEVELS: dict[str, int] = {
"api_contract": 0,
"symbol_spotlight": 1,
"file_page": 2,
"scc_page": 3,
"module_page": 4,
- "cross_package": 5,
"repo_overview": 6,
"architecture_diagram": 6,
"infra_page": 7,
- "diff_summary": 7,
+ "onboarding": 8,
}
FreshnessStatus = Literal["fresh", "stale", "expired", "unknown"]
@@ -74,19 +75,61 @@ class GenerationConfig:
jobs_dir: Directory for job checkpoint JSON files.
"""
- max_tokens: int = 16000
+ max_tokens: int = 20000
temperature: float = 0.3
token_budget: int = 48000
- max_concurrency: int = 5
+ max_concurrency: int = 12
embed_concurrency: int | None = None
reasoning: ReasoningMode = "auto"
cache_enabled: bool = True
staleness_threshold_days: int = 7
expiry_threshold_days: int = 30
- top_symbol_percentile: float = 0.10 # top N% public symbols by PageRank → symbol_spotlight
- file_page_top_percentile: float = 0.10 # top N% code files by PageRank → file_page
- file_page_min_symbols: int = 1 # files with fewer symbols are skipped for file_page
- max_pages_pct: float = 0.10 # hard cap: total pages ≤ max(50, N_files * this)
+ # ---- Coverage budget (enforced by generation.selection) ----------
+ # ``coverage_pct`` is the single knob users care about: the fraction
+ # of repo files that should produce a wiki page. The selection
+ # subsystem (``generation.selection``) is the *single source of
+ # truth* — it scores every candidate, allocates the budget across
+ # buckets via the share fields below, and returns an allow-set that
+ # both page_generator and cost_estimator honor verbatim. There is
+ # no longer an absolute cap — the percentage scales linearly.
+ coverage_pct: float = 0.20
+ file_page_share: float = 0.50
+ symbol_spotlight_share: float = 0.15
+ module_page_share: float = 0.10
+ api_contract_share: float = 0.08
+ infra_page_share: float = 0.05
+ scc_share: float = 0.04
+ # ``max_pages_pct`` is kept as a deprecated alias for backwards
+ # compatibility — older tests and CLI flows still read it. The
+ # selector picks ``coverage_pct`` when set and falls back here.
+ max_pages_pct: float = 0.20
+ # Legacy percentile knobs are retained for callers that want fine
+ # control but no longer drive page selection on their own.
+ top_symbol_percentile: float = 0.20
+ file_page_top_percentile: float = 0.10
+ file_page_min_symbols: int = 1
+ skip_trivial_files: bool = True
+ dedupe_near_clones: bool = True
+ # Phase 2: switch module_page grouping from top-directory to graph
+ # communities. min_module_size is the floor below which a community
+ # doesn't get its own page (its files still appear under file_page).
+ module_grouping: Literal["community", "top_dir"] = "community"
+ min_module_size: int = 3
+ # Phase 3: emit the curated Onboarding collection at level 8. Each
+ # subkind defines its own gate; slots whose gates fail are silently
+ # skipped (no UI nav entry either).
+ enable_onboarding: bool = True
+ # When True, file_page generation runs a vector-store search (one
+ # embedder round-trip per page) to inject related-page snippets into
+ # the prompt. On cheap models the extra latency is often more costly
+ # than the marginal quality lift — turn off to skip the search.
+ # See also rag_min_store_size below for the auto-bypass on small stores.
+ enable_rag_context: bool = True
+ # RAG search is bypassed entirely until the vector store has at least
+ # this many pages. The first wave of file_page generation runs against
+ # an empty / nearly-empty store anyway, so the search is a wasted
+ # round-trip until enough content is indexed to return useful hits.
+ rag_min_store_size: int = 10
jobs_dir: str = ".repowise/jobs"
large_file_source_pct: float = 0.4 # use structural summary when source tokens > budget * this
language: str = "en"
diff --git a/packages/core/src/repowise/core/generation/onboarding/__init__.py b/packages/core/src/repowise/core/generation/onboarding/__init__.py
new file mode 100644
index 000000000..6d3f569e6
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/__init__.py
@@ -0,0 +1,59 @@
+"""Onboarding documentation collection.
+
+A curated set of up to eight pages — Project Overview, Architecture Guide,
+Getting Started, Codebase Map, Key Concepts, How It Works, Development Guide,
+Active Landscape — designed to be the first thing a new contributor (or LLM
+agent) reads.
+
+Two slots ("project_overview", "architecture_guide") are *promoted*: they
+reuse the existing ``repo_overview`` and ``architecture_diagram`` pages,
+tagged via ``metadata.onboarding_slot``. The other six slots are new pages
+generated at level 8 with ``page_type='onboarding'`` and a
+``metadata.subkind`` discriminator.
+
+Architecture:
+ - :mod:`slots` — slot identifiers, fixed reading order, promoted map.
+ - :mod:`signals` — typed bundle of inputs passed to every subkind builder.
+ - :mod:`registry` — :class:`SubkindSpec` + register/get/iter API.
+ - :mod:`subkinds` — subkind modules that register themselves on import.
+"""
+
+from .registry import SubkindSpec, get_spec, iter_specs, register
+from .signals import OnboardingSignals
+from .slots import (
+ ONBOARDING_ORDER,
+ PROMOTED_SLOTS,
+ SLOT_ACTIVE_LANDSCAPE,
+ SLOT_ARCHITECTURE_GUIDE,
+ SLOT_CODEBASE_MAP,
+ SLOT_DEVELOPMENT_GUIDE,
+ SLOT_GETTING_STARTED,
+ SLOT_HOW_IT_WORKS,
+ SLOT_KEY_CONCEPTS,
+ SLOT_PROJECT_OVERVIEW,
+ SLOT_TITLES,
+ target_path,
+)
+
+# Side-effect import: registers every implemented subkind.
+from . import subkinds # noqa: E402, F401
+
+__all__ = [
+ "ONBOARDING_ORDER",
+ "OnboardingSignals",
+ "PROMOTED_SLOTS",
+ "SLOT_ACTIVE_LANDSCAPE",
+ "SLOT_ARCHITECTURE_GUIDE",
+ "SLOT_CODEBASE_MAP",
+ "SLOT_DEVELOPMENT_GUIDE",
+ "SLOT_GETTING_STARTED",
+ "SLOT_HOW_IT_WORKS",
+ "SLOT_KEY_CONCEPTS",
+ "SLOT_PROJECT_OVERVIEW",
+ "SLOT_TITLES",
+ "SubkindSpec",
+ "get_spec",
+ "iter_specs",
+ "register",
+ "target_path",
+]
diff --git a/packages/core/src/repowise/core/generation/onboarding/registry.py b/packages/core/src/repowise/core/generation/onboarding/registry.py
new file mode 100644
index 000000000..1f9ae5082
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/registry.py
@@ -0,0 +1,81 @@
+"""Registry of onboarding subkind specifications.
+
+Each subkind module declares a :class:`SubkindSpec` and calls :func:`register`
+on import. The dispatcher in :mod:`page_generator` iterates :func:`iter_specs`
+to drive generation — adding a new subkind never requires changes to the
+dispatcher itself.
+
+A subkind's ``build_context`` is allowed to return ``None`` when the slot's
+gate (e.g. "needs >= 50 commits") fails for this repo. The dispatcher then
+silently skips the slot — both the page and any UI nav entry.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from .signals import OnboardingSignals
+from .slots import ONBOARDING_ORDER, PROMOTED_SLOTS
+
+# A builder returns either a context object the template can render, or None
+# to indicate the gate failed and the slot should be skipped for this repo.
+BuildContext = Callable[[OnboardingSignals], object | None]
+
+
+@dataclass(frozen=True)
+class SubkindSpec:
+ """One onboarding subkind.
+
+ Attributes:
+ slot: Slot identifier from :mod:`onboarding.slots`.
+ title: Display title for the generated page.
+ template: Jinja template filename, relative to
+ ``templates/onboarding/``.
+ build_context: Returns the template context, or ``None`` to skip.
+ """
+
+ slot: str
+ title: str
+ template: str
+ build_context: BuildContext
+
+
+_REGISTRY: dict[str, SubkindSpec] = {}
+
+
+def register(spec: SubkindSpec) -> None:
+ """Register a subkind spec. Idempotent re-registration is rejected."""
+ if spec.slot in PROMOTED_SLOTS.values():
+ # Defensive: promoted slots are not generated through this path.
+ raise ValueError(
+ f"Slot '{spec.slot}' is promoted and must not be registered "
+ "as a generated subkind."
+ )
+ if spec.slot in _REGISTRY:
+ raise ValueError(f"Duplicate onboarding subkind: {spec.slot}")
+ if spec.slot not in ONBOARDING_ORDER:
+ raise ValueError(
+ f"Unknown onboarding slot '{spec.slot}'. "
+ f"Add it to ONBOARDING_ORDER in slots.py first."
+ )
+ _REGISTRY[spec.slot] = spec
+
+
+def get_spec(slot: str) -> SubkindSpec | None:
+ """Return the spec for a slot, or ``None`` if unregistered."""
+ return _REGISTRY.get(slot)
+
+
+def iter_specs() -> list[SubkindSpec]:
+ """Return registered specs in canonical reading order.
+
+ Promoted slots (project_overview, architecture_guide) are not generated
+ through this path and are excluded here.
+ """
+ promoted = set(PROMOTED_SLOTS.values())
+ return [
+ _REGISTRY[slot]
+ for slot in ONBOARDING_ORDER
+ if slot not in promoted and slot in _REGISTRY
+ ]
diff --git a/packages/core/src/repowise/core/generation/onboarding/signals.py b/packages/core/src/repowise/core/generation/onboarding/signals.py
new file mode 100644
index 000000000..a6aff8e0b
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/signals.py
@@ -0,0 +1,41 @@
+"""Shared signal bundle passed to onboarding subkind builders.
+
+Each subkind reads only what it needs from this object. Keeping all signals
+in one typed bundle means subkinds compose easily — adding a new subkind
+doesn't require new plumbing in :func:`PageGenerator.generate_all`.
+
+Signals are read-only snapshots assembled at the start of level 8; subkind
+builders must not mutate them.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from repowise.core.ingestion.models import ParsedFile, RepoStructure
+
+
+@dataclass(frozen=True)
+class OnboardingSignals:
+ """Inputs available to every onboarding subkind context builder.
+
+ Attributes mirror the data that :func:`PageGenerator.generate_all`
+ already computes for earlier levels — no new ingestion is required.
+ """
+
+ repo_name: str
+ repo_structure: RepoStructure
+ parsed_files: tuple[ParsedFile, ...]
+ source_map: dict[str, bytes]
+ graph_builder: Any
+ pagerank: dict[str, float]
+ betweenness: dict[str, float]
+ community: dict[str, int]
+ sccs: tuple[Any, ...]
+ git_meta_map: dict[str, dict] | None = None
+ dead_code_by_file: dict[str, list[dict]] = field(default_factory=dict)
+ decisions_all: tuple[dict, ...] = ()
+ external_systems: tuple[dict, ...] = ()
+ # Summaries of pages already generated at earlier levels (target_path → blurb).
+ completed_page_summaries: dict[str, str] = field(default_factory=dict)
diff --git a/packages/core/src/repowise/core/generation/onboarding/slots.py b/packages/core/src/repowise/core/generation/onboarding/slots.py
new file mode 100644
index 000000000..7034f1b7f
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/slots.py
@@ -0,0 +1,67 @@
+"""Onboarding slot identifiers and reading order.
+
+The Onboarding collection is a fixed set of eight curated pages. Two slots are
+"promoted" — they reuse the existing `repo_overview` and `architecture_diagram`
+pages, tagged via ``metadata.onboarding_slot``. The other six are new pages
+with ``page_type='onboarding'`` and a ``metadata.subkind`` discriminator.
+
+Slot identifiers are used three ways:
+ - as the ``metadata.subkind`` value on generated pages,
+ - as the trailing path of ``target_path = f"onboarding/{slot}"``,
+ - as the ordering key in the web UI's Onboarding folder.
+
+Keep the order list in lockstep with ``packages/ui/src/lib/page-types.ts``
+``ONBOARDING_ORDER``.
+"""
+
+from __future__ import annotations
+
+# ---- Slot identifiers ----
+
+SLOT_PROJECT_OVERVIEW = "project_overview"
+SLOT_ARCHITECTURE_GUIDE = "architecture_guide"
+SLOT_GETTING_STARTED = "getting_started"
+SLOT_CODEBASE_MAP = "codebase_map"
+SLOT_KEY_CONCEPTS = "key_concepts"
+SLOT_HOW_IT_WORKS = "how_it_works"
+SLOT_DEVELOPMENT_GUIDE = "development_guide"
+SLOT_ACTIVE_LANDSCAPE = "active_landscape"
+
+# Fixed reading order. Slots not yet implemented are silently skipped at
+# generation time and absent from the UI tree.
+ONBOARDING_ORDER: tuple[str, ...] = (
+ SLOT_PROJECT_OVERVIEW,
+ SLOT_ARCHITECTURE_GUIDE,
+ SLOT_GETTING_STARTED,
+ SLOT_CODEBASE_MAP,
+ SLOT_KEY_CONCEPTS,
+ SLOT_HOW_IT_WORKS,
+ SLOT_DEVELOPMENT_GUIDE,
+ SLOT_ACTIVE_LANDSCAPE,
+)
+
+# Maps existing page_type → onboarding slot. The generator tags these pages
+# with ``metadata.onboarding_slot`` after they're produced at level 6; no
+# extra content is generated for promoted slots.
+PROMOTED_SLOTS: dict[str, str] = {
+ "repo_overview": SLOT_PROJECT_OVERVIEW,
+ "architecture_diagram": SLOT_ARCHITECTURE_GUIDE,
+}
+
+# Human-readable titles used both server-side (page title) and as a fallback
+# label in the UI when a page hasn't been hydrated yet.
+SLOT_TITLES: dict[str, str] = {
+ SLOT_PROJECT_OVERVIEW: "Project Overview",
+ SLOT_ARCHITECTURE_GUIDE: "Architecture Guide",
+ SLOT_GETTING_STARTED: "Getting Started",
+ SLOT_CODEBASE_MAP: "Codebase Map",
+ SLOT_KEY_CONCEPTS: "Key Concepts",
+ SLOT_HOW_IT_WORKS: "How It Works",
+ SLOT_DEVELOPMENT_GUIDE: "Development Guide",
+ SLOT_ACTIVE_LANDSCAPE: "Active Landscape",
+}
+
+
+def target_path(slot: str) -> str:
+ """Canonical wiki ``target_path`` for an onboarding slot."""
+ return f"onboarding/{slot}"
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py
new file mode 100644
index 000000000..a611a755e
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py
@@ -0,0 +1,17 @@
+"""Onboarding subkind modules.
+
+Importing this package triggers registration of each known subkind into the
+shared registry (see ``onboarding.registry``). Subkinds register themselves
+at import time via a module-level ``register(...)`` call.
+
+The import order here is the canonical declaration order; the registry
+re-orders to ``ONBOARDING_ORDER`` for iteration.
+"""
+
+# Side-effect imports — each module calls register() at import time.
+from . import codebase_map # noqa: F401
+from . import getting_started # noqa: F401
+from . import key_concepts # noqa: F401
+from . import how_it_works # noqa: F401
+from . import development_guide # noqa: F401
+from . import active_landscape # noqa: F401
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/active_landscape.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/active_landscape.py
new file mode 100644
index 000000000..14919b048
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/active_landscape.py
@@ -0,0 +1,156 @@
+"""Onboarding subkind: Active Landscape.
+
+Where work is actually happening right now. The page that says
+"don't refactor auth/ this week, three people are in there." Driven by
+git churn signals — hot files, hot directories, dead-code findings in
+hot areas, and stable counter-balance.
+
+Gate: ≥ 50 commits in the last 90 days across the indexed corpus AND
+≥ 10 distinct files touched. Without that floor the page reads like
+nervous twitching about a fresh repo.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import PurePosixPath
+
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_ACTIVE_LANDSCAPE, SLOT_TITLES
+
+_GATE_MIN_COMMITS_90D = 50
+_GATE_MIN_FILES_TOUCHED = 10
+_TOP_HOT_FILES = 12
+_TOP_HOT_DIRS = 6
+
+
+@dataclass
+class HotFile:
+ path: str
+ commit_count_90d: int
+ primary_owner: str = ""
+ is_hotspot: bool = False
+ age_days: int = 0
+
+
+@dataclass
+class HotDir:
+ path: str
+ total_commits_90d: int
+ hotspot_count: int
+ file_count: int
+
+
+@dataclass
+class ActiveLandscapeContext:
+ repo_name: str
+ total_commits_90d: int
+ files_touched_90d: int
+ hot_files: list[HotFile] = field(default_factory=list)
+ hot_dirs: list[HotDir] = field(default_factory=list)
+ # Dead-code findings localized to currently-hot files. Often the most
+ # actionable callout a reader can get on day one.
+ dead_code_in_hot_files: list[dict] = field(default_factory=list)
+ stable_file_count: int = 0
+
+
+def _top_level_dir(path: str) -> str:
+ parts = PurePosixPath(path).parts
+ return parts[0] if len(parts) > 1 else "(root)"
+
+
+def _build(signals: OnboardingSignals) -> ActiveLandscapeContext | None:
+ git_meta_map = signals.git_meta_map or {}
+ if not git_meta_map:
+ return None
+
+ # Aggregate gate metrics in a single pass.
+ total_commits_90d = 0
+ files_touched = 0
+ for meta in git_meta_map.values():
+ commits = int(meta.get("commit_count_90d", 0) or 0)
+ if commits > 0:
+ files_touched += 1
+ total_commits_90d += commits
+
+ if (
+ total_commits_90d < _GATE_MIN_COMMITS_90D
+ or files_touched < _GATE_MIN_FILES_TOUCHED
+ ):
+ return None
+
+ # Build hot files — sorted by 90d churn, then commit recency proxy (age).
+ hot_files_all = [
+ HotFile(
+ path=path,
+ commit_count_90d=int(meta.get("commit_count_90d", 0) or 0),
+ primary_owner=str(meta.get("primary_owner_name", "") or ""),
+ is_hotspot=bool(meta.get("is_hotspot", False)),
+ age_days=int(meta.get("age_days", 0) or 0),
+ )
+ for path, meta in git_meta_map.items()
+ if int(meta.get("commit_count_90d", 0) or 0) > 0
+ ]
+ hot_files_all.sort(
+ key=lambda h: (h.commit_count_90d, -h.age_days),
+ reverse=True,
+ )
+ hot_files = hot_files_all[:_TOP_HOT_FILES]
+
+ # Roll up to directories.
+ dir_commits: dict[str, int] = defaultdict(int)
+ dir_hotspots: dict[str, int] = defaultdict(int)
+ dir_files: dict[str, int] = defaultdict(int)
+ for hf in hot_files_all:
+ d = _top_level_dir(hf.path)
+ dir_commits[d] += hf.commit_count_90d
+ dir_files[d] += 1
+ if hf.is_hotspot:
+ dir_hotspots[d] += 1
+ hot_dirs = sorted(
+ (
+ HotDir(
+ path=d,
+ total_commits_90d=dir_commits[d],
+ hotspot_count=dir_hotspots[d],
+ file_count=dir_files[d],
+ )
+ for d in dir_commits
+ ),
+ key=lambda d: d.total_commits_90d,
+ reverse=True,
+ )[:_TOP_HOT_DIRS]
+
+ # Dead-code findings inside the top hot files only — keeps the prompt
+ # focused on actionable, recent overlap rather than the full report.
+ hot_paths = {h.path for h in hot_files}
+ dead_code_in_hot: list[dict] = []
+ for path in hot_paths:
+ for finding in signals.dead_code_by_file.get(path, []):
+ dead_code_in_hot.append({"file_path": path, **finding})
+
+ stable_file_count = sum(
+ 1 for meta in git_meta_map.values() if bool(meta.get("is_stable", False))
+ )
+
+ return ActiveLandscapeContext(
+ repo_name=signals.repo_name,
+ total_commits_90d=total_commits_90d,
+ files_touched_90d=files_touched,
+ hot_files=hot_files,
+ hot_dirs=hot_dirs,
+ dead_code_in_hot_files=dead_code_in_hot[:15],
+ stable_file_count=stable_file_count,
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_ACTIVE_LANDSCAPE,
+ title=SLOT_TITLES[SLOT_ACTIVE_LANDSCAPE],
+ template="active_landscape.j2",
+ build_context=_build,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/codebase_map.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/codebase_map.py
new file mode 100644
index 000000000..2a1a57752
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/codebase_map.py
@@ -0,0 +1,140 @@
+"""Onboarding subkind: Codebase Map.
+
+The wayfinding page — where things physically live. Different from the
+Architecture Guide (which is conceptual) because this is spatial: it
+walks the file tree top-down and tells the reader "if you're looking
+for X, go to Y."
+
+Gate: always generates. Every repo has a directory structure.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from pathlib import PurePosixPath
+
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_CODEBASE_MAP, SLOT_TITLES
+
+# Tunables — kept inline because they're specific to this subkind only.
+_MIN_FILES_PER_DIR = 3
+_TOP_FILES_PER_DIR = 5
+_MAX_DIRS = 18
+_MAX_COMMUNITY_LABELS_PER_DIR = 3
+
+
+@dataclass
+class DirectorySummary:
+ """One top-level directory's snapshot in the Codebase Map."""
+
+ path: str
+ file_count: int
+ dominant_language: str
+ entry_points: list[str] = field(default_factory=list)
+ top_files: list[str] = field(default_factory=list)
+ community_labels: list[str] = field(default_factory=list)
+
+
+@dataclass
+class CodebaseMapContext:
+ repo_name: str
+ total_files: int
+ total_loc: int
+ directories: list[DirectorySummary] = field(default_factory=list)
+ # Repo-wide entry points hoisted to a dedicated section.
+ entry_points: list[str] = field(default_factory=list)
+
+
+def _top_level_dir(path: str) -> str:
+ """Return the first path segment, or "(root)" for top-level files."""
+ parts = PurePosixPath(path).parts
+ return parts[0] if len(parts) > 1 else "(root)"
+
+
+def _build(signals: OnboardingSignals) -> CodebaseMapContext | None:
+ """Codebase Map always builds — there's no gate."""
+ # Group parsed files by top-level directory.
+ by_dir: dict[str, list] = defaultdict(list)
+ for pf in signals.parsed_files:
+ by_dir[_top_level_dir(pf.file_info.path)].append(pf)
+
+ # Resolve community labels once, then map path → label.
+ label_by_cid: dict[int, str] = {}
+ try:
+ info = signals.graph_builder.community_info() or {}
+ for cid, ci in info.items() if hasattr(info, "items") else ():
+ label = getattr(ci, "label", "") or ""
+ if label:
+ label_by_cid[int(cid)] = label
+ except Exception:
+ # community_info is best-effort enrichment; the page renders fine
+ # without it.
+ pass
+
+ directories: list[DirectorySummary] = []
+ for dir_name, files in by_dir.items():
+ if len(files) < _MIN_FILES_PER_DIR and dir_name != "(root)":
+ continue
+
+ # Dominant language: most-common language by file count.
+ lang_counts: Counter[str] = Counter(f.file_info.language for f in files)
+ dominant_language = lang_counts.most_common(1)[0][0] if lang_counts else "unknown"
+
+ # Entry points within the directory (cap at 3 — overflow goes in the
+ # full top_files list).
+ entry_points = sorted(
+ (f.file_info.path for f in files if f.file_info.is_entry_point)
+ )[:3]
+
+ # Top files by PageRank within the directory (skip generated/clones).
+ ranked = sorted(
+ ((signals.pagerank.get(f.file_info.path, 0.0), f.file_info.path) for f in files),
+ reverse=True,
+ )
+ top_files = [path for _, path in ranked[:_TOP_FILES_PER_DIR]]
+
+ # Distinct community labels touching files in this dir.
+ cids_here = {
+ signals.community.get(f.file_info.path)
+ for f in files
+ if signals.community.get(f.file_info.path) is not None
+ }
+ labels = sorted(
+ {label_by_cid[cid] for cid in cids_here if cid in label_by_cid}
+ )[:_MAX_COMMUNITY_LABELS_PER_DIR]
+
+ directories.append(
+ DirectorySummary(
+ path=dir_name,
+ file_count=len(files),
+ dominant_language=dominant_language,
+ entry_points=entry_points,
+ top_files=top_files,
+ community_labels=labels,
+ )
+ )
+
+ # Largest directories first — the reader sees the load-bearing dirs
+ # before utility ones.
+ directories.sort(key=lambda d: d.file_count, reverse=True)
+ directories = directories[:_MAX_DIRS]
+
+ return CodebaseMapContext(
+ repo_name=signals.repo_name,
+ total_files=getattr(signals.repo_structure, "total_files", len(signals.parsed_files)),
+ total_loc=getattr(signals.repo_structure, "total_loc", 0),
+ directories=directories,
+ entry_points=list(getattr(signals.repo_structure, "entry_points", []))[:10],
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_CODEBASE_MAP,
+ title=SLOT_TITLES[SLOT_CODEBASE_MAP],
+ template="codebase_map.j2",
+ build_context=_build,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/development_guide.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/development_guide.py
new file mode 100644
index 000000000..27a317950
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/development_guide.py
@@ -0,0 +1,205 @@
+"""Onboarding subkind: Development Guide.
+
+How to do common things in this codebase — add a feature, write a test,
+follow naming conventions — derived from structural repetition that's
+already visible in the file tree.
+
+Gate: at least 2 detectable repeated patterns (e.g., a strong "test
+mirror" plus a recurring filename suffix, or two distinct filename
+suffix groups). Without that signal, the guide would be inventing
+conventions rather than reporting them.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import PurePosixPath
+
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_DEVELOPMENT_GUIDE, SLOT_TITLES
+
+_GATE_MIN_SIGNALS = 2
+_MIN_SUFFIX_GROUP = 3
+_TOP_SUFFIX_GROUPS = 6
+_MIN_TEST_MIRROR_RATIO = 0.5
+
+# Recognised test-tree roots — anything else with "test" in the name still
+# counts but is treated as a weaker mirror signal.
+_TEST_ROOTS: tuple[str, ...] = (
+ "tests/", "test/", "__tests__/", "spec/", "specs/",
+)
+
+
+@dataclass
+class SuffixPattern:
+ """A repeating filename suffix — e.g. `_handler.py` x 7."""
+
+ suffix: str
+ examples: list[str] = field(default_factory=list)
+ file_count: int = 0
+
+
+@dataclass
+class TestMirror:
+ test_root: str
+ matched_files: int
+ source_files: int
+
+
+@dataclass
+class DevelopmentGuideContext:
+ repo_name: str
+ suffix_patterns: list[SuffixPattern] = field(default_factory=list)
+ test_mirror: TestMirror | None = None
+ parallel_dirs: list[str] = field(default_factory=list)
+ entry_points: list[str] = field(default_factory=list)
+
+
+def _split_basename(name: str) -> list[str]:
+ """Split a basename into trailing tokens for suffix grouping.
+
+ Handles snake_case and camelCase, stripping the file extension.
+ Example: `UserHandlerService.cs` → ['user', 'handler', 'service']
+ `api_route_handler.py` → ['api', 'route', 'handler']
+ """
+ stem = PurePosixPath(name).stem
+ # CamelCase → space, then lower + split on underscore/hyphen/space.
+ spaced = re.sub(r"(?<!^)(?=[A-Z])", " ", stem).lower()
+ tokens = re.split(r"[\s_\-]+", spaced)
+ return [t for t in tokens if t]
+
+
+def _find_suffix_patterns(signals: OnboardingSignals) -> list[SuffixPattern]:
+ by_suffix: dict[str, list[str]] = defaultdict(list)
+ for pf in signals.parsed_files:
+ path = pf.file_info.path
+ # Skip test files — patterns inside tests/ are the mirror signal,
+ # not the suffix signal.
+ if any(seg in path for seg in _TEST_ROOTS):
+ continue
+ tokens = _split_basename(PurePosixPath(path).name)
+ if len(tokens) < 2:
+ continue
+ suffix = tokens[-1]
+ # Filter out generic single-letter / stop-word suffixes.
+ if len(suffix) <= 2 or suffix in {"py", "ts", "js", "cs", "go", "rs"}:
+ continue
+ by_suffix[suffix].append(path)
+
+ groups: list[SuffixPattern] = []
+ for suffix, paths in by_suffix.items():
+ if len(paths) < _MIN_SUFFIX_GROUP:
+ continue
+ groups.append(
+ SuffixPattern(
+ suffix=suffix,
+ file_count=len(paths),
+ examples=sorted(paths)[:3],
+ )
+ )
+ groups.sort(key=lambda g: g.file_count, reverse=True)
+ return groups[:_TOP_SUFFIX_GROUPS]
+
+
+def _find_test_mirror(signals: OnboardingSignals) -> TestMirror | None:
+ """Detect a "tests mirror source layout" convention.
+
+ For every file under a known test root, strip the root and try to
+ locate a non-test file that shares the remaining path stem. A mirror
+ is reported when ≥ 50% of test files match.
+ """
+ source_paths: set[str] = {
+ pf.file_info.path
+ for pf in signals.parsed_files
+ if not any(seg in pf.file_info.path for seg in _TEST_ROOTS)
+ }
+
+ best: TestMirror | None = None
+ for root in _TEST_ROOTS:
+ test_files = [pf.file_info.path for pf in signals.parsed_files if pf.file_info.path.startswith(root)]
+ if len(test_files) < _MIN_SUFFIX_GROUP:
+ continue
+ # Strip leading "test_" / "_test" / "spec_" from the basename and
+ # look for the resulting stem anywhere in the source tree.
+ matched = 0
+ for tf in test_files:
+ stem = PurePosixPath(tf).stem
+ stem_normalised = re.sub(r"^test_|^spec_|_test$|_spec$", "", stem, flags=re.IGNORECASE)
+ if not stem_normalised:
+ continue
+ if any(stem_normalised in PurePosixPath(p).stem for p in source_paths):
+ matched += 1
+ if test_files and matched / len(test_files) >= _MIN_TEST_MIRROR_RATIO:
+ candidate = TestMirror(
+ test_root=root.rstrip("/"),
+ matched_files=matched,
+ source_files=len(test_files),
+ )
+ if best is None or candidate.matched_files > best.matched_files:
+ best = candidate
+ return best
+
+
+def _find_parallel_dirs(signals: OnboardingSignals) -> list[str]:
+ """Find sibling directories that look structurally parallel.
+
+ A parallel layout shows up when multiple siblings under the same parent
+ each contain the same set of basenames (e.g. `services/auth/{api.py,
+ service.py, models.py}` repeated under `services/billing/…`).
+ """
+ by_parent: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
+ for pf in signals.parsed_files:
+ parts = PurePosixPath(pf.file_info.path).parts
+ if len(parts) < 3:
+ continue
+ grandparent = parts[-3]
+ parent = parts[-2]
+ by_parent[grandparent][parent].add(parts[-1])
+
+ parallel: list[str] = []
+ for grandparent, siblings in by_parent.items():
+ if len(siblings) < 3:
+ continue
+ # Look for a shared core of basenames across siblings.
+ all_names: list[set[str]] = list(siblings.values())
+ if not all_names:
+ continue
+ shared = set.intersection(*all_names)
+ if len(shared) >= 2:
+ parallel.append(grandparent)
+ return sorted(parallel)[:5]
+
+
+def _build(signals: OnboardingSignals) -> DevelopmentGuideContext | None:
+ suffix_patterns = _find_suffix_patterns(signals)
+ test_mirror = _find_test_mirror(signals)
+ parallel_dirs = _find_parallel_dirs(signals)
+
+ signal_count = (
+ (1 if suffix_patterns else 0)
+ + (1 if test_mirror else 0)
+ + (1 if parallel_dirs else 0)
+ )
+ if signal_count < _GATE_MIN_SIGNALS:
+ return None
+
+ return DevelopmentGuideContext(
+ repo_name=signals.repo_name,
+ suffix_patterns=suffix_patterns,
+ test_mirror=test_mirror,
+ parallel_dirs=parallel_dirs,
+ entry_points=list(getattr(signals.repo_structure, "entry_points", []))[:5],
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_DEVELOPMENT_GUIDE,
+ title=SLOT_TITLES[SLOT_DEVELOPMENT_GUIDE],
+ template="development_guide.j2",
+ build_context=_build,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/getting_started.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/getting_started.py
new file mode 100644
index 000000000..ed20f630a
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/getting_started.py
@@ -0,0 +1,180 @@
+"""Onboarding subkind: Getting Started.
+
+The page where reading stops and doing starts — clone, install, build,
+run, test. Purely mechanical, grounded in either a recognised manifest
+(``package.json``, ``pyproject.toml``, ``go.mod`` …) or a README section
+that already explains setup.
+
+Gate: at least one parsed manifest **or** a README at the repo root with
+a recognisable install/run/build/test heading. Skip for pure-docs or
+config-only repos with no detectable build system.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_GETTING_STARTED, SLOT_TITLES
+
+# Heading patterns we treat as "setup-relevant" when scanning a README.
+# Lowercased; matched against the trimmed line after the leading `#`s.
+_README_HEADINGS: tuple[tuple[str, str], ...] = (
+ ("install", "Install"),
+ ("installation", "Install"),
+ ("quickstart", "Quickstart"),
+ ("quick start", "Quickstart"),
+ ("getting started", "Getting Started"),
+ ("setup", "Setup"),
+ ("run", "Run"),
+ ("running", "Run"),
+ ("usage", "Usage"),
+ ("build", "Build"),
+ ("test", "Test"),
+ ("testing", "Test"),
+ ("development", "Development"),
+ ("contributing", "Contributing"),
+)
+
+_README_FILENAMES = ("README.md", "readme.md", "README.MD", "README", "Readme.md")
+_MAX_README_SECTION_CHARS = 800
+_MAX_DEPS_LISTED = 12
+
+
+@dataclass
+class ReadmeSection:
+ heading: str
+ body: str
+
+
+@dataclass
+class GettingStartedContext:
+ repo_name: str
+ package_managers: list[str] = field(default_factory=list)
+ runtime_dependencies: list[dict] = field(default_factory=list)
+ dev_dependencies: list[dict] = field(default_factory=list)
+ readme_sections: list[ReadmeSection] = field(default_factory=list)
+ entry_points: list[str] = field(default_factory=list)
+
+
+def _find_readme(source_map: dict[str, bytes]) -> bytes | None:
+ """Return the first repo-root README we can find, or None."""
+ for name in _README_FILENAMES:
+ data = source_map.get(name)
+ if data:
+ return data
+ # Fallback: any path whose basename matches and lives at repo root.
+ for path, data in source_map.items():
+ if "/" not in path and path.lower() in ("readme", "readme.md", "readme.txt"):
+ return data
+ return None
+
+
+def _extract_setup_sections(readme: bytes) -> list[ReadmeSection]:
+ """Pull setup-relevant sections out of a README body.
+
+ Stops each section at the next heading of any level, then truncates to
+ keep the prompt budget small.
+ """
+ try:
+ text = readme.decode("utf-8", errors="replace")
+ except Exception:
+ return []
+
+ lines = text.splitlines()
+ sections: list[ReadmeSection] = []
+ i = 0
+ while i < len(lines):
+ line = lines[i]
+ heading_match = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
+ if not heading_match:
+ i += 1
+ continue
+ heading_text = heading_match.group(2).strip().lower()
+ # Match against any of our known setup headings.
+ canonical: str | None = None
+ for pattern, label in _README_HEADINGS:
+ if pattern == heading_text or heading_text.startswith(pattern + " "):
+ canonical = label
+ break
+ if canonical is None:
+ i += 1
+ continue
+ # Collect body until the next heading of any depth.
+ body_lines: list[str] = []
+ j = i + 1
+ while j < len(lines) and not re.match(r"^#{1,6}\s+\S", lines[j]):
+ body_lines.append(lines[j])
+ j += 1
+ body = "\n".join(body_lines).strip()
+ if body:
+ sections.append(
+ ReadmeSection(heading=canonical, body=body[:_MAX_README_SECTION_CHARS])
+ )
+ i = j
+ return sections
+
+
+def _partition_dependencies(
+ external_systems: tuple[dict, ...],
+) -> tuple[list[str], list[dict], list[dict]]:
+ """Split external_systems into (package_managers, runtime_deps, dev_deps).
+
+ The manifest parsers already record ``ecosystem`` (npm/pypi/cargo/…)
+ and ``is_dev``. Anything we can't classify falls into runtime.
+ """
+ package_managers: list[str] = []
+ runtime: list[dict] = []
+ dev: list[dict] = []
+ seen_ecosystems: set[str] = set()
+
+ for sys in external_systems:
+ eco = str(sys.get("ecosystem", "") or "").strip()
+ if eco and eco not in seen_ecosystems:
+ package_managers.append(eco)
+ seen_ecosystems.add(eco)
+ entry = {
+ "name": sys.get("name", ""),
+ "ecosystem": eco,
+ "category": sys.get("category", "library"),
+ }
+ if sys.get("is_dev"):
+ dev.append(entry)
+ else:
+ runtime.append(entry)
+
+ return package_managers, runtime[:_MAX_DEPS_LISTED], dev[:_MAX_DEPS_LISTED]
+
+
+def _build(signals: OnboardingSignals) -> GettingStartedContext | None:
+ package_managers, runtime, dev = _partition_dependencies(signals.external_systems)
+ readme_sections: list[ReadmeSection] = []
+ readme_bytes = _find_readme(signals.source_map)
+ if readme_bytes is not None:
+ readme_sections = _extract_setup_sections(readme_bytes)
+
+ # Gate: need at least one signal source. Without a manifest *and*
+ # without a README setup section, the page would be all speculation.
+ if not package_managers and not readme_sections:
+ return None
+
+ return GettingStartedContext(
+ repo_name=signals.repo_name,
+ package_managers=package_managers,
+ runtime_dependencies=runtime,
+ dev_dependencies=dev,
+ readme_sections=readme_sections,
+ entry_points=list(getattr(signals.repo_structure, "entry_points", []))[:6],
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_GETTING_STARTED,
+ title=SLOT_TITLES[SLOT_GETTING_STARTED],
+ template="getting_started.j2",
+ build_context=_build,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/how_it_works.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/how_it_works.py
new file mode 100644
index 000000000..a31ba0ab5
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/how_it_works.py
@@ -0,0 +1,165 @@
+"""Onboarding subkind: How It Works.
+
+The page that turns the static architecture into a movie — one end-to-end
+trace through the system. The content adapts to repo archetype:
+
+- service / app → "when a request hits /api/X, here's what happens"
+- library → "when a consumer calls the public API, here's what runs"
+- CLI tool → "when you run `tool build`, here's what happens"
+- pipeline → "input becomes output, phase by phase"
+- module → generic fallback for utility collections
+
+Gate: at least one execution flow with ≥ 3 hops *or* a detectable
+non-trivial archetype. Skip flat utility-collection repos with no clear
+flow.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_HOW_IT_WORKS, SLOT_TITLES
+
+Archetype = Literal["service", "cli", "library", "pipeline", "module"]
+
+_MIN_TRACE_HOPS = 3
+_TOP_FLOWS = 3
+_TRACE_DISPLAY_HOPS = 8
+
+# Framework hints by ecosystem — anything matching tilts the archetype.
+_SERVICE_FRAMEWORK_HINTS: frozenset[str] = frozenset({
+ "fastapi", "flask", "starlette", "django", "uvicorn", "gunicorn",
+ "express", "koa", "hapi", "nestjs",
+ "Microsoft.AspNetCore", "Microsoft.AspNetCore.App",
+ "actix-web", "axum", "rocket",
+ "gin", "echo", "fiber",
+ "spring-boot", "spring-webmvc",
+})
+_CLI_FRAMEWORK_HINTS: frozenset[str] = frozenset({
+ "click", "typer", "argparse", "fire",
+ "commander", "yargs", "oclif",
+ "cobra", "urfave/cli",
+ "clap", "structopt",
+})
+_PIPELINE_HINTS: frozenset[str] = frozenset({
+ "celery", "airflow", "prefect", "dagster",
+ "apache-beam", "kafka", "rabbitmq",
+})
+
+
+@dataclass
+class FlowTrace:
+ entry_point: str
+ hops: list[str] = field(default_factory=list)
+ score: float = 0.0
+
+
+@dataclass
+class HowItWorksContext:
+ repo_name: str
+ archetype: Archetype
+ archetype_evidence: list[str] = field(default_factory=list)
+ flows: list[FlowTrace] = field(default_factory=list)
+ entry_points: list[str] = field(default_factory=list)
+
+
+def _classify_archetype(signals: OnboardingSignals) -> tuple[Archetype, list[str]]:
+ """Return the most-likely archetype plus the evidence backing it.
+
+ Heuristic, not authoritative — the LLM is told to treat this as a hint.
+ """
+ evidence: list[str] = []
+ dep_names = {str(s.get("name", "")) for s in signals.external_systems}
+
+ service_hits = dep_names & _SERVICE_FRAMEWORK_HINTS
+ cli_hits = dep_names & _CLI_FRAMEWORK_HINTS
+ pipeline_hits = dep_names & _PIPELINE_HINTS
+
+ api_contract_count = sum(
+ 1 for pf in signals.parsed_files if pf.file_info.is_api_contract
+ )
+
+ # Service tilt wins if we have either framework deps or API contract files.
+ if service_hits or api_contract_count > 0:
+ if service_hits:
+ evidence.append(f"service framework(s) detected: {', '.join(sorted(service_hits))}")
+ if api_contract_count:
+ evidence.append(f"{api_contract_count} API contract file(s) (OpenAPI/proto/GraphQL)")
+ return "service", evidence
+
+ if cli_hits:
+ evidence.append(f"CLI framework(s) detected: {', '.join(sorted(cli_hits))}")
+ return "cli", evidence
+
+ if pipeline_hits:
+ evidence.append(f"pipeline framework(s) detected: {', '.join(sorted(pipeline_hits))}")
+ return "pipeline", evidence
+
+ # Entry-point shape — `__main__.py` or `bin/` is CLI-shaped.
+ entry_points = list(getattr(signals.repo_structure, "entry_points", []))
+ if any(ep.endswith("__main__.py") or "/bin/" in ep or ep.startswith("bin/") for ep in entry_points):
+ evidence.append("entry point shape suggests a CLI (__main__ or bin/)")
+ return "cli", evidence
+
+ # If packages have a clear public API but no service / CLI signal, it's a library.
+ packages = getattr(signals.repo_structure, "packages", []) or []
+ if packages:
+ evidence.append(f"{len(packages)} package(s) with declared public surface")
+ return "library", evidence
+
+ evidence.append("no service / CLI / library signal — treating as module collection")
+ return "module", evidence
+
+
+def _collect_flows(signals: OnboardingSignals) -> list[FlowTrace]:
+ """Pull execution flows from the graph builder (best-effort)."""
+ try:
+ report = signals.graph_builder.execution_flows()
+ except Exception:
+ return []
+ if not report or not hasattr(report, "flows"):
+ return []
+
+ flows: list[FlowTrace] = []
+ for flow in getattr(report, "flows", [])[:_TOP_FLOWS]:
+ trace = list(getattr(flow, "trace", []) or [])
+ if len(trace) < _MIN_TRACE_HOPS:
+ continue
+ flows.append(
+ FlowTrace(
+ entry_point=str(getattr(flow, "entry_point", "")),
+ hops=trace[:_TRACE_DISPLAY_HOPS],
+ score=float(getattr(flow, "score", 0.0) or 0.0),
+ )
+ )
+ return flows
+
+
+def _build(signals: OnboardingSignals) -> HowItWorksContext | None:
+ archetype, evidence = _classify_archetype(signals)
+ flows = _collect_flows(signals)
+
+ # Gate: need either a real trace or a non-trivial archetype.
+ if not flows and archetype == "module":
+ return None
+
+ return HowItWorksContext(
+ repo_name=signals.repo_name,
+ archetype=archetype,
+ archetype_evidence=evidence,
+ flows=flows,
+ entry_points=list(getattr(signals.repo_structure, "entry_points", []))[:5],
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_HOW_IT_WORKS,
+ title=SLOT_TITLES[SLOT_HOW_IT_WORKS],
+ template="how_it_works.j2",
+ build_context=_build,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py
new file mode 100644
index 000000000..2a5b1f6f6
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py
@@ -0,0 +1,177 @@
+"""Onboarding subkind: Key Concepts.
+
+The abstractions and vocabulary this codebase uses — the mental model
+needed to read the code. Not a glossary dump but a narrative that
+identifies 4–6 load-bearing types/functions and the architectural
+clusters they belong to.
+
+Gate: at least 4 public symbols clear the PageRank P90 threshold. Below
+that, the page would be guessing at "core concepts."
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_KEY_CONCEPTS, SLOT_TITLES
+
+_GATE_MIN_CONCEPTS = 4
+_PAGERANK_PERCENTILE = 0.90 # top 10% of file PageRank scores
+_TOP_SYMBOLS = 10
+_MAX_DECISION_RECORDS = 6
+_MAX_COMMUNITY_LABELS = 6
+
+
+@dataclass
+class ConceptSymbol:
+ """A candidate "core concept" symbol with the context needed to write
+ about it without re-reading source."""
+
+ name: str
+ kind: str
+ file_path: str
+ docstring: str = ""
+ pagerank: float = 0.0
+ community_label: str = ""
+
+
+@dataclass
+class KeyConceptsContext:
+ repo_name: str
+ concept_symbols: list[ConceptSymbol] = field(default_factory=list)
+ community_labels: list[str] = field(default_factory=list)
+ decision_titles: list[str] = field(default_factory=list)
+
+
+def _pagerank_threshold(pagerank: dict[str, float]) -> float:
+ """Return the file-level PageRank cutoff at the configured percentile.
+
+ The percentile is tightened to whichever yields the *more permissive*
+ cutoff between the configured P90 and "top N files" (where N is the
+ gate minimum). On a 5-file fixture pure P90 would only admit one
+ file — too tight to express the gate's intent.
+ """
+ scores = sorted(pagerank.values(), reverse=True)
+ if not scores:
+ return 0.0
+ idx_percentile = int(len(scores) * (1.0 - _PAGERANK_PERCENTILE))
+ # Allow at least the top N files in (N-1 is the highest index that
+ # should still pass the threshold). Higher index = lower score =
+ # more permissive.
+ idx_min_admission = _GATE_MIN_CONCEPTS - 1
+ idx = max(idx_percentile, idx_min_admission)
+ return scores[min(idx, len(scores) - 1)]
+
+
+def _resolve_community_labels(graph_builder: Any) -> dict[int, str]:
+ labels: dict[int, str] = {}
+ try:
+ info = graph_builder.community_info() or {}
+ items = info.items() if hasattr(info, "items") else ()
+ for cid, ci in items:
+ label = getattr(ci, "label", "") or ""
+ if label:
+ labels[int(cid)] = label
+ except Exception:
+ pass
+ return labels
+
+
+def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:
+ threshold = _pagerank_threshold(signals.pagerank)
+ if threshold <= 0.0:
+ return None
+
+ labels_by_cid = _resolve_community_labels(signals.graph_builder)
+
+ # Score: file PageRank lifts every public symbol it contains. Symbol
+ # frequency (defined more than once) acts as a tiebreaker, modelling
+ # the heuristic that re-implemented concepts tend to be foundational.
+ name_freq: Counter[str] = Counter()
+ for pf in signals.parsed_files:
+ for sym in pf.symbols:
+ if getattr(sym, "visibility", "public") == "public":
+ name_freq[sym.name] += 1
+
+ candidates: list[ConceptSymbol] = []
+ for pf in signals.parsed_files:
+ path = pf.file_info.path
+ pr = signals.pagerank.get(path, 0.0)
+ if pr < threshold:
+ continue
+ cid = signals.community.get(path)
+ community_label = labels_by_cid.get(int(cid)) if cid is not None else ""
+ for sym in pf.symbols:
+ if getattr(sym, "visibility", "public") != "public":
+ continue
+ # Skip near-noise: tiny utility functions and lowercase-only names
+ # that almost never represent real concepts.
+ if len(sym.name) <= 2:
+ continue
+ candidates.append(
+ ConceptSymbol(
+ name=sym.name,
+ kind=str(getattr(sym, "kind", "symbol")),
+ file_path=path,
+ docstring=(getattr(sym, "docstring", "") or "").strip()[:300],
+ pagerank=pr,
+ community_label=community_label or "",
+ )
+ )
+
+ if len(candidates) < _GATE_MIN_CONCEPTS:
+ return None
+
+ # Rank by (pagerank, name_frequency, has_docstring) — docstrings often
+ # mark the deliberately-public surface.
+ candidates.sort(
+ key=lambda c: (
+ c.pagerank,
+ name_freq.get(c.name, 0),
+ 1 if c.docstring else 0,
+ ),
+ reverse=True,
+ )
+
+ # De-duplicate by symbol name to avoid the same concept appearing
+ # multiple times when re-exported from several files.
+ seen: set[str] = set()
+ concept_symbols: list[ConceptSymbol] = []
+ for c in candidates:
+ if c.name in seen:
+ continue
+ seen.add(c.name)
+ concept_symbols.append(c)
+ if len(concept_symbols) >= _TOP_SYMBOLS:
+ break
+
+ if len(concept_symbols) < _GATE_MIN_CONCEPTS:
+ return None
+
+ community_labels = sorted(set(labels_by_cid.values()))[:_MAX_COMMUNITY_LABELS]
+ decision_titles = [
+ str(d.get("title", "")).strip()
+ for d in signals.decisions_all[:_MAX_DECISION_RECORDS]
+ if d.get("title")
+ ]
+
+ return KeyConceptsContext(
+ repo_name=signals.repo_name,
+ concept_symbols=concept_symbols,
+ community_labels=community_labels,
+ decision_titles=decision_titles,
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_KEY_CONCEPTS,
+ title=SLOT_TITLES[SLOT_KEY_CONCEPTS],
+ template="key_concepts.j2",
+ build_context=_build,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/page_generator.py b/packages/core/src/repowise/core/generation/page_generator.py
index 9d97c47d8..6bfb8683d 100644
--- a/packages/core/src/repowise/core/generation/page_generator.py
+++ b/packages/core/src/repowise/core/generation/page_generator.py
@@ -18,6 +18,7 @@ import hashlib
import re
import uuid
from collections.abc import Callable
+from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -27,8 +28,9 @@ import structlog
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
from repowise.core.ingestion.models import ParsedFile, RepoStructure
-from repowise.core.providers.llm.base import BaseProvider, GeneratedResponse
+from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse
+from . import onboarding as _onboarding
from .context_assembler import ContextAssembler, FilePageContext
from .models import (
GENERATION_LEVELS,
@@ -40,6 +42,23 @@ from .models import (
log = structlog.get_logger(__name__)
+
+@dataclass(frozen=True)
+class PriorPage:
+ """Snapshot of a previously-generated page used for cross-run reuse.
+
+ Lives in :class:`PageGenerator` keyed by ``page_id``. When the freshly
+ rendered prompt produces a matching ``source_hash`` under the same
+ ``model_name``, the LLM call is skipped and ``content`` is reused.
+ """
+
+ source_hash: str
+ model_name: str
+ content: str
+ input_tokens: int = 0
+ output_tokens: int = 0
+ cached_tokens: int = 0
+
_LANGUAGE_NAMES = {
"en": "English",
"ru": "Russian",
@@ -112,17 +131,13 @@ SYSTEM_PROMPTS: dict[str, str] = {
"Output markdown only. "
"Required sections: ## Purpose, ## Key Targets/Stages, ## Configuration, ## Operational Notes."
),
- "diff_summary": (
- "You are repowise, an expert technical documentation generator. "
- "Summarise the changes between two git refs and their documentation impact. "
- "Output markdown only. "
- "Required sections: ## Summary, ## Changed Files, ## Symbol Changes, ## Affected Documentation."
- ),
- "cross_package": (
- "You are repowise, an expert technical documentation generator. "
- "Document the cross-package boundary and integration points. "
- "Output markdown only. "
- "Required sections: ## Overview, ## Package Boundaries, ## Integration Points."
+ "onboarding": (
+ "You are repowise, an expert technical documentation generator producing "
+ "a single page in a curated Onboarding collection that a new contributor "
+ "or LLM agent reads first. "
+ "Write concise, navigable prose grounded in the structured signals supplied. "
+ "Do not invent file paths, symbol names, or rationale that is not in the context. "
+ "Output markdown only — follow the exact section structure the user prompt prescribes."
),
}
@@ -155,6 +170,7 @@ class PageGenerator:
jinja_env: jinja2.Environment | None = None,
vector_store: Any | None = None,
language: str = "en",
+ prior_pages: dict[str, "PriorPage"] | None = None,
) -> None:
self._provider = provider
self._assembler = assembler
@@ -162,6 +178,13 @@ class PageGenerator:
self._vector_store = vector_store
self._language = language
self._cache: dict[str, GeneratedResponse] = {}
+ # Map of page_id → PriorPage from previous generation runs. When the
+ # rendered prompt's source_hash matches the prior page's hash AND the
+ # model is the same, the LLM call is skipped and the prior content is
+ # reused. Wired by the orchestrator from the persisted wiki_pages
+ # table.
+ self._prior_pages: dict[str, PriorPage] = prior_pages or {}
+ self._reuse_count: int = 0
if jinja_env is None:
templates_dir = Path(__file__).parent / "templates"
@@ -190,7 +213,9 @@ class PageGenerator:
parsed, graph, pagerank, betweenness, community, source_bytes
)
user_prompt = self._render("file_page.j2", ctx=ctx)
- response = await self._call_provider("file_page", user_prompt, str(uuid.uuid4()))
+ response = await self._call_provider(
+ "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
return self._build_generated_page(
"file_page",
parsed.file_info.path,
@@ -216,7 +241,10 @@ class PageGenerator:
source_bytes=(source_map or {}).get(parsed.file_info.path, b""),
)
user_prompt = self._render("symbol_spotlight.j2", ctx=ctx)
- response = await self._call_provider("symbol_spotlight", user_prompt, str(uuid.uuid4()))
+ target = f"{parsed.file_info.path}::{symbol.name}"
+ response = await self._call_provider(
+ "symbol_spotlight", user_prompt, str(uuid.uuid4()), target_path=target
+ )
return self._build_generated_page(
"symbol_spotlight",
f"{parsed.file_info.path}::{symbol.name}",
@@ -234,6 +262,12 @@ class PageGenerator:
graph: Any,
git_meta_map: dict[str, dict] | None = None,
page_summaries: dict[str, str] | None = None,
+ decision_records: list[dict] | None = None,
+ dead_code_findings: list[dict] | None = None,
+ external_systems: list[dict] | None = None,
+ community_label: str | None = None,
+ community_cohesion: float | None = None,
+ target_path: str | None = None,
) -> GeneratedPage:
ctx = self._assembler.assemble_module_page(
module_path,
@@ -241,6 +275,12 @@ class PageGenerator:
file_contexts,
graph,
page_summaries=page_summaries,
+ git_meta_map=git_meta_map,
+ decision_records=decision_records,
+ dead_code_findings=dead_code_findings,
+ external_systems=external_systems,
+ community_label=community_label,
+ community_cohesion=community_cohesion,
)
module_git_summary = None
if git_meta_map:
@@ -261,10 +301,13 @@ class PageGenerator:
"most_active_commits_90d": most_active.get("commit_count_90d", 0),
}
user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)
- response = await self._call_provider("module_page", user_prompt, str(uuid.uuid4()))
+ page_target = target_path or module_path
+ response = await self._call_provider(
+ "module_page", user_prompt, str(uuid.uuid4()), target_path=page_target
+ )
return self._build_generated_page(
"module_page",
- module_path,
+ page_target,
f"Module: {module_path}",
response,
compute_source_hash(user_prompt),
@@ -279,7 +322,9 @@ class PageGenerator:
) -> GeneratedPage:
ctx = self._assembler.assemble_scc_page(scc_id, scc_files, file_contexts)
user_prompt = self._render("scc_page.j2", ctx=ctx)
- response = await self._call_provider("scc_page", user_prompt, str(uuid.uuid4()))
+ response = await self._call_provider(
+ "scc_page", user_prompt, str(uuid.uuid4()), target_path=scc_id
+ )
return self._build_generated_page(
"scc_page",
scc_id,
@@ -298,6 +343,8 @@ class PageGenerator:
git_meta_map: dict[str, dict] | None = None,
graph_builder: Any | None = None,
repo_name: str | None = None,
+ external_systems: list[dict] | None = None,
+ decision_records: list[dict] | None = None,
) -> GeneratedPage:
ctx = self._assembler.assemble_repo_overview(
repo_structure,
@@ -305,6 +352,8 @@ class PageGenerator:
sccs,
community,
graph_builder=graph_builder,
+ external_systems=external_systems,
+ decision_records=decision_records,
)
repo_git_summary = None
if git_meta_map:
@@ -322,10 +371,12 @@ class PageGenerator:
"oldest_file": oldest.get("file_path", "") if oldest else "",
"oldest_file_age_days": oldest.get("age_days", 0) if oldest else 0,
}
- user_prompt = self._render("repo_overview.j2", ctx=ctx, repo_git_summary=repo_git_summary)
- response = await self._call_provider("repo_overview", user_prompt, str(uuid.uuid4()))
if not repo_name:
repo_name = getattr(repo_structure, "name", None) or "repo"
+ user_prompt = self._render("repo_overview.j2", ctx=ctx, repo_git_summary=repo_git_summary)
+ response = await self._call_provider(
+ "repo_overview", user_prompt, str(uuid.uuid4()), target_path=repo_name
+ )
return self._build_generated_page(
"repo_overview",
repo_name,
@@ -347,7 +398,9 @@ class PageGenerator:
graph, pagerank, community, sccs, repo_name
)
user_prompt = self._render("architecture_diagram.j2", ctx=ctx)
- response = await self._call_provider("architecture_diagram", user_prompt, str(uuid.uuid4()))
+ response = await self._call_provider(
+ "architecture_diagram", user_prompt, str(uuid.uuid4()), target_path=repo_name
+ )
return self._build_generated_page(
"architecture_diagram",
repo_name,
@@ -364,7 +417,9 @@ class PageGenerator:
) -> GeneratedPage:
ctx = self._assembler.assemble_api_contract(parsed, source_bytes)
user_prompt = self._render("api_contract.j2", ctx=ctx)
- response = await self._call_provider("api_contract", user_prompt, str(uuid.uuid4()))
+ response = await self._call_provider(
+ "api_contract", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
return self._build_generated_page(
"api_contract",
parsed.file_info.path,
@@ -374,6 +429,54 @@ class PageGenerator:
GENERATION_LEVELS["api_contract"],
)
+ async def generate_onboarding_page(
+ self,
+ spec: "_onboarding.SubkindSpec",
+ signals: "_onboarding.OnboardingSignals",
+ ) -> GeneratedPage | None:
+ """Generate one onboarding page from a registered subkind spec.
+
+ Returns ``None`` when the subkind's gate fails (``build_context``
+ returned ``None``) — the slot is silently skipped for this repo.
+ """
+ ctx = spec.build_context(signals)
+ if ctx is None:
+ log.debug("onboarding.gate_skipped", slot=spec.slot)
+ return None
+
+ template_name = f"onboarding/{spec.template}"
+ user_prompt = self._render(template_name, ctx=ctx, slot=spec.slot)
+ target = _onboarding.target_path(spec.slot)
+ response = await self._call_provider(
+ "onboarding", user_prompt, str(uuid.uuid4()), target_path=target
+ )
+ page = self._build_generated_page(
+ "onboarding",
+ target,
+ spec.title,
+ response,
+ compute_source_hash(user_prompt),
+ GENERATION_LEVELS["onboarding"],
+ )
+ # Subkind discriminator lives in metadata; page_type alone is shared
+ # across all six generated onboarding slots.
+ page.metadata["subkind"] = spec.slot
+ page.metadata["onboarding_slot"] = spec.slot
+ return page
+
+ @staticmethod
+ def _tag_promoted_pages(pages: list[GeneratedPage]) -> None:
+ """Tag repo_overview / architecture_diagram pages with their slot.
+
+ Mutates each matching page's ``metadata["onboarding_slot"]`` so the
+ UI groups them into the Onboarding folder without changing their
+ underlying ``page_type``. Idempotent and tolerant of missing pages.
+ """
+ for page in pages:
+ slot = _onboarding.PROMOTED_SLOTS.get(page.page_type)
+ if slot is not None:
+ page.metadata["onboarding_slot"] = slot
+
async def generate_infra_page(
self,
parsed: ParsedFile,
@@ -381,7 +484,9 @@ class PageGenerator:
) -> GeneratedPage:
ctx = self._assembler.assemble_infra_page(parsed, source_bytes)
user_prompt = self._render("infra_page.j2", ctx=ctx)
- response = await self._call_provider("infra_page", user_prompt, str(uuid.uuid4()))
+ response = await self._call_provider(
+ "infra_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
return self._build_generated_page(
"infra_page",
parsed.file_info.path,
@@ -391,28 +496,6 @@ class PageGenerator:
GENERATION_LEVELS["infra_page"],
)
- async def generate_cross_package(
- self,
- source_pkg: str,
- target_pkg: str,
- source_fcs: list[FilePageContext],
- target_fcs: list[FilePageContext],
- graph: Any,
- ) -> GeneratedPage:
- ctx = self._assembler.assemble_cross_package(
- source_pkg, target_pkg, source_fcs, target_fcs, graph
- )
- user_prompt = self._render("cross_package.j2", ctx=ctx)
- response = await self._call_provider("cross_package", user_prompt, str(uuid.uuid4()))
- return self._build_generated_page(
- "cross_package",
- f"{source_pkg}->{target_pkg}",
- f"Cross-Package: {source_pkg} → {target_pkg}",
- response,
- compute_source_hash(user_prompt),
- GENERATION_LEVELS["cross_package"],
- )
-
# ------------------------------------------------------------------
# generate_all — orchestration
# ------------------------------------------------------------------
@@ -427,9 +510,13 @@ class PageGenerator:
job_system: Any | None = None, # JobSystem | None
on_page_done: Callable[[str], None] | None = None,
on_total_known: Callable[[int], None] | None = None,
+ on_subphase: Callable[[str, int | None], None] | None = None,
git_meta_map: dict[str, dict] | None = None,
resume: bool = False,
repo_path: Path | str | None = None,
+ dead_code_report: Any | None = None,
+ decision_report: Any | None = None,
+ external_systems: list[dict] | None = None,
) -> list[GeneratedPage]:
"""Generate all wiki pages for a repository.
@@ -456,6 +543,39 @@ class PageGenerator:
community = graph_builder.community_detection()
sccs = graph_builder.strongly_connected_components()
+ # ---- Build per-file lookup maps from phase-2 signals ----
+ dead_code_by_file: dict[str, list[dict]] = {}
+ if dead_code_report is not None and getattr(dead_code_report, "findings", None):
+ for f in dead_code_report.findings:
+ dead_code_by_file.setdefault(f.file_path, []).append(
+ {
+ "symbol_name": f.symbol_name,
+ "symbol_kind": f.symbol_kind,
+ "kind": str(f.kind),
+ "reason": f.reason,
+ "confidence": f.confidence,
+ "safe_to_delete": f.safe_to_delete,
+ }
+ )
+
+ decisions_by_file: dict[str, list[dict]] = {}
+ decisions_all: list[dict] = []
+ if decision_report is not None and getattr(decision_report, "decisions", None):
+ for d in decision_report.decisions:
+ payload = {
+ "title": d.title,
+ "decision": d.decision,
+ "rationale": d.rationale,
+ "source": d.source,
+ "confidence": d.confidence,
+ "evidence_file": d.evidence_file,
+ }
+ decisions_all.append(payload)
+ for fp in d.affected_files or []:
+ decisions_by_file.setdefault(fp, []).append(payload)
+
+ external_systems = external_systems or []
+
all_pages: list[GeneratedPage] = []
semaphore = asyncio.Semaphore(self._config.max_concurrency)
embed_semaphore = asyncio.Semaphore(self._config.embed_concurrency or 1)
@@ -493,40 +613,61 @@ class PageGenerator:
self._provider.model_name,
)
+ async def _embed_async(page: GeneratedPage) -> None:
+ """Embed a finished page into the vector store. Safe to fire-and-forget.
+
+ Errors are swallowed at debug level — embedding is a RAG
+ enhancement, not load-bearing for the page itself.
+ """
+ try:
+ summary = _extract_summary(page.content)
+ async with embed_semaphore:
+ await self._vector_store.embed_and_upsert(
+ page.page_id,
+ page.content,
+ {
+ "page_type": page.page_type,
+ "target_path": page.target_path,
+ "content": page.content[:600],
+ "summary": summary,
+ },
+ )
+ except Exception as e:
+ log.debug("rag.embed_failed", page_id=page.page_id, error=str(e))
+
async def run_level(named_coros: list[tuple[str, Any]], level: int) -> list[GeneratedPage]:
if job_system is not None and job_id is not None:
job_system.update_level(job_id, level)
+ # Embed tasks spawned during this level. Drained after the
+ # gather() so the next level's RAG search sees a fully-indexed
+ # store, but never blocks the per-page wave.
+ pending_embeds: list[asyncio.Task[None]] = []
+
async def guarded_named(page_id: str, coro: Any) -> Any:
try:
async with semaphore:
result = await coro
- # Embed page for RAG (B1)
- if self._vector_store is not None and isinstance(result, GeneratedPage):
- try:
- page_summary = _extract_summary(result.content)
- async with embed_semaphore:
- await self._vector_store.embed_and_upsert(
- result.page_id,
- result.content,
- {
- "page_type": result.page_type,
- "target_path": result.target_path,
- "content": result.content[:600],
- "summary": page_summary,
- },
- )
- except Exception as e:
- log.debug("rag.embed_failed", page_id=result.page_id, error=str(e))
- # Store summary for dependency context (B2)
if isinstance(result, GeneratedPage):
+ # Summary capture is cheap (string ops) — keep
+ # inline so the next page's context assembly sees
+ # it immediately.
completed_page_summaries[result.target_path] = _extract_summary(
result.content
)
- # Report progress immediately (not batched after gather)
+ # Progress tick fires the moment the LLM call
+ # returns — the user sees the page as done before
+ # the embed completes.
if on_page_done is not None:
on_page_done(result.page_type)
+ # Fire-and-forget the embed/upsert. Removes ~1
+ # embedder round-trip from this task's critical
+ # path so the LLM slot frees up immediately.
+ if self._vector_store is not None:
+ pending_embeds.append(
+ asyncio.create_task(_embed_async(result))
+ )
return result
except Exception as exc:
if job_system is not None and job_id is not None:
@@ -541,13 +682,23 @@ class PageGenerator:
tasks = [guarded_named(pid, c) for pid, c in named_coros]
results = await asyncio.gather(*tasks)
+ # Drain pending embeds before declaring the level done — the
+ # next level's RAG search depends on these landing in the store.
+ if pending_embeds:
+ await asyncio.gather(*pending_embeds, return_exceptions=True)
pages = [r for r in results if isinstance(r, GeneratedPage)]
if job_system is not None and job_id is not None:
for r in pages:
job_system.complete_page(job_id, r.page_id)
return pages
- # ---- Budget pre-computation ----
+ # ---- Page selection (single source of truth) ----
+ # The selection subsystem scores every candidate, allocates the
+ # global budget across page-type buckets, and returns an
+ # allow-set. Each level's emit loop iterates only over members
+ # of that allow-set — there are no bypass paths. The same
+ # function is called by the cost estimator so the pre-run
+ # estimate cannot drift from the actual run.
code_files = [
p
for p in parsed_files
@@ -556,97 +707,70 @@ class PageGenerator:
and p.file_info.language in _CODE_LANGUAGES
]
- # Sort: entry points first, then hotspots, then high PageRank (A1)
- if git_meta_map:
- code_files = sorted(
- code_files,
- key=lambda p: (
- not p.file_info.is_entry_point,
- not git_meta_map.get(p.file_info.path, {}).get("is_hotspot", False),
- -pagerank.get(p.file_info.path, 0.0),
- ),
- )
+ # Near-clone dedupe runs before scoring so clone losers never
+ # consume scoring budget. Entry points are never dropped.
+ if getattr(self._config, "dedupe_near_clones", True):
+ drop_paths = _select_clone_representatives(code_files, pagerank)
+ if drop_paths:
+ log.info("page_selection.clone_dedupe", dropped=len(drop_paths))
+ code_files = [p for p in code_files if p.file_info.path not in drop_paths]
+ parsed_files_for_selection = [
+ p for p in parsed_files if p.file_info.path not in drop_paths
+ ]
+ else:
+ parsed_files_for_selection = parsed_files
else:
- code_files = sorted(
- code_files,
- key=lambda p: (
- not p.file_info.is_entry_point,
- -pagerank.get(p.file_info.path, 0.0),
- ),
- )
-
- code_pr_scores = sorted(
- [pagerank.get(p.file_info.path, 0.0) for p in code_files],
- reverse=True,
- )
- _all_public_symbols: list[tuple[Any, Any]] = [
- (sym, p) for p in parsed_files for sym in p.symbols if sym.visibility == "public"
- ]
+ parsed_files_for_selection = parsed_files
- budget = max(50, int(len(parsed_files) * self._config.max_pages_pct))
- # Estimate fixed overhead (api, scc, module, repo_overview, arch_diagram)
- _fixed_overhead = (
- sum(1 for p in parsed_files if p.file_info.is_api_contract)
- + sum(1 for scc in sccs if len(scc) > 1)
- + len(
- {
- (
- Path(p.file_info.path).parts[0]
- if len(Path(p.file_info.path).parts) > 1
- else "root"
- )
- for p in code_files
- }
+ try:
+ community_info_map = graph_builder.community_info() or {}
+ except Exception:
+ community_info_map = {}
+
+ from .selection import SelectionInputs, select_pages
+
+ selection = select_pages(
+ SelectionInputs(
+ parsed_files=parsed_files_for_selection,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ community_info=community_info_map,
+ sccs=list(sccs),
+ git_meta_map=git_meta_map,
+ config=self._config,
)
- + 2 # repo_overview + architecture_diagram
)
- _remaining = max(0, budget - _fixed_overhead)
- # File page gets priority over symbol_spotlight
- _n_file_uncapped = (
- max(1, int(len(code_pr_scores) * self._config.file_page_top_percentile))
- if code_pr_scores
- else 0
- )
- _n_file_cap = min(_n_file_uncapped, _remaining)
- pr_threshold = (
- code_pr_scores[_n_file_cap - 1] if code_pr_scores and _n_file_cap > 0 else 0.0
+ # Bucket allow-sets — O(1) membership checks in the level loops.
+ sel_file_paths: set[str] = set(selection.file_page_paths)
+ sel_symbol_keys: set[tuple[str, str]] = set(selection.symbol_spotlights)
+ sel_api_paths: set[str] = set(selection.api_contract_paths)
+ sel_infra_paths: set[str] = set(selection.infra_paths)
+ sel_module_groups = list(selection.module_groups)
+ sel_scc_groups = list(selection.scc_groups)
+
+ # Sort code_files for stable level-2 ordering: selected files
+ # first (so dep summaries land in the store earliest), then by
+ # PageRank desc. The topo-sort below further refines this.
+ code_files = sorted(
+ code_files,
+ key=lambda p: (
+ p.file_info.path not in sel_file_paths,
+ not p.file_info.is_entry_point,
+ -pagerank.get(p.file_info.path, 0.0),
+ ),
)
- _sym_budget = max(0, _remaining - _n_file_cap)
- _n_sym_uncapped = (
- max(1, int(len(_all_public_symbols) * self._config.top_symbol_percentile))
- if _all_public_symbols
- else 0
- )
- _n_sym_cap = min(_n_sym_uncapped, _sym_budget)
-
- # Compute estimated total and notify progress (A7)
- # Use the actual file_page count (files passing _is_significant_file), not
- # _n_file_cap. The cap sets pr_threshold, but files with high betweenness or
- # entry_point status bypass that threshold, so actual count > _n_file_cap.
- _actual_file_page_count = sum(
- 1
- for p in code_files
- if _is_significant_file(p, pagerank, betweenness, self._config, pr_threshold)
- )
estimated_total = (
- sum(1 for p in parsed_files if p.file_info.is_api_contract)
- + _n_sym_cap
- + _actual_file_page_count
- + sum(1 for scc in sccs if len(scc) > 1)
- + len(
- {
- (
- Path(p.file_info.path).parts[0]
- if len(Path(p.file_info.path).parts) > 1
- else "root"
- )
- for p in code_files
- }
- )
- + 2 # repo_overview + arch_diagram
- + sum(1 for p in parsed_files if _is_infra_file(p))
+ selection.counts()["api_contract"]
+ + selection.counts()["symbol_spotlight"]
+ + selection.counts()["file_page"]
+ + selection.counts()["scc_page"]
+ + selection.counts()["module_page"]
+ + int(selection.emit_repo_overview)
+ + int(selection.emit_arch_diagram)
+ + selection.counts()["infra_page"]
)
remaining_total = max(0, estimated_total - len(completed_ids))
if on_total_known is not None:
@@ -654,8 +778,12 @@ class PageGenerator:
if job_system is not None and job_id is not None:
job_system.start_job(job_id, estimated_total)
- # ---- Level 0: api_contract ----
- api_files = [p for p in parsed_files if p.file_info.is_api_contract]
+ # ---- Level 0: api_contract (allow-set filtered) ----
+ api_files = [
+ p
+ for p in parsed_files
+ if p.file_info.is_api_contract and p.file_info.path in sel_api_paths
+ ]
level0_coros = [
(
compute_page_id("api_contract", p.file_info.path),
@@ -664,19 +792,20 @@ class PageGenerator:
for p in api_files
if compute_page_id("api_contract", p.file_info.path) not in completed_ids
]
- level0_pages = await run_level(level0_coros, 0)
- all_pages.extend(level0_pages)
-
- # ---- Level 1: symbol_spotlight (top percentile by PageRank) ----
- all_symbols_with_file: list[tuple[Any, ParsedFile]] = _all_public_symbols
-
- if all_symbols_with_file and _n_sym_cap > 0:
- all_symbols_with_file.sort(
- key=lambda x: pagerank.get(x[1].file_info.path, 0.0), reverse=True
- )
- top_symbols = all_symbols_with_file[:_n_sym_cap]
- else:
- top_symbols = []
+ # ---- Level 1: symbol_spotlight (allow-set filtered) ----
+ # The selection layer already picked the top symbols by score;
+ # here we just resolve them back to (Symbol, ParsedFile) pairs.
+ parsed_by_path: dict[str, ParsedFile] = {
+ p.file_info.path: p for p in parsed_files
+ }
+ top_symbols: list[tuple[Any, ParsedFile]] = []
+ for file_path, sym_name in selection.symbol_spotlights:
+ pf = parsed_by_path.get(file_path)
+ if pf is None:
+ continue
+ sym = next((s for s in pf.symbols if s.name == sym_name), None)
+ if sym is not None:
+ top_symbols.append((sym, pf))
level1_coros = [
(
@@ -687,8 +816,15 @@ class PageGenerator:
if compute_page_id("symbol_spotlight", f"{pf.file_info.path}::{sym.name}")
not in completed_ids
]
- level1_pages = await run_level(level1_coros, 1)
- all_pages.extend(level1_pages)
+
+ # Levels 0 (api_contract) and 1 (symbol_spotlight) share no
+ # data dependencies — both feed into nothing else upstream of
+ # Level 2 — so they run in one merged batch instead of two
+ # sequential barriers. ``run_level`` already bounds total
+ # concurrency via ``self._config.max_concurrency`` so the merge
+ # only removes idle slots, never over-saturates the provider.
+ level01_pages = await run_level(level0_coros + level1_coros, 1)
+ all_pages.extend(level01_pages)
# ---- Level 2: file_page (significant code files only) ----
# Context is assembled for ALL code files (module pages need it).
@@ -739,23 +875,36 @@ class PageGenerator:
file_page_contexts: dict[str, FilePageContext] = {}
- level2_coros: list[tuple[str, Any]] = []
- for p in code_files:
- # Pre-fetch dependency summaries from vector store for deps not yet
- # in the completed_page_summaries accumulator (e.g. from prior runs).
- if self._vector_store is not None:
+ # Batch-prefetch dependency summaries from the vector store in a
+ # SINGLE call covering every code file's dependencies — replaces
+ # the prior per-file serial loop that turned N×M awaits into a
+ # measurable bottleneck on the level-2 critical path.
+ if self._vector_store is not None:
+ needed_deps: set[str] = set()
+ for p in code_files:
path_ = p.file_info.path
- out_edges = list(graph.successors(path_)) if path_ in graph else []
- internal_deps = [e for e in out_edges if not e.startswith("external:")]
- for dep in internal_deps:
- if dep not in completed_page_summaries:
- try:
- result = await self._vector_store.get_page_summary_by_path(dep)
- if result and result.get("summary"):
- completed_page_summaries[dep] = result["summary"]
- except Exception:
- pass # Non-fatal — dep context is optional
+ if path_ not in graph:
+ continue
+ for dep in graph.successors(path_):
+ if dep.startswith("external:"):
+ continue
+ if dep in completed_page_summaries:
+ continue
+ needed_deps.add(dep)
+ if needed_deps:
+ try:
+ batch = await self._vector_store.get_page_summaries_by_paths(
+ list(needed_deps)
+ )
+ for dep_path, payload in batch.items():
+ summary = payload.get("summary") if payload else None
+ if summary:
+ completed_page_summaries[dep_path] = summary
+ except Exception as exc:
+ log.debug("rag.batch_dep_prefetch_failed", error=str(exc))
+ level2_coros: list[tuple[str, Any]] = []
+ for p in code_files:
ctx = self._assembler.assemble_file_page(
p,
graph,
@@ -765,25 +914,20 @@ class PageGenerator:
source_map.get(p.file_info.path, b""),
git_meta=git_meta_map.get(p.file_info.path) if git_meta_map else None,
page_summaries=completed_page_summaries,
+ dead_code_findings=dead_code_by_file.get(p.file_info.path),
+ decision_records=decisions_by_file.get(p.file_info.path),
)
file_page_contexts[p.file_info.path] = ctx
pid = compute_page_id("file_page", p.file_info.path)
- if (
- _is_significant_file(p, pagerank, betweenness, self._config, pr_threshold)
- and pid not in completed_ids
- ):
+ if p.file_info.path in sel_file_paths and pid not in completed_ids:
level2_coros.append((pid, self._generate_file_page_from_ctx(p, ctx)))
level2_pages = await run_level(level2_coros, 2)
all_pages.extend(level2_pages)
- # ---- Level 3: scc_page (only true cycles: len > 1) ----
+ # ---- Level 3: scc_page (allow-set filtered) ----
scc_coros: list[tuple[str, Any]] = []
- for i, scc in enumerate(sccs):
- if len(scc) <= 1:
- continue
- scc_id = f"scc-{i}"
- scc_files = sorted(scc)
+ for scc_id, scc_files in sel_scc_groups:
fc_list = [file_page_contexts[f] for f in scc_files if f in file_page_contexts]
pid = compute_page_id("scc_page", scc_id)
if pid not in completed_ids:
@@ -791,65 +935,46 @@ class PageGenerator:
level3_pages = await run_level(scc_coros, 3)
all_pages.extend(level3_pages)
- # ---- Level 4: module_page (grouped by top-level directory) ----
- module_groups: dict[str, list[FilePageContext]] = {}
- module_languages: dict[str, str] = {}
- for p in code_files:
- parts = Path(p.file_info.path).parts
- module = parts[0] if len(parts) > 1 else "root"
- fc = file_page_contexts.get(p.file_info.path)
- if fc is not None:
- module_groups.setdefault(module, []).append(fc)
- module_languages[module] = p.file_info.language
-
- level4_coros: list[tuple[str, Any]] = [
- (
- compute_page_id("module_page", module),
- self.generate_module_page(
- module,
- module_languages.get(module, "unknown"),
- fcs,
- graph,
- git_meta_map=git_meta_map,
- page_summaries=completed_page_summaries,
- ),
+ # ---- Level 4: module_page (allow-set filtered) ----
+ # Module groups come from the selection layer; we only need to
+ # resolve FilePageContext objects for the files we already
+ # built contexts for in Level 2.
+ level4_coros: list[tuple[str, Any]] = []
+ for mg in sel_module_groups:
+ fcs = [
+ file_page_contexts[fp]
+ for fp in mg.file_paths
+ if fp in file_page_contexts
+ ]
+ if not fcs:
+ continue
+ page_id = compute_page_id("module_page", mg.key)
+ if page_id in completed_ids:
+ continue
+ level4_coros.append(
+ (
+ page_id,
+ self.generate_module_page(
+ mg.display,
+ mg.language,
+ fcs,
+ graph,
+ git_meta_map=git_meta_map,
+ page_summaries=completed_page_summaries,
+ decision_records=decisions_all,
+ dead_code_findings=[
+ d for fc in fcs for d in dead_code_by_file.get(fc.file_path, [])
+ ],
+ external_systems=external_systems,
+ community_label=mg.label,
+ community_cohesion=mg.cohesion,
+ target_path=mg.key,
+ ),
+ )
)
- for module, fcs in module_groups.items()
- if compute_page_id("module_page", module) not in completed_ids
- ]
level4_pages = await run_level(level4_coros, 4)
all_pages.extend(level4_pages)
- # ---- Level 5: cross_package (only if monorepo) ----
- if repo_structure.is_monorepo:
- seen_pairs: set[tuple[str, str]] = set()
- cross_coros: list[tuple[str, Any]] = []
- for src_pkg, src_fcs in module_groups.items():
- for fc in src_fcs:
- for dep in fc.dependencies:
- dep_parts = Path(dep).parts
- dep_pkg = dep_parts[0] if len(dep_parts) > 1 else "root"
- pair = (src_pkg, dep_pkg)
- if dep_pkg != src_pkg and pair not in seen_pairs:
- seen_pairs.add(pair)
- dep_fcs = module_groups.get(dep_pkg, [])
- ctx_xpkg = self._assembler.assemble_cross_package(
- src_pkg, dep_pkg, src_fcs, dep_fcs, graph
- )
- if ctx_xpkg.coupling_strength >= 2:
- pid = compute_page_id("cross_package", f"{src_pkg}->{dep_pkg}")
- if pid not in completed_ids:
- cross_coros.append(
- (
- pid,
- self.generate_cross_package(
- src_pkg, dep_pkg, src_fcs, dep_fcs, graph
- ),
- )
- )
- level5_pages = await run_level(cross_coros, 5)
- all_pages.extend(level5_pages)
-
# ---- Level 6: repo_overview + architecture_diagram ----
level6_coros: list[tuple[str, Any]] = []
if compute_page_id("repo_overview", repo_name) not in completed_ids:
@@ -864,6 +989,8 @@ class PageGenerator:
git_meta_map=git_meta_map,
graph_builder=graph_builder,
repo_name=repo_name,
+ external_systems=external_systems,
+ decision_records=decisions_all[:10],
),
)
)
@@ -874,11 +1001,12 @@ class PageGenerator:
self.generate_architecture_diagram(graph, pagerank, community, sccs, repo_name),
)
)
- level6_pages = await run_level(level6_coros, 6)
- all_pages.extend(level6_pages)
-
- # ---- Level 7: infra_page ----
- infra_files = [p for p in parsed_files if _is_infra_file(p)]
+ # ---- Level 7: infra_page (allow-set filtered) ----
+ infra_files = [
+ p
+ for p in parsed_files
+ if _is_infra_file(p) and p.file_info.path in sel_infra_paths
+ ]
level7_coros: list[tuple[str, Any]] = [
(
compute_page_id("infra_page", p.file_info.path),
@@ -887,8 +1015,69 @@ class PageGenerator:
for p in infra_files
if compute_page_id("infra_page", p.file_info.path) not in completed_ids
]
- level7_pages = await run_level(level7_coros, 7)
- all_pages.extend(level7_pages)
+
+ # ---- Level 8: onboarding (curated collection) ----
+ # Each subkind defines its own gate inside build_context — slots
+ # whose gates fail return None and are skipped entirely. Promoted
+ # slots (project_overview / architecture_guide) are tagged onto the
+ # level-6 pages and don't appear here.
+ level8_coros: list[tuple[str, Any]] = []
+ if getattr(self._config, "enable_onboarding", True):
+ specs = _onboarding.iter_specs()
+ if specs:
+ if on_subphase is not None:
+ try:
+ on_subphase("onboarding", len(specs))
+ except Exception:
+ pass
+ signals = _onboarding.OnboardingSignals(
+ repo_name=repo_name,
+ repo_structure=repo_structure,
+ parsed_files=tuple(parsed_files),
+ source_map=source_map,
+ graph_builder=graph_builder,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ sccs=tuple(sccs),
+ git_meta_map=git_meta_map,
+ dead_code_by_file=dead_code_by_file,
+ decisions_all=tuple(decisions_all),
+ external_systems=tuple(external_systems),
+ completed_page_summaries=dict(completed_page_summaries),
+ )
+ for spec in specs:
+ page_id = compute_page_id(
+ "onboarding", _onboarding.target_path(spec.slot)
+ )
+ if page_id in completed_ids:
+ continue
+ level8_coros.append(
+ (page_id, self.generate_onboarding_page(spec, signals))
+ )
+
+ # Levels 6, 7, and 8 share no data dependencies with each other
+ # — Level 6 needs only the graph + repo metadata; Level 7
+ # documents standalone infra files; Level 8 consumes a frozen
+ # ``OnboardingSignals`` snapshot. Run them in a single merged
+ # batch instead of three sequential barriers.
+ final_pages = await run_level(level6_coros + level7_coros + level8_coros, 8)
+ # Tag promoted onboarding slots (repo_overview / architecture_diagram)
+ # so the UI groups them into the Onboarding folder. No content change.
+ self._tag_promoted_pages(final_pages)
+ all_pages.extend(final_pages)
+
+ # Post-generation: resolve backtick-quoted refs in every page's
+ # markdown to other pages' ``page_id``s and stash the result in
+ # ``metadata["wiki_links"]``. The reverse index lands in
+ # ``metadata["backlinks"]``. Pure regex + dict lookup — no LLM
+ # call, safe to run on every generation.
+ try:
+ from .interlinking import attach_wiki_links_and_backlinks
+
+ attach_wiki_links_and_backlinks(all_pages, parsed_files)
+ except Exception as exc:
+ log.debug("interlinking.failed", error=str(exc))
# Finalize job
if job_system is not None and job_id is not None:
@@ -912,22 +1101,49 @@ class PageGenerator:
ctx: FilePageContext,
) -> GeneratedPage:
"""Generate a file_page from a pre-assembled context (avoids double-assembly)."""
- # RAG context: query vector store for related pages (B1)
- if self._vector_store is not None:
- query_terms = parsed.exports or [
- s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"
- ]
- if query_terms:
+ # RAG context: query vector store for related pages (B1).
+ # Gated by two short-circuits so we don't burn an embedder
+ # round-trip on every page when the result wouldn't help:
+ # 1. ``enable_rag_context`` config flag (off → fully skip).
+ # 2. ``rag_min_store_size`` — early pages run against an empty
+ # or near-empty store and the search returns nothing useful.
+ if self._vector_store is not None and getattr(
+ self._config, "enable_rag_context", True
+ ):
+ min_store_size = max(0, int(getattr(self._config, "rag_min_store_size", 10) or 0))
+ store_ok = True
+ if min_store_size > 0:
try:
- results = await self._vector_store.search(", ".join(query_terms[:5]), limit=3)
- self_id = f"file_page:{parsed.file_info.path}"
- ctx.rag_context = [
- f"[{r.page_id}]\n{r.snippet}" for r in results if r.page_id != self_id
- ]
- except Exception as e:
- log.debug("rag.search_failed", path=parsed.file_info.path, error=str(e))
+ current_ids = await self._vector_store.list_page_ids()
+ store_ok = len(current_ids) >= min_store_size
+ except Exception:
+ # If the store can't be sized cheaply, fall through to
+ # the search — it'll either succeed or hit the
+ # existing exception path.
+ store_ok = True
+ if store_ok:
+ query_terms = parsed.exports or [
+ s["name"] for s in ctx.symbols[:3] if s.get("visibility") == "public"
+ ]
+ if query_terms:
+ try:
+ results = await self._vector_store.search(
+ ", ".join(query_terms[:5]), limit=3
+ )
+ self_id = f"file_page:{parsed.file_info.path}"
+ ctx.rag_context = [
+ f"[{r.page_id}]\n{r.snippet}"
+ for r in results
+ if r.page_id != self_id
+ ]
+ except Exception as e:
+ log.debug(
+ "rag.search_failed", path=parsed.file_info.path, error=str(e)
+ )
user_prompt = self._render("file_page.j2", ctx=ctx)
- response = await self._call_provider("file_page", user_prompt, str(uuid.uuid4()))
+ response = await self._call_provider(
+ "file_page", user_prompt, str(uuid.uuid4()), target_path=parsed.file_info.path
+ )
page = self._build_generated_page(
"file_page",
parsed.file_info.path,
@@ -953,8 +1169,32 @@ class PageGenerator:
page_type: str,
user_prompt: str,
request_id: str,
+ target_path: str | None = None,
) -> GeneratedResponse:
"""Call the provider with caching, optionally prefixing a language instruction."""
+ # Persistent cross-run cache: if the page exists from a prior run, was
+ # produced by the same model, and the prompt's source_hash matches,
+ # reuse the stored content without an LLM call.
+ if self._config.cache_enabled and target_path is not None:
+ page_id = compute_page_id(page_type, target_path)
+ prior = self._prior_pages.get(page_id)
+ if prior is not None and prior.model_name == self._provider.model_name:
+ current_hash = compute_source_hash(user_prompt)
+ if prior.source_hash == current_hash:
+ self._reuse_count += 1
+ log.debug(
+ "page_cache.persistent_hit",
+ page_type=page_type,
+ target_path=target_path,
+ )
+ return GeneratedResponse(
+ content=prior.content,
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ usage={"reused_from_prior_run": True},
+ )
+
key = self._compute_cache_key(page_type, user_prompt)
if self._config.cache_enabled and key in self._cache:
log.debug("Cache hit", page_type=page_type, key=key[:8])
@@ -962,6 +1202,13 @@ class PageGenerator:
system_prompt = self._build_system_prompt(page_type)
+ # The same system prompt is reused for every page of a given type, so
+ # mark it as cacheable. Providers without server-side prompt caching
+ # ignore the hint safely.
+ cache_hints: tuple[CacheHint, ...] = (
+ (CacheHint(segment="system"),) if self._config.cache_enabled else ()
+ )
+
response = await self._provider.generate(
system_prompt,
user_prompt,
@@ -969,6 +1216,7 @@ class PageGenerator:
temperature=self._config.temperature,
request_id=request_id,
reasoning=self._config.reasoning,
+ cache_hints=cache_hints,
)
if self._config.cache_enabled:
@@ -1126,6 +1374,18 @@ def _is_significant_file(
if not (is_entry or pr >= pr_threshold or bet > 0.0):
return False
+ # Trivial-file gate: small files with almost no symbols (data classes,
+ # marker classes, single-message wrappers like Messages/*.cs) produce
+ # low-value pages. Entry points and graph hubs (high PageRank) bypass.
+ if (
+ getattr(config, "skip_trivial_files", True)
+ and not is_entry
+ and pr < pr_threshold * 2
+ and len(parsed.symbols) <= 2
+ and parsed.file_info.size_bytes < 1500
+ ):
+ return False
+
# Waive the symbol-count requirement for graph-connected files that have
# no original definitions of their own (e.g. state/config modules that
# are imported by many files but mostly re-export or assemble values).
@@ -1135,6 +1395,42 @@ def _is_significant_file(
return True
+def _select_clone_representatives(
+ code_files: list[ParsedFile],
+ pagerank: dict[str, float],
+ *,
+ min_cluster_size: int = 3,
+) -> set[str]:
+ """Return paths of files to *drop* because they are near-clones.
+
+ Groups files by (parent_directory, signature shape), where the shape is the
+ sorted tuple of ``(symbol_kind, symbol_name)`` pairs from the parser. When
+ a cluster has at least ``min_cluster_size`` members, the highest-PageRank
+ member is kept and the rest are dropped. Entry points are never dropped.
+
+ Language-agnostic: works for any language whose symbols carry a kind+name,
+ which the parser guarantees.
+ """
+ from collections import defaultdict
+
+ clusters: dict[tuple[str, tuple[tuple[str, str], ...]], list[ParsedFile]] = defaultdict(list)
+ for p in code_files:
+ if p.file_info.is_entry_point or not p.symbols:
+ continue
+ parent = str(Path(p.file_info.path).parent.as_posix())
+ shape = tuple(sorted((str(s.kind), s.name) for s in p.symbols))
+ clusters[(parent, shape)].append(p)
+
+ drop: set[str] = set()
+ for members in clusters.values():
+ if len(members) < min_cluster_size:
+ continue
+ members.sort(key=lambda p: pagerank.get(p.file_info.path, 0.0), reverse=True)
+ for loser in members[1:]:
+ drop.add(loser.file_info.path)
+ return drop
+
+
# ---------------------------------------------------------------------------
# LLM output validation
# ---------------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/selection/__init__.py b/packages/core/src/repowise/core/generation/selection/__init__.py
new file mode 100644
index 000000000..d8873f71f
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/selection/__init__.py
@@ -0,0 +1,47 @@
+"""Page selection — single source of truth for which pages get generated.
+
+Used by both ``page_generator.generate_all()`` (to decide what to emit)
+and ``cost_estimator.build_generation_plan()`` (to estimate cost without
+running the LLM). Both paths call the same :func:`select_pages` function
+so the estimate and the actual run can never drift apart.
+
+The selection is a pure function of (parsed_files, graph metrics,
+config). It scores every candidate, allocates a share of the global
+budget to each page-type bucket, and returns the allow-set.
+
+Import direction (one-way):
+ ingestion.models ← generation.models ← selection
+"""
+
+from .budget import BucketAllocation, allocate_budget
+from .scoring import (
+ score_api_contract,
+ score_file,
+ score_infra,
+ score_module,
+ score_scc,
+ score_symbol,
+)
+from .selector import (
+ ModuleGroup,
+ Selection,
+ SelectionInputs,
+ select_pages,
+ summarize_selection,
+)
+
+__all__ = [
+ "BucketAllocation",
+ "ModuleGroup",
+ "Selection",
+ "SelectionInputs",
+ "allocate_budget",
+ "score_api_contract",
+ "score_file",
+ "score_infra",
+ "score_module",
+ "score_scc",
+ "score_symbol",
+ "select_pages",
+ "summarize_selection",
+]
diff --git a/packages/core/src/repowise/core/generation/selection/budget.py b/packages/core/src/repowise/core/generation/selection/budget.py
new file mode 100644
index 000000000..468bc42f0
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/selection/budget.py
@@ -0,0 +1,154 @@
+"""Budget allocation across page-type buckets.
+
+Pure functions — no I/O, no dependency on ParsedFile or graph types.
+Given ``coverage_pct`` and per-bucket shares, compute how many pages of
+each type to emit.
+
+Design choices
+--------------
+- **No absolute cap.** Budget = ``int(N_files * coverage_pct)``. The user
+ chose the coverage percentage; we honor it even on huge repos.
+- **Per-bucket floors** ensure tiny repos still get at least one
+ ``repo_overview`` / ``architecture_diagram`` / onboarding slot.
+- **Unused share spills into ``file_page``.** When a bucket can't fill
+ its share (e.g. only 2 SCCs exist but share gives 5), the difference
+ is reallocated to the file_page bucket — file pages are the most
+ flexible bucket.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+# Page-type identifiers handled by the budget. Onboarding is curated and
+# sized by ``len(specs)``; it is not budgeted here.
+BUCKET_TYPES = (
+ "file_page",
+ "symbol_spotlight",
+ "module_page",
+ "api_contract",
+ "infra_page",
+ "scc_page",
+)
+
+# Per-bucket floor — minimum pages emitted when the share would round to
+# below this and at least one candidate exists. ``api_contract``,
+# ``infra_page``, and ``scc_page`` are inherently small, bounded buckets
+# (a repo has at most a handful of each); when candidates are present
+# we always emit at least one so important infra/API/cycle documentation
+# isn't lost to percentage rounding. ``repo_overview`` +
+# ``architecture_diagram`` are not budgeted (always 1 each).
+_BUCKET_FLOOR: dict[str, int] = {
+ "file_page": 0,
+ "symbol_spotlight": 0,
+ "module_page": 0,
+ "api_contract": 1,
+ "infra_page": 1,
+ "scc_page": 1,
+}
+
+
+@dataclass(frozen=True)
+class BucketAllocation:
+ """Per-bucket page count target.
+
+ Caller picks the top-K candidates from each bucket where K equals
+ the matching field. ``file_page`` absorbs any leftover share that
+ other buckets couldn't fill.
+ """
+
+ file_page: int
+ symbol_spotlight: int
+ module_page: int
+ api_contract: int
+ infra_page: int
+ scc_page: int
+
+ @property
+ def total(self) -> int:
+ return (
+ self.file_page
+ + self.symbol_spotlight
+ + self.module_page
+ + self.api_contract
+ + self.infra_page
+ + self.scc_page
+ )
+
+
+# Repos with fewer files than this are considered "small" — the
+# coverage percentage rounds badly, so every available bucket gets at
+# least one page instead of being squeezed out by share arithmetic.
+SMALL_REPO_THRESHOLD = 20
+
+
+def compute_budget(n_files: int, coverage_pct: float) -> int:
+ """Return the global page budget for a repo of ``n_files``.
+
+ Pure percentage; no absolute cap. For repos at or below
+ :data:`SMALL_REPO_THRESHOLD` files, the budget floors at ``n_files``
+ so percentage rounding never zeros out tiny repos.
+ """
+ if n_files <= 0:
+ return 0
+ pct = max(0.0, min(coverage_pct, 1.0))
+ raw = int(n_files * pct)
+ if n_files <= SMALL_REPO_THRESHOLD:
+ return max(raw, n_files)
+ return raw
+
+
+def allocate_budget(
+ *,
+ budget: int,
+ candidates_per_bucket: dict[str, int],
+ shares: dict[str, float],
+ n_files: int = 0,
+) -> BucketAllocation:
+ """Split *budget* into per-bucket targets.
+
+ Parameters
+ ----------
+ budget:
+ Total page budget (see :func:`compute_budget`).
+ candidates_per_bucket:
+ Number of *available* candidates per bucket. A bucket's target
+ is capped at its candidate count — no point allocating 20 infra
+ pages when only 7 infra files exist.
+ shares:
+ Fractional share of the budget per bucket. Keys must include
+ every entry in :data:`BUCKET_TYPES`. Shares need not sum to 1;
+ any leftover budget after capping spills into ``file_page``.
+ """
+ # For small repos, every bucket with at least one candidate gets
+ # at least one page — this avoids percentage rounding zeroing out
+ # buckets when the budget is tiny.
+ small_repo_floor = 1 if 0 < n_files <= SMALL_REPO_THRESHOLD else 0
+
+ raw: dict[str, int] = {}
+ for bucket in BUCKET_TYPES:
+ share = max(0.0, shares.get(bucket, 0.0))
+ target = int(round(budget * share))
+ target = max(target, _BUCKET_FLOOR[bucket])
+ available = candidates_per_bucket.get(bucket, 0)
+ if available > 0:
+ target = max(target, small_repo_floor)
+ target = min(target, available)
+ raw[bucket] = target
+
+ # Spill unused share into file_page (the most flexible bucket).
+ spent = sum(raw.values())
+ spill = max(0, budget - spent)
+ if spill > 0:
+ max_file = candidates_per_bucket.get("file_page", 0)
+ raw["file_page"] = min(max_file, raw["file_page"] + spill)
+
+ return BucketAllocation(
+ file_page=raw["file_page"],
+ symbol_spotlight=raw["symbol_spotlight"],
+ module_page=raw["module_page"],
+ api_contract=raw["api_contract"],
+ infra_page=raw["infra_page"],
+ scc_page=raw["scc_page"],
+ )
diff --git a/packages/core/src/repowise/core/generation/selection/scoring.py b/packages/core/src/repowise/core/generation/selection/scoring.py
new file mode 100644
index 000000000..721d7f719
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/selection/scoring.py
@@ -0,0 +1,170 @@
+"""Per-candidate scoring functions.
+
+Each scorer takes a candidate plus the precomputed graph metrics and
+returns a float in roughly ``[0, 2]``. Higher = more important.
+
+Scoring is intentionally simple and explainable. The previous bypass
+rules in ``_is_significant_file`` (test files, ``__init__.py``,
+``betweenness > 0``, ``entry_point``) are folded in here as **bonuses**,
+not hard overrides — high-value files still compete against each other
+for the bucket's slot count.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+# -- File score --------------------------------------------------------
+
+# Bonuses applied additively on top of normalized PageRank+betweenness.
+# Tuned so a file with even one strong signal scores above a generic
+# mid-PageRank file with no other characteristics.
+_BONUS_ENTRY_POINT = 0.40
+_BONUS_HOTSPOT = 0.20
+_BONUS_INIT_PY_RE_EXPORTER = 0.15
+_BONUS_BETWEENNESS_BRIDGE = 0.10 # any bridge file (bet > 0)
+
+# Penalties — applied multiplicatively at the end. Keeps tests/trivial
+# files in the pool but ranks them below substantive code unless they
+# have countervailing signals.
+_PENALTY_TEST = 0.60
+_PENALTY_TRIVIAL_SIZE_BYTES = 4000
+_PENALTY_TRIVIAL_SYMBOL_CAP = 4
+_PENALTY_TRIVIAL = 0.40
+
+
+def _normalize(value: float, max_value: float) -> float:
+ """Return ``value / max_value`` clamped to ``[0, 1]``; safe on 0."""
+ if max_value <= 0:
+ return 0.0
+ return max(0.0, min(value / max_value, 1.0))
+
+
+def score_file(
+ parsed: Any,
+ *,
+ pagerank: float,
+ betweenness: float,
+ max_pagerank: float,
+ max_betweenness: float,
+ is_hotspot: bool,
+) -> float:
+ """Score a code file for ``file_page`` candidacy.
+
+ ``parsed`` is a :class:`ParsedFile`. Files with zero symbols and no
+ entry/hotspot signal return 0.0 — they will never be selected.
+ """
+ fi = parsed.file_info
+ n_symbols = len(parsed.symbols)
+
+ # Files with no symbols and no architectural signal are noise.
+ if n_symbols == 0 and not fi.is_entry_point and not is_hotspot:
+ return 0.0
+
+ base = _normalize(pagerank, max_pagerank) + _normalize(betweenness, max_betweenness) * 0.5
+
+ if fi.is_entry_point:
+ base += _BONUS_ENTRY_POINT
+ if is_hotspot:
+ base += _BONUS_HOTSPOT
+ if betweenness > 0 and not fi.is_entry_point:
+ base += _BONUS_BETWEENNESS_BRIDGE
+ if fi.path.endswith("__init__.py") and n_symbols >= 2:
+ base += _BONUS_INIT_PY_RE_EXPORTER
+
+ # Penalties.
+ if fi.is_test:
+ base *= _PENALTY_TEST
+ if (
+ n_symbols <= _PENALTY_TRIVIAL_SYMBOL_CAP
+ and fi.size_bytes < _PENALTY_TRIVIAL_SIZE_BYTES
+ and not fi.is_entry_point
+ ):
+ base *= _PENALTY_TRIVIAL
+
+ return base
+
+
+# -- Symbol score ------------------------------------------------------
+
+# Kind weights — bias symbol spotlights toward functions/classes over
+# variables/constants.
+_KIND_WEIGHT = {
+ "function": 1.0,
+ "method": 0.9,
+ "class": 1.0,
+ "interface": 0.9,
+ "struct": 0.8,
+ "enum": 0.7,
+ "type_alias": 0.5,
+ "variable": 0.3,
+ "constant": 0.4,
+}
+
+
+def score_symbol(symbol: Any, file_pagerank: float, max_pagerank: float) -> float:
+ """Score a public symbol for ``symbol_spotlight`` candidacy."""
+ if symbol.visibility != "public":
+ return 0.0
+ weight = _KIND_WEIGHT.get(str(symbol.kind), 0.5)
+ return _normalize(file_pagerank, max_pagerank) * weight
+
+
+# -- Module score ------------------------------------------------------
+
+
+def score_module(*, size: int, cohesion: float, min_module_size: int) -> float:
+ """Score a community/directory cluster for ``module_page`` candidacy.
+
+ Communities below ``min_module_size`` score 0 — they fold into the
+ parent directory's module page (or are dropped if there is no
+ parent).
+ """
+ if size < min_module_size:
+ return 0.0
+ # Cohesion is typically [0, 1]. Multiply by log-ish size factor.
+ return (size**0.5) * (0.5 + cohesion)
+
+
+# -- SCC score ---------------------------------------------------------
+
+
+def score_scc(*, cycle_size: int) -> float:
+ """Score a strongly-connected component for ``scc_page`` candidacy.
+
+ Two-file cycles are easy refactors and score low; large cycles
+ score high because they are the most architecturally interesting.
+ """
+ if cycle_size <= 1:
+ return 0.0
+ return float(cycle_size)
+
+
+# -- API contract score ------------------------------------------------
+
+
+def score_api_contract(parsed: Any) -> float:
+ """Score an API contract file. Bigger surface = higher score."""
+ # Approximate: number of public symbols + a small file-size factor.
+ n_public = sum(1 for s in parsed.symbols if s.visibility == "public")
+ size_kb = max(1, parsed.file_info.size_bytes // 1024)
+ return float(n_public) + min(size_kb / 10.0, 5.0)
+
+
+# -- Infra score -------------------------------------------------------
+
+
+def score_infra(parsed: Any) -> float:
+ """Score an infrastructure file.
+
+ Larger / more-referenced files (Dockerfile vs a one-line .env)
+ score higher.
+ """
+ size_kb = max(1, parsed.file_info.size_bytes // 1024)
+ name = Path(parsed.file_info.path).name
+ # Boost canonical roots: a project's main Dockerfile/Makefile
+ # should always rank above sibling override files.
+ boost = 0.5 if name in {"Dockerfile", "Makefile", "GNUmakefile"} else 0.0
+ return min(size_kb / 5.0, 5.0) + boost
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
new file mode 100644
index 000000000..ddbb31b64
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -0,0 +1,374 @@
+"""End-to-end page selection.
+
+The single ``select_pages`` entry point returns an allow-set that both
+``PageGenerator.generate_all`` and ``cost_estimator.build_generation_plan``
+honor verbatim. No bypass paths — if a candidate isn't here, it isn't
+emitted.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+
+from .budget import BucketAllocation, allocate_budget, compute_budget
+from .scoring import (
+ score_api_contract,
+ score_file,
+ score_infra,
+ score_module,
+ score_scc,
+ score_symbol,
+)
+
+log = structlog.get_logger(__name__)
+
+_INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()
+_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})
+_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()
+
+
+# ---------------------------------------------------------------------------
+# Output dataclasses
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class ModuleGroup:
+ """One ``module_page`` worth of files.
+
+ ``key`` is the stable identifier persisted as ``target_path``:
+ ``community-<id>`` when grouping by community, the top-level
+ directory when falling back to ``top_dir``.
+ """
+
+ key: str
+ display: str
+ language: str
+ file_paths: tuple[str, ...]
+ label: str | None = None
+ cohesion: float | None = None
+
+
+@dataclass
+class Selection:
+ """Allow-set returned by :func:`select_pages`."""
+
+ file_page_paths: list[str] = field(default_factory=list)
+ symbol_spotlights: list[tuple[str, str]] = field(default_factory=list) # (file_path, symbol_name)
+ module_groups: list[ModuleGroup] = field(default_factory=list)
+ api_contract_paths: list[str] = field(default_factory=list)
+ infra_paths: list[str] = field(default_factory=list)
+ scc_groups: list[tuple[str, list[str]]] = field(default_factory=list) # (scc_id, files)
+ emit_repo_overview: bool = True
+ emit_arch_diagram: bool = True
+ allocation: BucketAllocation | None = None
+
+ def counts(self) -> dict[str, int]:
+ """Per-page-type counts (for cost estimation and the init UI)."""
+ return {
+ "api_contract": len(self.api_contract_paths),
+ "symbol_spotlight": len(self.symbol_spotlights),
+ "file_page": len(self.file_page_paths),
+ "scc_page": len(self.scc_groups),
+ "module_page": len(self.module_groups),
+ "repo_overview": int(self.emit_repo_overview),
+ "architecture_diagram": int(self.emit_arch_diagram),
+ "infra_page": len(self.infra_paths),
+ }
+
+
+# ---------------------------------------------------------------------------
+# Input bundle
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SelectionInputs:
+ """All inputs the selector needs.
+
+ Bundling them in one dataclass keeps the public signature small —
+ both ``PageGenerator`` and the cost estimator construct one of
+ these and hand it to :func:`select_pages`.
+ """
+
+ parsed_files: list[Any]
+ pagerank: dict[str, float]
+ betweenness: dict[str, float]
+ community: dict[str, int]
+ community_info: dict[int, Any] | None # cid → CommunityInfo (label, cohesion)
+ sccs: list[Any]
+ git_meta_map: dict[str, dict] | None
+ config: Any # GenerationConfig — duck-typed to avoid the import cycle
+
+
+# ---------------------------------------------------------------------------
+# Helpers — file classification
+# ---------------------------------------------------------------------------
+
+
+def _is_infra_file(parsed: Any) -> bool:
+ fi = parsed.file_info
+ if fi.language in _INFRA_LANGUAGES:
+ return True
+ return Path(fi.path).name in _INFRA_FILENAMES
+
+
+def _is_code_file(parsed: Any) -> bool:
+ fi = parsed.file_info
+ return (
+ not fi.is_api_contract
+ and not _is_infra_file(parsed)
+ and fi.language in _CODE_LANGUAGES
+ )
+
+
+# ---------------------------------------------------------------------------
+# Helpers — bucket candidate building
+# ---------------------------------------------------------------------------
+
+
+def _build_file_candidates(
+ inputs: SelectionInputs,
+) -> list[tuple[float, str]]:
+ """Return ``[(score, file_path), ...]`` for code files, descending."""
+ max_pr = max(inputs.pagerank.values(), default=0.0)
+ max_bet = max(inputs.betweenness.values(), default=0.0)
+ git = inputs.git_meta_map or {}
+
+ scored: list[tuple[float, str]] = []
+ for p in inputs.parsed_files:
+ if not _is_code_file(p):
+ continue
+ path = p.file_info.path
+ is_hotspot = bool(git.get(path, {}).get("is_hotspot", False))
+ s = score_file(
+ p,
+ pagerank=inputs.pagerank.get(path, 0.0),
+ betweenness=inputs.betweenness.get(path, 0.0),
+ max_pagerank=max_pr,
+ max_betweenness=max_bet,
+ is_hotspot=is_hotspot,
+ )
+ if s > 0.0:
+ scored.append((s, path))
+ scored.sort(key=lambda x: x[0], reverse=True)
+ return scored
+
+
+def _build_symbol_candidates(
+ inputs: SelectionInputs,
+) -> list[tuple[float, tuple[str, str]]]:
+ """Return ``[(score, (file_path, symbol_name)), ...]`` descending."""
+ max_pr = max(inputs.pagerank.values(), default=0.0)
+ scored: list[tuple[float, tuple[str, str]]] = []
+ for p in inputs.parsed_files:
+ file_pr = inputs.pagerank.get(p.file_info.path, 0.0)
+ for sym in p.symbols:
+ if sym.visibility != "public":
+ continue
+ s = score_symbol(sym, file_pr, max_pr)
+ if s > 0.0:
+ scored.append((s, (p.file_info.path, sym.name)))
+ scored.sort(key=lambda x: x[0], reverse=True)
+ return scored
+
+
+def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGroup]]:
+ """Return scored module groups — either community-based or top-dir."""
+ cfg = inputs.config
+ min_size = max(1, getattr(cfg, "min_module_size", 3))
+ use_communities = getattr(cfg, "module_grouping", "community") == "community"
+
+ # Bucket files into groups.
+ groups: dict[str, list[Any]] = {}
+ group_lang: dict[str, str] = {}
+ group_label: dict[str, str | None] = {}
+ group_cohesion: dict[str, float | None] = {}
+ group_display: dict[str, str] = {}
+
+ if use_communities and inputs.community_info:
+ for p in inputs.parsed_files:
+ if not _is_code_file(p):
+ continue
+ cid = inputs.community.get(p.file_info.path)
+ if cid is None:
+ continue
+ key = f"community-{cid}"
+ groups.setdefault(key, []).append(p)
+ group_lang.setdefault(key, p.file_info.language)
+ if key not in group_display:
+ ci = inputs.community_info.get(cid)
+ label = getattr(ci, "label", "") or f"cluster_{cid}"
+ group_display[key] = label
+ group_label[key] = label
+ group_cohesion[key] = float(getattr(ci, "cohesion", 0.0) or 0.0)
+ else:
+ for p in inputs.parsed_files:
+ if not _is_code_file(p):
+ continue
+ parts = Path(p.file_info.path).parts
+ key = parts[0] if len(parts) > 1 else "root"
+ groups.setdefault(key, []).append(p)
+ group_lang.setdefault(key, p.file_info.language)
+ group_display.setdefault(key, key)
+ group_label.setdefault(key, None)
+ group_cohesion.setdefault(key, None)
+
+ scored: list[tuple[float, ModuleGroup]] = []
+ for key, files in groups.items():
+ s = score_module(
+ size=len(files),
+ cohesion=group_cohesion.get(key) or 0.0,
+ min_module_size=min_size,
+ )
+ if s <= 0.0:
+ continue
+ scored.append(
+ (
+ s,
+ ModuleGroup(
+ key=key,
+ display=group_display.get(key, key),
+ language=group_lang.get(key, "unknown"),
+ file_paths=tuple(p.file_info.path for p in files),
+ label=group_label.get(key),
+ cohesion=group_cohesion.get(key),
+ ),
+ )
+ )
+ scored.sort(key=lambda x: x[0], reverse=True)
+ return scored
+
+
+def _build_api_candidates(inputs: SelectionInputs) -> list[tuple[float, str]]:
+ scored: list[tuple[float, str]] = []
+ for p in inputs.parsed_files:
+ if not p.file_info.is_api_contract:
+ continue
+ scored.append((score_api_contract(p), p.file_info.path))
+ scored.sort(key=lambda x: x[0], reverse=True)
+ return scored
+
+
+def _build_infra_candidates(inputs: SelectionInputs) -> list[tuple[float, str]]:
+ scored: list[tuple[float, str]] = []
+ for p in inputs.parsed_files:
+ if not _is_infra_file(p):
+ continue
+ scored.append((score_infra(p), p.file_info.path))
+ scored.sort(key=lambda x: x[0], reverse=True)
+ return scored
+
+
+def _build_scc_candidates(
+ inputs: SelectionInputs,
+) -> list[tuple[float, tuple[str, list[str]]]]:
+ scored: list[tuple[float, tuple[str, list[str]]]] = []
+ for i, scc in enumerate(inputs.sccs):
+ files = list(scc)
+ s = score_scc(cycle_size=len(files))
+ if s <= 0.0:
+ continue
+ scored.append((s, (f"scc-{i}", sorted(files))))
+ scored.sort(key=lambda x: x[0], reverse=True)
+ return scored
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def _shares_from_config(cfg: Any) -> dict[str, float]:
+ return {
+ "file_page": getattr(cfg, "file_page_share", 0.50),
+ "symbol_spotlight": getattr(cfg, "symbol_spotlight_share", 0.15),
+ "module_page": getattr(cfg, "module_page_share", 0.10),
+ "api_contract": getattr(cfg, "api_contract_share", 0.08),
+ "infra_page": getattr(cfg, "infra_page_share", 0.05),
+ "scc_page": getattr(cfg, "scc_share", 0.04),
+ }
+
+
+def _coverage_pct(cfg: Any) -> float:
+ """Read ``coverage_pct``, falling back to legacy ``max_pages_pct``."""
+ return float(getattr(cfg, "coverage_pct", None) or getattr(cfg, "max_pages_pct", 0.20))
+
+
+def select_pages(inputs: SelectionInputs) -> Selection:
+ """Return the allow-set of pages to generate for one run.
+
+ Deterministic given identical inputs. Safe to call from both the
+ generator and the cost estimator.
+ """
+ cfg = inputs.config
+ pct = _coverage_pct(cfg)
+ budget = compute_budget(len(inputs.parsed_files), pct)
+
+ # Build scored candidates for every bucket.
+ files = _build_file_candidates(inputs)
+ symbols = _build_symbol_candidates(inputs)
+ modules = _build_module_groups(inputs)
+ apis = _build_api_candidates(inputs)
+ infras = _build_infra_candidates(inputs)
+ sccs = _build_scc_candidates(inputs)
+
+ available = {
+ "file_page": len(files),
+ "symbol_spotlight": len(symbols),
+ "module_page": len(modules),
+ "api_contract": len(apis),
+ "infra_page": len(infras),
+ "scc_page": len(sccs),
+ }
+
+ allocation = allocate_budget(
+ budget=budget,
+ candidates_per_bucket=available,
+ shares=_shares_from_config(cfg),
+ n_files=len(inputs.parsed_files),
+ )
+
+ sel = Selection(
+ file_page_paths=[p for _, p in files[: allocation.file_page]],
+ symbol_spotlights=[t for _, t in symbols[: allocation.symbol_spotlight]],
+ module_groups=[m for _, m in modules[: allocation.module_page]],
+ api_contract_paths=[p for _, p in apis[: allocation.api_contract]],
+ infra_paths=[p for _, p in infras[: allocation.infra_page]],
+ scc_groups=[g for _, g in sccs[: allocation.scc_page]],
+ emit_repo_overview=True,
+ emit_arch_diagram=True,
+ allocation=allocation,
+ )
+
+ log.info(
+ "page_selection.complete",
+ coverage_pct=pct,
+ budget=budget,
+ counts=sel.counts(),
+ )
+ return sel
+
+
+def summarize_selection(sel: Selection) -> dict[str, int]:
+ """Convenience wrapper returning the counts dict.
+
+ Kept as a separate helper so the init UI can hand a Selection
+ directly to its rendering layer without depending on the dataclass
+ internals.
+ """
+ return sel.counts()
+
+
+def language_summary(parsed_files: list[Any]) -> dict[str, int]:
+ """Return ``{language: file_count}`` — used by the init UI to
+ describe the repo shape next to the coverage table.
+ """
+ return dict(Counter(p.file_info.language for p in parsed_files))
diff --git a/packages/core/src/repowise/core/generation/templates/cross_package.j2 b/packages/core/src/repowise/core/generation/templates/cross_package.j2
deleted file mode 100644
index 6fe6d4ab0..000000000
--- a/packages/core/src/repowise/core/generation/templates/cross_package.j2
+++ /dev/null
@@ -1,18 +0,0 @@
-You are documenting the inter-package dependency: `{{ ctx.source_package }}` → `{{ ctx.target_package }}`.
-
-- **Source**: `{{ ctx.source_package }}` (mean PageRank: {{ "%.4f"|format(ctx.source_pagerank_mean) }})
-- **Target**: `{{ ctx.target_package }}` (mean PageRank: {{ "%.4f"|format(ctx.target_pagerank_mean) }})
-- **Coupling**: {{ ctx.coupling_strength }} files in source depend on target
-
-## Files Crossing the Boundary
-{% for f in ctx.boundary_files %}
-- `{{ f }}`
-{% endfor %}
-
-## Public API Used from `{{ ctx.target_package }}`
-{% for sym in ctx.used_symbols %}
-- `{{ sym }}`
-{% endfor %}
-
-Generate a complete wiki page with sections:
-## Overview, ## Integration Points, ## API Dependency Summary, ## Coupling Risk.
diff --git a/packages/core/src/repowise/core/generation/templates/diff_summary.j2 b/packages/core/src/repowise/core/generation/templates/diff_summary.j2
deleted file mode 100644
index 0b4079723..000000000
--- a/packages/core/src/repowise/core/generation/templates/diff_summary.j2
+++ /dev/null
@@ -1,46 +0,0 @@
-You are summarizing changes between `{{ ctx.from_ref }}` and `{{ ctx.to_ref }}`.
-
-## Change Summary
-- **From**: `{{ ctx.from_ref }}`
-- **To**: `{{ ctx.to_ref }}`
-
-## Added Files ({{ ctx.added_files | length }})
-{% for f in ctx.added_files %}
-- `{{ f }}`
-{% endfor %}
-
-## Deleted Files ({{ ctx.deleted_files | length }})
-{% for f in ctx.deleted_files %}
-- `{{ f }}`
-{% endfor %}
-
-## Modified Files ({{ ctx.modified_files | length }})
-{% for f in ctx.modified_files %}
-- `{{ f }}`
-{% endfor %}
-
-## Symbol Changes
-{% for diff in ctx.symbol_diffs %}
-- **{{ diff.path }}**: +{{ diff.added | length }} added, -{{ diff.removed | length }} removed, {{ diff.renamed | length }} renamed
-{% endfor %}
-
-## Affected Documentation Pages
-{% for page_id in ctx.affected_page_ids %}
-- `{{ page_id }}`
-{% endfor %}
-
-{% if ctx.trigger_commit_sha is defined and ctx.trigger_commit_sha %}
-## Trigger Commit
-- **SHA**: `{{ ctx.trigger_commit_sha }}`
-{% if ctx.trigger_commit_message %}- **Message**: {{ ctx.trigger_commit_message }}{% endif %}
-{% if ctx.trigger_commit_author %}- **Author**: {{ ctx.trigger_commit_author }}{% endif %}
-{% endif %}
-
-{% if ctx.diff_text is defined and ctx.diff_text %}
-## Diff Context
-```
-{{ ctx.diff_text }}
-```
-{% endif %}
-
-Generate a complete wiki page following the sections above.
diff --git a/packages/core/src/repowise/core/generation/templates/file_page.j2 b/packages/core/src/repowise/core/generation/templates/file_page.j2
index dd27ebffc..c5f9a5a88 100644
--- a/packages/core/src/repowise/core/generation/templates/file_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/file_page.j2
@@ -143,6 +143,14 @@ This file belongs to the **{{ ctx.community_label }}** community{% if ctx.commun
Use this to explain the file's role within its architectural cluster.
{% endif %}
+{% if ctx.decision_records %}
+## Architectural decisions affecting this file
+Use these as the source of truth for the "why" when explaining design choices below.
+{% for dr in ctx.decision_records[:5] %}
+- **{{ dr.title }}** — {{ dr.decision }}{% if dr.rationale %} *(Why: {{ dr.rationale }})*{% endif %}
+{% endfor %}
+{% endif %}
+
{% if ctx.dead_code_findings %}
## Potentially unused code
{% for finding in ctx.dead_code_findings %}
diff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2
index 4927fc750..4c5349452 100644
--- a/packages/core/src/repowise/core/generation/templates/module_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2
@@ -1,57 +1,87 @@
-You are documenting the module at `{{ ctx.module_path }}` (language: {{ ctx.language }}).
+You are documenting the module **{{ ctx.module_path }}** (language: {{ ctx.language }}).
## Module Overview
-- **Path**: `{{ ctx.module_path }}`
+- **Label**: {{ ctx.module_path }}
- **Language**: {{ ctx.language }}
-- **Total symbols**: {{ ctx.total_symbols }}
-- **Public symbols**: {{ ctx.public_symbols }}
+- **Files in module**: {{ ctx.files | length }}
+- **Total symbols**: {{ ctx.total_symbols }} ({{ ctx.public_symbols }} public)
- **Mean PageRank**: {{ "%.4f"|format(ctx.pagerank_mean) }}
+{% if ctx.community_label %}
+- **Community label**: {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}
+{% endif %}
+{% if ctx.entry_points %}
## Entry Points
{% for ep in ctx.entry_points %}
- `{{ ep }}`
{% endfor %}
+{% endif %}
+
+{% if ctx.key_files %}
+## Key Files
+The highest-PageRank files in this module — the ones to read first when learning the area.
+{% for kf in ctx.key_files %}
+- `{{ kf.path }}`{% if kf.is_entry_point %} *(entry point)*{% endif %}{% if kf.summary %} — {{ kf.summary }}{% endif %}
+{% endfor %}
+{% endif %}
+
+{% if ctx.decision_records %}
+## Architectural decisions
+Use these to explain *why* the module is shaped the way it is. Do not invent rationale beyond what is listed.
+{% for dr in ctx.decision_records[:5] %}
+- **{{ dr.title }}** — {{ dr.decision }}{% if dr.rationale %} *(Why: {{ dr.rationale }})*{% endif %}
+{% endfor %}
+{% endif %}
+{% if ctx.dependencies %}
## Dependencies (modules this imports)
-{% for dep in ctx.dependencies %}
+{% for dep in ctx.dependencies[:25] %}
- `{{ dep }}`
{% endfor %}
+{% endif %}
+{% if ctx.dependents %}
## Dependents (modules that import this)
-{% for dep in ctx.dependents %}
+{% for dep in ctx.dependents[:25] %}
- `{{ dep }}`
{% endfor %}
-
-{% if ctx.community_label %}
-## Architectural Community
-This module belongs to the **{{ ctx.community_label }}** cluster{% if ctx.community_cohesion > 0 %} (cohesion: {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}.
{% endif %}
-## Files in Module
-{% for f in ctx.files %}
-{% set summary = ctx.file_summaries.get(f, '') %}
-{% if summary %}
-- `{{ f }}` — {{ summary }}
-{% else %}
-- `{{ f }}`
+{% if ctx.external_systems %}
+## External systems used
+{% for sys in ctx.external_systems[:15] %}
+- `{{ sys.name }}`{% if sys.category and sys.category != 'library' %} *({{ sys.category }})*{% endif %}{% if sys.ecosystem %} — {{ sys.ecosystem }}{% endif %}
+{% endfor %}
{% endif %}
+
+{% if ctx.top_owners %}
+## Ownership
+{% for o in ctx.top_owners %}
+- **{{ o.name }}** — primary maintainer of {{ o.file_count }} file(s)
{% endfor %}
+{% endif %}
{% if ctx.key_classes %}
## Class Hierarchy
{% for h in ctx.key_classes %}
- `{{ h.child }}` {{ h.kind }} `{{ h.parent }}`{% if h.parent_file is defined and h.parent_file %} (in `{{ h.parent_file }}`){% endif %}
+{% endfor %}
+{% endif %}
+{% if ctx.dead_code_findings %}
+## Potentially unused code in this module
+{% for finding in ctx.dead_code_findings[:10] %}
+- `{{ finding.symbol_name or '?' }}` — {{ finding.reason }} ({{ (finding.confidence * 100) | int }}%{% if finding.safe_to_delete %}, safe to remove{% endif %})
{% endfor %}
{% endif %}
{% if module_git_summary is defined and module_git_summary %}
-## Team ownership
-{% for pkg_owner in module_git_summary.top_owners %}
-- **{{ pkg_owner.file_count }} files** primarily maintained by {{ pkg_owner.name }}
-{% endfor %}
-Most active recently: `{{ module_git_summary.most_active_file }}`
+## Recent activity
+Most active file: `{{ module_git_summary.most_active_file }}`
({{ module_git_summary.most_active_commits_90d }} commits in 90 days)
{% endif %}
-Generate a complete wiki page following the sections above.
+Generate a complete wiki page following the sections above. Required sections in the
+output: ## Overview, ## Public API Summary, ## Architecture Notes. Use the supplied
+key files, decisions, and external systems to ground the narrative; do not fabricate
+modules, files, or rationale that are not listed here.
diff --git a/packages/core/src/repowise/core/generation/templates/onboarding/active_landscape.j2 b/packages/core/src/repowise/core/generation/templates/onboarding/active_landscape.j2
new file mode 100644
index 000000000..4e1c58d62
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/onboarding/active_landscape.j2
@@ -0,0 +1,51 @@
+You are writing the "Active Landscape" page for the repository
+`{{ ctx.repo_name }}` — a snapshot of where work is happening right now,
+intended for a contributor about to pick up their first task.
+
+## Activity signal (do not invent beyond these numbers)
+- **Total commits (90 days)**: {{ ctx.total_commits_90d }}
+- **Files touched (90 days)**: {{ ctx.files_touched_90d }}
+- **Stable files** (untouched 90+ days): {{ ctx.stable_file_count }}
+
+## Hot files (top by 90-day churn)
+{% for f in ctx.hot_files %}
+- `{{ f.path }}` — {{ f.commit_count_90d }} commits{% if f.primary_owner %}, primary owner {{ f.primary_owner }}{% endif %}{% if f.is_hotspot %} *(flagged hotspot)*{% endif %}
+{% endfor %}
+
+{% if ctx.hot_dirs %}
+## Hot directories
+{% for d in ctx.hot_dirs %}
+- `{{ d.path }}/` — {{ d.total_commits_90d }} commits across {{ d.file_count }} files{% if d.hotspot_count %}, {{ d.hotspot_count }} hotspot(s){% endif %}
+{% endfor %}
+{% endif %}
+
+{% if ctx.dead_code_in_hot_files %}
+## Dead-code findings inside hot files (real callouts)
+{% for f in ctx.dead_code_in_hot_files %}
+- `{{ f.file_path }}` — {{ f.symbol_kind }} `{{ f.symbol_name }}` ({{ f.reason }}, confidence {{ f.confidence }})
+{% endfor %}
+{% endif %}
+
+## What to write
+Produce an Active Landscape markdown page. Required sections, in order:
+
+1. **## Where work is concentrated** — 2–3 sentences naming the hottest
+ directories and what kind of work appears to be driving the churn
+ (refactor, feature work, bug-chasing). Ground every claim in the file
+ and directory data above. Do not invent commit messages or features.
+2. **## Files in motion** — bulleted list of the hot files with a one-line
+ note each: who tends to touch it and one neutral observation about why
+ it's hot (e.g. "central abstraction — most features funnel through it",
+ "currently being refactored", "test churn matches feature churn"). If
+ you're not sure, say so — do not fabricate intent.
+3. **## Steady ground** — 1–2 sentences identifying the stable areas (use
+ the stable_file_count above). The page should leave the reader with a
+ sense of which corners of the codebase are *safe to learn first* before
+ touching the hot zones.
+4. **## Watch-outs** — bulleted callouts based on the dead-code findings
+ above and any hotspot files. If there are no findings, write a single
+ line: "No live dead-code findings in the current hot set."
+
+Tone: matter-of-fact, like an on-call handoff. No marketing language.
+
+Output markdown only. No preamble, no apologies, no closing summary.
diff --git a/packages/core/src/repowise/core/generation/templates/onboarding/codebase_map.j2 b/packages/core/src/repowise/core/generation/templates/onboarding/codebase_map.j2
new file mode 100644
index 000000000..754ea68ae
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/onboarding/codebase_map.j2
@@ -0,0 +1,56 @@
+You are writing the "Codebase Map" page for the repository `{{ ctx.repo_name }}`.
+
+This is the wayfinding page — where things physically live. The reader has
+already read the Architecture Guide (which is conceptual); this page is
+spatial. The right tone is: "if you're looking for X, go to Y."
+
+## Repo facts (do not invent beyond these)
+- **Total files**: {{ ctx.total_files }}
+- **Total lines of code**: {{ ctx.total_loc }}
+- **Top-level directories surveyed**: {{ ctx.directories | length }}
+
+{% if ctx.entry_points %}
+## Repo-wide entry points
+{% for ep in ctx.entry_points %}
+- `{{ ep }}`
+{% endfor %}
+{% endif %}
+
+## Top-level directories
+{% for d in ctx.directories %}
+### `{{ d.path }}/` ({{ d.file_count }} files, mostly {{ d.dominant_language }})
+{% if d.community_labels %}
+- Architectural clusters present: {{ d.community_labels | join(", ") }}
+{% endif %}
+{% if d.entry_points %}
+- Entry points: {% for ep in d.entry_points %}`{{ ep }}`{% if not loop.last %}, {% endif %}{% endfor %}
+{% endif %}
+{% if d.top_files %}
+- Key files (by PageRank):
+{% for f in d.top_files %}
+ - `{{ f }}`
+{% endfor %}
+{% endif %}
+
+{% endfor %}
+
+## What to write
+Produce a Codebase Map markdown page. Required sections, in this order:
+
+1. **## Overview** — one paragraph: the shape of the file tree at a glance,
+ how many top-level directories matter, and which one a new contributor
+ should explore first. Ground every claim in the facts above.
+2. **## Top-level directories** — one short subsection per directory listed
+ above (heading: `### \`<path>/\``). Each subsection should answer two
+ questions in 2–4 sentences: (a) what kind of code lives here, (b) the
+ first file or two to read. Reference the listed entry points and key
+ files. Do not invent files that aren't in the data.
+3. **## If you're looking for…** — a bulleted lookup list of 5–8 entries
+ that map common questions to directories. Examples to model on (adapt
+ to what's actually here): "the HTTP layer → `server/`", "tests → mirror
+ the layout under `tests/`", "the public API surface → `<top file>`".
+4. **## What's not here** — one short paragraph naming up to three
+ directories that exist in the tree but are *not* the primary place a
+ contributor should start (build infrastructure, vendor folders, etc.).
+
+Output markdown only. No preamble, no apologies, no closing summary.
diff --git a/packages/core/src/repowise/core/generation/templates/onboarding/development_guide.j2 b/packages/core/src/repowise/core/generation/templates/onboarding/development_guide.j2
new file mode 100644
index 000000000..050eef64c
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/onboarding/development_guide.j2
@@ -0,0 +1,63 @@
+You are writing the "Development Guide" page for `{{ ctx.repo_name }}` —
+how to do common things in this codebase, derived from conventions that
+already exist in the file tree. Convention-based, not aspirational: only
+report rules the data actually supports.
+
+## Detected conventions (use only these — do not invent new ones)
+
+{% if ctx.suffix_patterns %}
+### Recurring filename suffixes
+{% for p in ctx.suffix_patterns %}
+- **`*_{{ p.suffix }}`** — {{ p.file_count }} files. Examples:
+{% for ex in p.examples %}
+ - `{{ ex }}`
+{% endfor %}
+{% endfor %}
+{% endif %}
+
+{% if ctx.test_mirror %}
+### Test mirror layout
+- **`{{ ctx.test_mirror.test_root }}/` mirrors the source tree** — {{ ctx.test_mirror.matched_files }} of {{ ctx.test_mirror.source_files }} test files match a source stem.
+{% endif %}
+
+{% if ctx.parallel_dirs %}
+### Parallel sibling directories
+The following parent directories contain ≥3 sibling subdirectories that
+share a common file layout (a strong "every X is built like Y" signal):
+{% for d in ctx.parallel_dirs %}
+- `{{ d }}/`
+{% endfor %}
+{% endif %}
+
+{% if ctx.entry_points %}
+## Entry points (anchor "where to start" advice in these)
+{% for ep in ctx.entry_points %}
+- `{{ ep }}`
+{% endfor %}
+{% endif %}
+
+## What to write
+Produce a Development Guide markdown page. Required structure:
+
+1. **## Overview** — one paragraph naming the strongest 2–3 conventions
+ the reader should internalise before making changes. Quote concrete
+ suffixes / directory names from the data above.
+2. **## Adding a new <thing>** — pick the dominant suffix pattern above
+ and write a 3–5 step playbook for adding another file of that kind.
+ Example: if `*_handler` is dominant, the section is "Adding a new
+ handler" with steps grounded in the example file paths above. If no
+ suffix pattern dominates, skip this section and replace it with
+ **"## Adding a new module"** based on the parallel-dirs signal.
+3. **## Writing tests** — only include this section if the test mirror
+ was detected above. State the rule explicitly: tests for
+ `path/to/foo.py` live at `<test_root>/.../test_foo.py` (or the
+ project's actual convention as evidenced by the data).
+4. **## House style** — at most three bullets covering observations the
+ data supports (naming, parallel-dir uniformity, etc.). If you can't
+ support a bullet with the data above, omit it.
+
+Hard rules:
+- Never invent a linter rule, a commit-message format, or a code style
+ the data above does not support.
+- Quote real file paths in every example.
+- Output markdown only. No preamble, no apologies, no closing summary.
diff --git a/packages/core/src/repowise/core/generation/templates/onboarding/getting_started.j2 b/packages/core/src/repowise/core/generation/templates/onboarding/getting_started.j2
new file mode 100644
index 000000000..78e6746d9
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/onboarding/getting_started.j2
@@ -0,0 +1,55 @@
+You are writing the "Getting Started" page for `{{ ctx.repo_name }}` —
+the page where reading stops and doing starts. The reader wants the
+shortest correct path to a working local checkout.
+
+## Detected build signal (ground every command in this data)
+{% if ctx.package_managers %}
+- **Package managers / ecosystems**: {{ ctx.package_managers | join(", ") }}
+{% else %}
+- **Package managers**: none detected (rely on README sections below)
+{% endif %}
+{% if ctx.runtime_dependencies %}
+- **Runtime dependencies sample**: {% for d in ctx.runtime_dependencies %}`{{ d.name }}`{% if not loop.last %}, {% endif %}{% endfor %}
+{% endif %}
+{% if ctx.dev_dependencies %}
+- **Dev / test dependencies sample**: {% for d in ctx.dev_dependencies %}`{{ d.name }}`{% if not loop.last %}, {% endif %}{% endfor %}
+{% endif %}
+{% if ctx.entry_points %}
+- **Entry points**: {% for ep in ctx.entry_points %}`{{ ep }}`{% if not loop.last %}, {% endif %}{% endfor %}
+{% endif %}
+
+{% if ctx.readme_sections %}
+## README setup excerpts (authoritative — quote, don't paraphrase commands)
+{% for s in ctx.readme_sections %}
+### From the README: {{ s.heading }}
+{{ s.body }}
+
+{% endfor %}
+{% endif %}
+
+## What to write
+Produce a Getting Started markdown page. Required sections, in order:
+
+1. **## Prerequisites** — bullet list of runtimes/tools the reader needs
+ before cloning (language runtime + version if knowable from ecosystem
+ conventions, package manager, anything the README explicitly calls
+ out). If unsure of a version, say "recent" and link to the project's
+ own setup section rather than inventing a number.
+2. **## Clone & install** — two or three numbered steps, each with a
+ single fenced shell command block. Quote commands from the README
+ verbatim where they appear; otherwise use the canonical command for
+ the detected package manager (`npm install`, `pip install -e .`,
+ `cargo build`, `go mod download`, etc.).
+3. **## Run** — how to actually start the thing. If entry points are
+ listed above, name them. If the README has a Run section, quote it.
+4. **## Test** — how to run the test suite. Same rules.
+5. **## Troubleshooting** — at most two short bullets, only if the README
+ sections above hint at common gotchas. Skip the section entirely
+ otherwise — do not invent troubleshooting tips.
+
+Hard rules:
+- Never invent a command. Either quote the README or use the canonical
+ command for a listed ecosystem.
+- Keep it short. A new contributor should be able to read the whole page
+ in under two minutes.
+- Output markdown only. No preamble, no apologies, no closing summary.
diff --git a/packages/core/src/repowise/core/generation/templates/onboarding/how_it_works.j2 b/packages/core/src/repowise/core/generation/templates/onboarding/how_it_works.j2
new file mode 100644
index 000000000..dc9b0385b
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/onboarding/how_it_works.j2
@@ -0,0 +1,63 @@
+You are writing the "How It Works" page for `{{ ctx.repo_name }}` — the
+page that turns the static architecture into a movie. One concrete
+end-to-end trace through the system, narrated phase by phase.
+
+## Detected archetype (treat as a hint, not gospel)
+- **Archetype**: `{{ ctx.archetype }}`
+- **Evidence**:
+{% for e in ctx.archetype_evidence %}
+ - {{ e }}
+{% endfor %}
+
+{% if ctx.entry_points %}
+## Entry points (ground each flow in one of these)
+{% for ep in ctx.entry_points %}
+- `{{ ep }}`
+{% endfor %}
+{% endif %}
+
+{% if ctx.flows %}
+## Detected execution flows (use these as the spine of the narrative)
+{% for f in ctx.flows %}
+### Flow from `{{ f.entry_point }}` (score: {{ "%.3f"|format(f.score) }})
+{% for h in f.hops %}
+{{ loop.index }}. `{{ h }}`
+{% endfor %}
+
+{% endfor %}
+{% else %}
+*No multi-hop execution flow was detected. Build the narrative from the
+entry points and the archetype hint above.*
+{% endif %}
+
+## What to write
+Produce a How It Works markdown page. Required structure:
+
+1. **## What this system does in one sentence** — open with a single,
+ concrete sentence that captures the system's job, adapted to the
+ archetype above (service / cli / library / pipeline / module).
+2. **## A walk through one example** — the heart of the page. Pick one
+ concrete example matching the archetype:
+ - service → "when a request hits `<a real entry point>`, …"
+ - cli → "when you run `<the real CLI entry>`, …"
+ - library → "when a consumer calls `<a real public symbol>`, …"
+ - pipeline → "when input enters at `<the real source>`, …"
+ - module → narrate the highest-scoring flow above as a generic
+ control-flow story.
+ Walk through the hops above (or, if no flow was detected, the entry
+ point and its immediate neighbors). Use the file/symbol names as-is.
+3. **## Why it's shaped this way** — 2–3 sentences linking the flow to
+ the archetype: why this structure makes sense for a service / cli /
+ etc. Do not invent decisions; restrict yourself to observations the
+ flow itself supports.
+4. **## Where to read next** — a short bulleted list of 2–4 file paths
+ the reader should open after this page, pulled from the flow hops
+ above. Each bullet gets a half-sentence reason ("the boundary where
+ request becomes domain object", etc.).
+
+Hard rules:
+- Every file path or symbol you name must come from the data above.
+- If a flow hop is unfamiliar, narrate the *transition* generically ("the
+ request crosses the framework boundary here") rather than inventing
+ what the file does internally.
+- Output markdown only. No preamble, no apologies, no closing summary.
diff --git a/packages/core/src/repowise/core/generation/templates/onboarding/key_concepts.j2 b/packages/core/src/repowise/core/generation/templates/onboarding/key_concepts.j2
new file mode 100644
index 000000000..2002c5642
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/onboarding/key_concepts.j2
@@ -0,0 +1,58 @@
+You are writing the "Key Concepts" page for `{{ ctx.repo_name }}` — a
+narrative that gives the reader the mental model needed to read the
+code. Not a glossary dump. Identify 3–6 load-bearing concepts and
+explain how they fit together.
+
+## Candidate concepts (top public symbols by PageRank — ground every
+## concept you name in this list; do not invent new ones)
+{% for c in ctx.concept_symbols %}
+- **`{{ c.name }}`** ({{ c.kind }}) — `{{ c.file_path }}`{% if c.community_label %}, in cluster *{{ c.community_label }}*{% endif %}
+{% if c.docstring %}
+ > {{ c.docstring }}
+{% endif %}
+{% endfor %}
+
+{% if ctx.community_labels %}
+## Architectural clusters present
+The codebase organises into these clusters (detected via dependency
+analysis). Use them as the structural backbone for grouping concepts.
+{% for label in ctx.community_labels %}
+- {{ label }}
+{% endfor %}
+{% endif %}
+
+{% if ctx.decision_titles %}
+## Architectural decisions you can lean on
+These ADR titles were extracted from the codebase. Reference them by
+title only — do not invent the body of a decision.
+{% for t in ctx.decision_titles %}
+- {{ t }}
+{% endfor %}
+{% endif %}
+
+## What to write
+Produce a Key Concepts markdown page. Required structure:
+
+1. **## Overview** — two or three sentences setting up what the reader is
+ about to learn: roughly how many concepts matter, and what kind of
+ system this is (service / library / CLI / compiler / …). Anchor the
+ framing to the cluster labels if present.
+2. **## Core concepts** — one subsection per concept (heading: `### \`Name\``,
+ matching the candidate list above exactly). Aim for 3–6 subsections.
+ For each:
+ - one short paragraph (2–4 sentences) explaining what the concept is
+ and the role it plays,
+ - a `**Where it lives:**` line citing the file path,
+ - if a docstring was supplied, weave it in — don't quote verbatim if
+ it's awkward.
+3. **## How they connect** — one paragraph (4–7 sentences) tracing how
+ the concepts collaborate. This is the section that turns the list
+ into a mental model. Be specific: name which concept calls / wraps /
+ produces which other.
+
+Hard rules:
+- Every concept name must appear in the candidate list above. If a
+ concept feels missing, it isn't — say so explicitly rather than
+ fabricating it.
+- Do not include code listings. Names + file paths are enough.
+- Output markdown only. No preamble, no apologies, no closing summary.
diff --git a/packages/core/src/repowise/core/generation/templates/repo_overview.j2 b/packages/core/src/repowise/core/generation/templates/repo_overview.j2
index f4e76bf6f..6de4ded94 100644
--- a/packages/core/src/repowise/core/generation/templates/repo_overview.j2
+++ b/packages/core/src/repowise/core/generation/templates/repo_overview.j2
@@ -43,6 +43,22 @@ The codebase organizes into these architectural clusters (detected via dependenc
{% endfor %}
{% endif %}
+{% if ctx.external_systems %}
+## External Systems
+Third-party libraries, services, and tools the repository depends on:
+{% for sys in ctx.external_systems[:25] %}
+- `{{ sys.name }}`{% if sys.category and sys.category != 'library' %} *({{ sys.category }})*{% endif %}{% if sys.ecosystem %} — {{ sys.ecosystem }}{% endif %}
+{% endfor %}
+{% endif %}
+
+{% if ctx.decision_records %}
+## Key Architectural Decisions
+Use these to ground the "why" of the project — do not invent rationale beyond what is listed.
+{% for dr in ctx.decision_records[:8] %}
+- **{{ dr.title }}** — {{ dr.decision }}{% if dr.rationale %} *(Why: {{ dr.rationale }})*{% endif %}
+{% endfor %}
+{% endif %}
+
{% if ctx.execution_flows %}
## Primary Execution Flows
Key entry points and their execution paths:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py
new file mode 100644
index 000000000..0a4918739
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py
@@ -0,0 +1,154 @@
+"""Pruned filesystem walk for dynamic-hint extractors.
+
+Every extractor in this package used to call ``repo_root.rglob(pattern)``
+directly, which descended into ``node_modules``, ``.venv``, ``.next``,
+``__pycache__`` and other multi-million-file junk trees. On a polyrepo
+with sibling ``backend/`` and ``frontend/`` directories the dynamic-hints
+phase could stall for 5–10+ minutes per extractor with the progress bar
+stuck at zero.
+
+``iter_glob`` uses :func:`os.walk` with in-place ``dirnames`` pruning so
+known-junk subdirectories are skipped at the point of traversal, not
+afterwards. It also guards against:
+
+ - **Symlink loops** — ``os.walk`` is called with ``followlinks=False``.
+ - **Junction loops** (Windows) — caught via :func:`os.path.realpath`
+ cycle detection.
+ - **Pathological recursive copies** — same cycle detector handles real
+ directories that share an inode through hard-linking, junctions, or
+ repeated nested-copy mistakes.
+ - **Catastrophic depth** — a hard depth cap as a final safety net.
+
+Semantics match :py:meth:`pathlib.Path.rglob`: both files and directories
+whose basename matches the glob are yielded.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+from collections.abc import Iterator
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+# Directory basenames that are never source-bearing. Pruned at every
+# level of the walk. Keep this list conservative: anything ambiguously
+# named (``bin``, ``obj``, ``lib``) belongs out of here so we don't
+# accidentally drop real source.
+PRUNED_DIRS: frozenset[str] = frozenset(
+ {
+ # VCS / metadata
+ ".git",
+ ".hg",
+ ".svn",
+ # Python environments / caches
+ ".venv",
+ "venv",
+ ".env",
+ "__pycache__",
+ ".mypy_cache",
+ ".pytest_cache",
+ ".ruff_cache",
+ ".tox",
+ ".nox",
+ # JS / TS
+ "node_modules",
+ ".next",
+ ".nuxt",
+ ".turbo",
+ ".parcel-cache",
+ # Build artifacts (multi-language)
+ "dist",
+ "build",
+ "out",
+ ".gradle",
+ ".cache",
+ # Editor / IDE
+ ".idea",
+ ".vscode",
+ # Coverage / test outputs
+ "coverage",
+ "htmlcov",
+ ".nyc_output",
+ # Repowise / data
+ ".repowise",
+ ".lancedb",
+ }
+)
+
+# Safety net: bail if walk depth grows pathologically. Real-world repos
+# rarely exceed depth 20; anything beyond strongly suggests a cycle.
+_MAX_WALK_DEPTH = 64
+
+
+def iter_glob(root: Path, pattern: str) -> Iterator[Path]:
+ """Recursively yield paths under ``root`` whose basename matches ``pattern``.
+
+ Mirrors :py:meth:`pathlib.Path.rglob` semantics — files *and*
+ directories with a matching basename are yielded — but:
+ - prunes well-known junk directories at every level,
+ - never follows symlinks (``os.walk(followlinks=False)``),
+ - detects junction / hard-link cycles via realpath tracking,
+ - caps total depth as a final safety net.
+
+ Args:
+ root: Directory to walk. May be a string or :class:`Path`.
+ pattern: A :mod:`fnmatch`-style glob applied to the basename.
+ Examples: ``"*.go"``, ``"settings.py"``, ``"package.json"``,
+ ``"tsconfig*.json"``.
+ """
+ root_path = Path(root)
+ if not root_path.exists():
+ return
+
+ try:
+ root_real = os.path.realpath(root_path)
+ except OSError:
+ return
+ base_depth = root_real.rstrip(os.sep).count(os.sep)
+ visited_real: set[str] = {root_real}
+
+ for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):
+ # Skip junk directories at every level.
+ dirnames[:] = [d for d in dirnames if d not in PRUNED_DIRS]
+
+ # Drop any dir whose realpath we've already entered — catches
+ # Windows junctions and any other cycle that os.walk can't detect
+ # via inode (Windows doesn't expose real inodes for filesystems
+ # outside of NTFS proper).
+ pruned: list[str] = []
+ for d in list(dirnames):
+ child = os.path.join(dirpath, d)
+ try:
+ child_real = os.path.realpath(child)
+ except OSError:
+ pruned.append(d)
+ continue
+ if child_real in visited_real:
+ pruned.append(d)
+ continue
+ visited_real.add(child_real)
+ if pruned:
+ dirnames[:] = [d for d in dirnames if d not in pruned]
+
+ # Final safety net: refuse to descend beyond _MAX_WALK_DEPTH.
+ cur_depth = dirpath.rstrip(os.sep).count(os.sep) - base_depth
+ if cur_depth >= _MAX_WALK_DEPTH:
+ log.warning(
+ "dynamic_hints.walk_depth_exceeded",
+ root=str(root_path),
+ depth=cur_depth,
+ at=dirpath,
+ )
+ dirnames[:] = []
+ continue
+
+ for name in filenames:
+ if fnmatch.fnmatch(name, pattern):
+ yield Path(dirpath) / name
+ for name in dirnames:
+ if fnmatch.fnmatch(name, pattern):
+ yield Path(dirpath) / name
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
index 53efa9c5f..b2d2baf6e 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
@@ -1,9 +1,12 @@
from __future__ import annotations
from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
+from collections.abc import Iterator
+from dataclasses import dataclass, field # noqa: F401 — re-exported
from pathlib import Path
+from ._walk import iter_glob
+
@dataclass
class DynamicEdge:
@@ -19,3 +22,13 @@ class DynamicHintExtractor(ABC):
@abstractmethod
def extract(self, repo_root: Path) -> list[DynamicEdge]: ...
+
+ @staticmethod
+ def _rglob(root: Path, pattern: str) -> Iterator[Path]:
+ """Pruned replacement for :py:meth:`pathlib.Path.rglob`.
+
+ Skips ``node_modules``, ``.venv``, ``.next``, etc. so large
+ polyrepos don't tank the dynamic-hints phase. See
+ :mod:`dynamic_hints._walk` for the full prune list.
+ """
+ return iter_glob(root, pattern)
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/c.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/c.py
index 57f4e5e30..2f13432dd 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/c.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/c.py
@@ -36,7 +36,7 @@ class CDynamicHints(DynamicHintExtractor):
sources: list[tuple[Path, str]] = []
repo_root_resolved = repo_root.resolve()
for ext in (".c", ".h"):
- for src in repo_root.rglob(f"*{ext}"):
+ for src in self._rglob(repo_root, f"*{ext}"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
index 0bce49da0..ebf7655d3 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
@@ -66,8 +66,8 @@ class DjangoDynamicHints(DynamicHintExtractor):
edges: list[DynamicEdge] = []
# Collect all settings files
- settings_files: list[Path] = list(repo_root.rglob("settings.py"))
- for settings_dir in repo_root.rglob("settings"):
+ settings_files: list[Path] = list(self._rglob(repo_root, "settings.py"))
+ for settings_dir in self._rglob(repo_root, "settings"):
if settings_dir.is_dir():
settings_files.extend(settings_dir.glob("*.py"))
@@ -137,7 +137,7 @@ class DjangoDynamicHints(DynamicHintExtractor):
edges: list[DynamicEdge] = []
include_re = re.compile(r"""include\(\s*['\"]([\w\.]+)['\"]""")
- for urls_file in repo_root.rglob("urls.py"):
+ for urls_file in self._rglob(repo_root, "urls.py"):
try:
source = urls_file.read_text(encoding="utf-8", errors="ignore")
rel_urls = urls_file.relative_to(repo_root).as_posix()
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/dotnet.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/dotnet.py
index 8f892d947..fc2534597 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/dotnet.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/dotnet.py
@@ -105,7 +105,7 @@ class DotNetDynamicHints(DynamicHintExtractor):
type_to_files: dict[str, list[str]] = {}
cs_files: list[tuple[Path, str]] = [] # (path, text)
repo_root_resolved = repo_root.resolve()
- for cs in repo_root.rglob("*.cs"):
+ for cs in self._rglob(repo_root, "*.cs"):
try:
rel_path = cs.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/go.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/go.py
index bafabf19e..cf9cf0e79 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/go.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/go.py
@@ -32,7 +32,7 @@ class GoDynamicHints(DynamicHintExtractor):
func_to_file: dict[str, str] = {}
sources: list[tuple[Path, str]] = []
repo_root_resolved = repo_root.resolve()
- for src in repo_root.rglob("*.go"):
+ for src in self._rglob(repo_root, "*.go"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/luau.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/luau.py
index c7fb430fd..7be0a7258 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/luau.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/luau.py
@@ -32,7 +32,7 @@ class LuauDynamicHints(DynamicHintExtractor):
repo_root_resolved = repo_root.resolve()
# luau plus lua extensions
for ext in (".luau", ".lua"):
- for src in repo_root.rglob(f"*{ext}"):
+ for src in self._rglob(repo_root, f"*{ext}"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/node.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/node.py
index 1895d1b16..93c27e5ff 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/node.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/node.py
@@ -43,10 +43,7 @@ class NodeDynamicHints(DynamicHintExtractor):
def _scan_package_json(self, repo_root: Path) -> list[DynamicEdge]:
edges: list[DynamicEdge] = []
- for pkg_file in repo_root.rglob("package.json"):
- # Skip node_modules
- if "node_modules" in pkg_file.parts:
- continue
+ for pkg_file in self._rglob(repo_root, "package.json"):
try:
text = pkg_file.read_text(encoding="utf-8", errors="ignore")
data = json.loads(text)
@@ -90,9 +87,7 @@ class NodeDynamicHints(DynamicHintExtractor):
def _scan_tsconfig(self, repo_root: Path) -> list[DynamicEdge]:
edges: list[DynamicEdge] = []
- for tsconfig in repo_root.rglob("tsconfig*.json"):
- if "node_modules" in tsconfig.parts:
- continue
+ for tsconfig in self._rglob(repo_root, "tsconfig*.json"):
try:
text = tsconfig.read_text(encoding="utf-8", errors="ignore")
data = _json_loads_lenient(text)
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/php.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/php.py
index 7c3ae720f..c8113d3e2 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/php.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/php.py
@@ -38,7 +38,7 @@ class PhpDynamicHints(DynamicHintExtractor):
type_to_file: dict[str, str] = {}
php_files: list[tuple[Path, str]] = []
repo_root_resolved = repo_root.resolve()
- for src in repo_root.rglob("*.php"):
+ for src in self._rglob(repo_root, "*.php"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/pytest_hints.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/pytest_hints.py
index d13da185a..bbb3795d8 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/pytest_hints.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/pytest_hints.py
@@ -55,7 +55,7 @@ class PytestDynamicHints(DynamicHintExtractor):
def extract(self, repo_root: Path) -> list[DynamicEdge]:
edges: list[DynamicEdge] = []
- for conftest in repo_root.rglob("conftest.py"):
+ for conftest in self._rglob(repo_root, "conftest.py"):
try:
source = conftest.read_text(encoding="utf-8", errors="ignore")
tree = ast.parse(source, filename=str(conftest))
@@ -72,7 +72,7 @@ class PytestDynamicHints(DynamicHintExtractor):
# Find all test files under the conftest's parent directory
for pattern in ("test_*.py", "*_test.py"):
- for test_file in conftest_dir.rglob(pattern):
+ for test_file in self._rglob(conftest_dir, pattern):
if test_file == conftest:
continue
try:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
index 06502d0fd..84eebc209 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
@@ -1,5 +1,21 @@
+"""Aggregates every dynamic-hint extractor and runs them concurrently.
+
+Each extractor walks the repo independently looking for language-specific
+runtime/dynamic call signals (Django URL routes, Spring annotations,
+pytest fixtures, etc.). The walks are I/O-bound and entirely independent,
+so they're fanned out across a thread pool — the Python GIL is released
+during ``read_text`` / ``os.walk`` syscalls, which is where the time
+actually goes.
+
+Combined with the pruned :func:`._walk.iter_glob` that skips
+``node_modules`` / ``.venv`` / ``.next`` / ``__pycache__`` etc. at the
+point of traversal, the phase that used to stall multi-minute on polyrepos
+now completes in seconds on most codebases regardless of language mix.
+"""
+
from __future__ import annotations
+from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import structlog
@@ -21,9 +37,19 @@ from .xaml import XamlDynamicHints
log = structlog.get_logger(__name__)
+# Capping thread count prevents thrash on machines with many extractors
+# but few cores. 8 is enough for full parallelism across the 13 current
+# extractors with headroom for the GIL release windows.
+_DEFAULT_MAX_WORKERS = 8
+
class HintRegistry:
- def __init__(self, extractors: list[DynamicHintExtractor] | None = None) -> None:
+ def __init__(
+ self,
+ extractors: list[DynamicHintExtractor] | None = None,
+ *,
+ max_workers: int | None = None,
+ ) -> None:
self._extractors = extractors or [
DjangoDynamicHints(),
PytestDynamicHints(),
@@ -39,14 +65,39 @@ class HintRegistry:
LuauDynamicHints(),
GoDynamicHints(),
]
+ self._max_workers = max_workers or min(_DEFAULT_MAX_WORKERS, len(self._extractors))
def extract_all(self, repo_root: Path) -> list[DynamicEdge]:
+ """Run every registered extractor and merge their edges.
+
+ Extractors run in a thread pool — their work is I/O-bound and
+ releases the GIL during filesystem reads, so threads are the
+ right tool (no per-process startup overhead, no pickling).
+ Order of edges in the returned list is not guaranteed.
+ """
edges: list[DynamicEdge] = []
- for ex in self._extractors:
- try:
- got = ex.extract(repo_root)
+ if not self._extractors:
+ return edges
+
+ with ThreadPoolExecutor(max_workers=self._max_workers) as pool:
+ futures = {
+ pool.submit(self._run_one, ex, repo_root): ex for ex in self._extractors
+ }
+ for future in as_completed(futures):
+ ex = futures[future]
+ try:
+ got = future.result()
+ except Exception as e:
+ log.warning(
+ "dynamic_hints_failed", extractor=ex.name, error=str(e)
+ )
+ continue
edges.extend(got)
log.debug("dynamic_hints", extractor=ex.name, count=len(got))
- except Exception as e:
- log.warning("dynamic_hints_failed", extractor=ex.name, error=str(e))
return edges
+
+ @staticmethod
+ def _run_one(
+ extractor: DynamicHintExtractor, repo_root: Path
+ ) -> list[DynamicEdge]:
+ return extractor.extract(repo_root)
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/ruby.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/ruby.py
index 5d0e0b268..1e8bf58a2 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/ruby.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/ruby.py
@@ -35,7 +35,7 @@ class RubyDynamicHints(DynamicHintExtractor):
method_to_files: dict[str, list[str]] = {}
rb_files: list[tuple[Path, str]] = []
repo_root_resolved = repo_root.resolve()
- for src in repo_root.rglob("*.rb"):
+ for src in self._rglob(repo_root, "*.rb"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/scala.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/scala.py
index a789c23d1..43fe900d4 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/scala.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/scala.py
@@ -30,7 +30,7 @@ class ScalaDynamicHints(DynamicHintExtractor):
type_to_file: dict[str, str] = {}
sources: list[tuple[Path, str]] = []
repo_root_resolved = repo_root.resolve()
- for src in repo_root.rglob("*.scala"):
+ for src in self._rglob(repo_root, "*.scala"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/spring.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/spring.py
index 300a7c66c..0fd5ec1b9 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/spring.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/spring.py
@@ -43,7 +43,7 @@ class SpringDynamicHints(DynamicHintExtractor):
sources: list[tuple[Path, str, str]] = [] # (path, text, lang)
repo_root_resolved = repo_root.resolve()
for ext, lang in ((".java", "java"), (".kt", "kotlin")):
- for src in repo_root.rglob(f"*{ext}"):
+ for src in self._rglob(repo_root, f"*{ext}"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/swift.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/swift.py
index e0a6e5977..e7963a91a 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/swift.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/swift.py
@@ -34,7 +34,7 @@ class SwiftDynamicHints(DynamicHintExtractor):
func_to_files: dict[str, list[str]] = {}
sources: list[tuple[Path, str]] = []
repo_root_resolved = repo_root.resolve()
- for src in repo_root.rglob("*.swift"):
+ for src in self._rglob(repo_root, "*.swift"):
try:
rel_path = src.resolve().relative_to(repo_root_resolved)
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py
index 4f09ecedb..dd5f84d3e 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py
@@ -173,8 +173,9 @@ class XamlDynamicHints(DynamicHintExtractor):
# ---------------------------------------------------------------------------
def _iter_xaml_files(repo_root: Path):
+ from ._walk import iter_glob as _iter_glob
for ext in _XAML_EXTS:
- for path in repo_root.rglob(f"*{ext}"):
+ for path in _iter_glob(repo_root, f"*{ext}"):
try:
rel = path.resolve().relative_to(repo_root.resolve())
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py b/packages/core/src/repowise/core/ingestion/external_systems/__init__.py
new file mode 100644
index 000000000..482b15e5c
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/external_systems/__init__.py
@@ -0,0 +1,105 @@
+"""Manifest-based extraction of external system dependencies.
+
+Public API:
+ extract_external_systems(repo_root, manifest_paths=None) -> list[ExternalSystemRecord]
+
+Each ecosystem-specific parser lives in its own submodule
+(:mod:`.npm`, :mod:`.pypi`, :mod:`.cargo`, :mod:`.go`, :mod:`.nuget`) and
+exposes ``filenames``, ``ecosystem``, and ``parse(manifest_path, repo_root)``.
+
+The extractor walks the repo (or accepts a pre-computed list of manifest
+paths from the traverser) and dispatches to the matching parser. Parsers
+never raise on malformed input.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+from types import ModuleType
+
+from . import cargo, go, npm, nuget, pypi
+from .base import ExternalSystemRecord, ManifestParser
+
+__all__ = [
+ "ExternalSystemRecord",
+ "ManifestParser",
+ "extract_external_systems",
+]
+
+_PARSERS: tuple[ModuleType, ...] = (npm, pypi, cargo, go, nuget)
+
+# Map exact filename → parser module
+_FILENAME_PARSERS: dict[str, ModuleType] = {
+ fname: parser for parser in _PARSERS for fname in parser.filenames
+}
+
+# Filename patterns that match by predicate (pypi requirements*.txt, nuget *.csproj)
+def _matches_pattern(filename: str) -> ModuleType | None:
+ lower = filename.lower()
+ if lower.startswith("requirements") and lower.endswith(".txt"):
+ return pypi
+ if lower.endswith(".csproj"):
+ return nuget
+ return None
+
+
+def _parser_for(path: Path) -> ModuleType | None:
+ parser = _FILENAME_PARSERS.get(path.name)
+ if parser is not None:
+ return parser
+ return _matches_pattern(path.name)
+
+
+def extract_external_systems(
+ repo_root: Path,
+ manifest_paths: Iterable[Path] | None = None,
+) -> list[ExternalSystemRecord]:
+ """Return the deduplicated list of external systems declared in ``repo_root``.
+
+ If ``manifest_paths`` is provided, only those files are inspected (the
+ traverser already walks the repo and can hand us the manifests it saw).
+ Otherwise the function does its own bounded walk (depth ≤ 4) so it can be
+ called standalone from tests.
+ """
+ candidates = list(manifest_paths) if manifest_paths is not None else _discover(repo_root)
+
+ records: list[ExternalSystemRecord] = []
+ for path in candidates:
+ parser = _parser_for(path)
+ if parser is None:
+ continue
+ try:
+ records.extend(parser.parse(path, repo_root))
+ except Exception: # parser bugs must not break ingestion
+ continue
+ return _deduplicate(records)
+
+
+def _discover(repo_root: Path) -> list[Path]:
+ """Walk the repo (depth ≤ 4) collecting recognised manifest files."""
+ seen: list[Path] = []
+ skip_dirs = {".git", "node_modules", "__pycache__", ".venv", "venv", "target", "dist", "build"}
+ root = Path(repo_root)
+ for path in root.rglob("*"):
+ rel_parts = path.relative_to(root).parts
+ if len(rel_parts) > 4 or any(part in skip_dirs for part in rel_parts):
+ continue
+ if not path.is_file():
+ continue
+ if _parser_for(path) is not None:
+ seen.append(path)
+ return seen
+
+
+def _deduplicate(records: list[ExternalSystemRecord]) -> list[ExternalSystemRecord]:
+ """Drop duplicates keyed on (name, declared_in). Stable order."""
+ out: list[ExternalSystemRecord] = []
+ seen: set[tuple[str, str]] = set()
+ for rec in records:
+ key = (rec.name, rec.declared_in)
+ if key in seen:
+ continue
+ seen.add(key)
+ out.append(rec)
+ return out
diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.py
index 66b0052b9..ed8af4092 100644
--- a/packages/core/src/repowise/core/persistence/__init__.py
+++ b/packages/core/src/repowise/core/persistence/__init__.py
@@ -58,6 +58,7 @@ from .crud import (
list_decisions,
list_page_ids,
list_pages,
+ load_prior_pages,
mark_webhook_processed,
recompute_decision_staleness,
save_dead_code_findings,
@@ -200,6 +201,7 @@ __all__ = [
"list_decisions",
"list_page_ids",
"list_pages",
+ "load_prior_pages",
"mark_webhook_processed",
"recompute_decision_staleness",
"resolve_db_url",
diff --git a/packages/core/src/repowise/core/persistence/crud.py b/packages/core/src/repowise/core/persistence/crud.py
index e34b77c9c..7bd3baef7 100644
--- a/packages/core/src/repowise/core/persistence/crud.py
+++ b/packages/core/src/repowise/core/persistence/crud.py
@@ -332,6 +332,37 @@ async def upsert_page(
return page
+async def load_prior_pages(
+ session: AsyncSession,
+ repository_id: str,
+) -> dict[str, Any]:
+ """Return a ``page_id → PriorPage`` map for cross-run cache reuse.
+
+ Loads every existing wiki page for the repository so the generator can
+ short-circuit the LLM call when the freshly rendered prompt produces a
+ matching ``source_hash`` under the same model. Returns an empty dict if
+ nothing has been generated yet.
+ """
+ # Import lazily — keeps persistence independent of generation models at
+ # module-load time.
+ from repowise.core.generation.page_generator import PriorPage
+
+ result = await session.execute(
+ select(Page).where(Page.repository_id == repository_id)
+ )
+ prior: dict[str, Any] = {}
+ for row in result.scalars():
+ prior[row.id] = PriorPage(
+ source_hash=row.source_hash,
+ model_name=row.model_name,
+ content=row.content,
+ input_tokens=row.input_tokens,
+ output_tokens=row.output_tokens,
+ cached_tokens=row.cached_tokens,
+ )
+ return prior
+
+
async def upsert_page_from_generated(
session: AsyncSession,
generated_page: object, # repowise.core.generation.models.GeneratedPage
diff --git a/packages/core/src/repowise/core/persistence/vector_store.py b/packages/core/src/repowise/core/persistence/vector_store.py
index 14e431b4d..6d61559eb 100644
--- a/packages/core/src/repowise/core/persistence/vector_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store.py
@@ -84,6 +84,30 @@ class VectorStore(ABC):
"""
return None # default: no-op (subclasses should override)
+ async def get_page_summaries_by_paths(
+ self, paths: list[str]
+ ) -> dict[str, dict]:
+ """Batch variant of :meth:`get_page_summary_by_path`.
+
+ Returns a mapping of resolved paths → summary dict for every
+ input path that produced a non-None result. The default
+ implementation fires all per-path calls concurrently via
+ ``asyncio.gather`` so callers don't have to await each one
+ sequentially — backends that can do a single SQL/index scan
+ should override this for the obvious efficiency gain.
+ """
+ import asyncio as _asyncio
+
+ if not paths:
+ return {}
+ coros = [self.get_page_summary_by_path(p) for p in paths]
+ results = await _asyncio.gather(*coros, return_exceptions=True)
+ out: dict[str, dict] = {}
+ for path, result in zip(paths, results, strict=False):
+ if isinstance(result, dict) and result.get("summary"):
+ out[path] = result
+ return out
+
# ---------------------------------------------------------------------------
# InMemoryVectorStore
@@ -170,6 +194,28 @@ class InMemoryVectorStore(VectorStore):
return {"summary": summary, "key_exports": list(key_exports)}
return None
+ async def get_page_summaries_by_paths(
+ self, paths: list[str]
+ ) -> dict[str, dict]:
+ """Single-pass scan over the store — avoids N full scans when
+ many paths are queried at once.
+ """
+ if not paths:
+ return {}
+ wanted = set(paths)
+ out: dict[str, dict] = {}
+ for _pid, (_, meta) in self._store.items():
+ tp = meta.get("target_path")
+ if tp in wanted and tp not in out:
+ summary = meta.get("summary") or str(meta.get("content", ""))[:500]
+ out[str(tp)] = {
+ "summary": summary,
+ "key_exports": list(meta.get("exports") or []),
+ }
+ if len(out) == len(wanted):
+ break
+ return out
+
def __len__(self) -> int:
return len(self._store)
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index bef413052..7621df7a8 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -417,6 +417,21 @@ async def run_pipeline(
reasoning=resolve_reasoning(config=load_repo_config(repo_path))
)
+ # Phase 2 enrichment: flag framework-defined HTTP surfaces (FastAPI,
+ # ASP.NET controllers, …) as api_contract so they route through the
+ # api_contract template instead of generic file_page. Lives in the
+ # generation package so language-specific heuristics stay co-located
+ # with the templates that consume them.
+ from repowise.core.generation import detect_code_api_contracts as _detect_apis
+ try:
+ flipped = _detect_apis(parsed_files)
+ if flipped and progress:
+ progress.on_message(
+ "info", f"→ Detected {flipped} additional API contract file(s)"
+ )
+ except Exception as _api_err:
+ logger.warning("api_contract_detection_failed", error=str(_api_err))
+
generated_pages = await run_generation(
repo_path=repo_path,
parsed_files=parsed_files,
@@ -431,6 +446,9 @@ async def run_pipeline(
progress=progress,
resume=resume,
generation_config=resolved_generation_config,
+ dead_code_report=dead_code_report,
+ decision_report=decision_report,
+ external_systems=external_systems,
)
# ---- Execution flow tracing -----------------------------------------------
@@ -940,6 +958,9 @@ async def run_generation(
resume: bool = False,
cost_tracker: Any | None = None,
generation_config: Any | None = None,
+ dead_code_report: Any | None = None,
+ decision_report: Any | None = None,
+ external_systems: list[dict] | None = None,
) -> list[Any]:
"""Run LLM-powered page generation.
@@ -979,14 +1000,17 @@ async def run_generation(
repo_name = repo_path.name
- # Track generation progress
+ # Track generation progress. Onboarding pages get routed to their own
+ # phase so the terminal UI shows them as a distinct, named step rather
+ # than blending into the long file_page run.
_pages_done = 0
def on_page_done(page_type: str) -> None:
nonlocal _pages_done
_pages_done += 1
if progress:
- progress.on_item_done("generation")
+ phase = "onboarding" if page_type == "onboarding" else "generation"
+ progress.on_item_done(phase)
# Push live cost update if the callback supports it
if cost_tracker is not None and hasattr(progress, "set_cost"):
progress.set_cost(cost_tracker.session_cost)
@@ -998,6 +1022,11 @@ async def run_generation(
if progress:
progress.on_phase_start("generation", total)
+ def on_subphase(name: str, total: int | None) -> None:
+ """Start a distinct sub-phase (currently used only for onboarding)."""
+ if progress:
+ progress.on_phase_start(name, total)
+
generator = PageGenerator(
llm_client,
assembler,
@@ -1015,13 +1044,36 @@ async def run_generation(
job_system=job_system,
on_page_done=on_page_done,
on_total_known=on_total_known,
+ on_subphase=on_subphase,
git_meta_map=git_meta_map if git_meta_map else None,
resume=resume,
repo_path=repo_path,
+ dead_code_report=dead_code_report,
+ decision_report=decision_report,
+ external_systems=external_systems,
)
+ # Onboarding summary — count generated slots and surface which ones
+ # were gated out so the user can see the curated collection's state.
+ onboarding_generated = [p for p in generated_pages if p.page_type == "onboarding"]
+ promoted_present = {
+ p.metadata.get("onboarding_slot")
+ for p in generated_pages
+ if p.metadata.get("onboarding_slot")
+ and p.page_type in ("repo_overview", "architecture_diagram")
+ }
if progress:
+ if onboarding_generated or promoted_present:
+ slots_made = sorted(
+ {p.metadata.get("subkind", "?") for p in onboarding_generated}
+ | promoted_present
+ )
+ progress.on_message(
+ "info",
+ f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",
+ )
progress.on_message("info", f"Generated {len(generated_pages)} pages")
+ _phase_done(progress, "onboarding")
_phase_done(progress, "generation")
return generated_pages
diff --git a/packages/core/src/repowise/core/providers/llm/anthropic.py b/packages/core/src/repowise/core/providers/llm/anthropic.py
index bdc5ee386..70eab6f34 100644
--- a/packages/core/src/repowise/core/providers/llm/anthropic.py
+++ b/packages/core/src/repowise/core/providers/llm/anthropic.py
@@ -30,6 +30,7 @@ from tenacity import (
from repowise.core.providers.llm.base import (
BaseProvider,
+ CacheHint,
ChatStreamEvent,
ChatToolCall,
GeneratedResponse,
@@ -97,6 +98,7 @@ class AnthropicProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple[CacheHint, ...] = (),
) -> GeneratedResponse:
ensure_reasoning_supported("anthropic", self._model, reasoning)
if self._rate_limiter:
@@ -116,6 +118,7 @@ class AnthropicProvider(BaseProvider):
max_tokens=max_tokens,
temperature=temperature,
request_id=request_id,
+ cache_hints=cache_hints,
)
except RetryError as exc:
raise ProviderError(
@@ -136,14 +139,18 @@ class AnthropicProvider(BaseProvider):
max_tokens: int,
temperature: float,
request_id: str | None,
+ cache_hints: tuple[CacheHint, ...] = (),
) -> GeneratedResponse:
+ system_param, messages_param = _build_cached_payload(
+ system_prompt, user_prompt, cache_hints
+ )
try:
response = await self._client.messages.create(
model=self._model,
max_tokens=max_tokens,
temperature=temperature,
- system=system_prompt,
- messages=[{"role": "user", "content": user_prompt}],
+ system=system_param,
+ messages=messages_param,
)
except _AnthropicRateLimitError as exc:
raise RateLimitError("anthropic", str(exc), status_code=429) from exc
@@ -287,6 +294,48 @@ class AnthropicProvider(BaseProvider):
raise ProviderError("anthropic", str(exc), status_code=exc.status_code) from exc
+def _build_cached_payload(
+ system_prompt: str,
+ user_prompt: str,
+ cache_hints: tuple[CacheHint, ...],
+) -> tuple[Any, list[dict[str, Any]]]:
+ """Translate cache hints into Anthropic-format system/messages blocks.
+
+ Without hints, returns the plain (system_string, [{user content}]) form.
+ With hints, returns content-block lists tagged with cache_control.
+ Anthropic enforces a hard limit of 4 cache breakpoints — we honor at most
+ two (system, then optional user prefix), matching the supported segments.
+ """
+ if not cache_hints:
+ return system_prompt, [{"role": "user", "content": user_prompt}]
+
+ system_param: Any = system_prompt
+ if any(h.segment == "system" for h in cache_hints) and system_prompt:
+ system_param = [
+ {
+ "type": "text",
+ "text": system_prompt,
+ "cache_control": {"type": "ephemeral"},
+ }
+ ]
+
+ user_prefix_hint = next(
+ (h for h in cache_hints if h.segment == "user_prefix" and h.prefix_chars > 0),
+ None,
+ )
+ if user_prefix_hint is not None and user_prefix_hint.prefix_chars < len(user_prompt):
+ prefix = user_prompt[: user_prefix_hint.prefix_chars]
+ rest = user_prompt[user_prefix_hint.prefix_chars :]
+ user_content: Any = [
+ {"type": "text", "text": prefix, "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": rest},
+ ]
+ else:
+ user_content = user_prompt
+
+ return system_param, [{"role": "user", "content": user_content}]
+
+
def _to_anthropic_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Convert OpenAI-format messages to Anthropic format.
diff --git a/packages/core/src/repowise/core/providers/llm/base.py b/packages/core/src/repowise/core/providers/llm/base.py
index ce8b5ed27..45f8603f0 100644
--- a/packages/core/src/repowise/core/providers/llm/base.py
+++ b/packages/core/src/repowise/core/providers/llm/base.py
@@ -15,11 +15,36 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
-from typing import Any, AsyncIterator, Protocol, runtime_checkable
+from typing import Any, AsyncIterator, Literal, Protocol, runtime_checkable
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
+CacheSegment = Literal["system", "user_prefix"]
+
+
+@dataclass(frozen=True)
+class CacheHint:
+ """Caller-provided hint that a prompt segment is reusable across calls.
+
+ Providers that support server-side prompt caching (Anthropic) use these
+ hints to mark cache breakpoints. Providers without an explicit caching
+ primitive (OpenAI auto-caches stable prefixes, Ollama is local) ignore
+ them — the contract is advisory, never required.
+
+ Attributes:
+ segment: Which part of the prompt the hint applies to.
+ - ``system``: the system_prompt argument.
+ - ``user_prefix``: a leading portion of the user_prompt;
+ ``prefix_chars`` specifies how many chars are stable.
+ prefix_chars: For ``user_prefix`` hints, the number of leading
+ characters that are reusable. Ignored for ``system``.
+ """
+
+ segment: CacheSegment
+ prefix_chars: int = 0
+
+
@dataclass
class GeneratedResponse:
"""Unified response shape returned by every provider.
@@ -33,7 +58,7 @@ class GeneratedResponse:
input_tokens: Tokens consumed by the prompt (system + user).
output_tokens: Tokens produced in the response.
cached_tokens: Tokens served from the provider's prompt cache (if any).
- Currently only Anthropic reports this natively.
+ Normalised across providers by the adapter.
usage: Provider-specific usage dict (stored as-is for auditing).
"""
@@ -72,6 +97,7 @@ class BaseProvider(ABC):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple[CacheHint, ...] = (),
) -> GeneratedResponse:
"""Generate a response from the LLM.
@@ -87,6 +113,10 @@ class BaseProvider(ABC):
reasoning: Provider-level reasoning intent. ``auto`` preserves
provider defaults; ``off`` and ``minimal`` are
translated by providers that support them.
+ cache_hints: Optional hints that one or more prompt segments are
+ reusable across calls. Providers with an explicit
+ caching primitive (Anthropic) use them; others
+ ignore them safely.
Returns:
GeneratedResponse with content and token usage.
diff --git a/packages/core/src/repowise/core/providers/llm/deepseek.py b/packages/core/src/repowise/core/providers/llm/deepseek.py
index 6864686ab..3749a4f8a 100644
--- a/packages/core/src/repowise/core/providers/llm/deepseek.py
+++ b/packages/core/src/repowise/core/providers/llm/deepseek.py
@@ -103,6 +103,7 @@ class DeepSeekProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
) -> GeneratedResponse:
ensure_reasoning_supported("deepseek", self._model, reasoning)
if self._rate_limiter:
diff --git a/packages/core/src/repowise/core/providers/llm/gemini.py b/packages/core/src/repowise/core/providers/llm/gemini.py
index 12c1c23fc..7811b0e59 100644
--- a/packages/core/src/repowise/core/providers/llm/gemini.py
+++ b/packages/core/src/repowise/core/providers/llm/gemini.py
@@ -102,6 +102,7 @@ class GeminiProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
) -> GeneratedResponse:
ensure_reasoning_supported("gemini", self._model, reasoning)
if self._rate_limiter:
diff --git a/packages/core/src/repowise/core/providers/llm/litellm.py b/packages/core/src/repowise/core/providers/llm/litellm.py
index 41bf47af5..d4152bb51 100644
--- a/packages/core/src/repowise/core/providers/llm/litellm.py
+++ b/packages/core/src/repowise/core/providers/llm/litellm.py
@@ -101,6 +101,7 @@ class LiteLLMProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
) -> GeneratedResponse:
ensure_reasoning_supported("litellm", self._model, reasoning)
if self._rate_limiter:
diff --git a/packages/core/src/repowise/core/providers/llm/mock.py b/packages/core/src/repowise/core/providers/llm/mock.py
index 247274e76..26d1ada4f 100644
--- a/packages/core/src/repowise/core/providers/llm/mock.py
+++ b/packages/core/src/repowise/core/providers/llm/mock.py
@@ -110,6 +110,7 @@ class MockProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
) -> GeneratedResponse:
"""Return a fixture response without calling any external API."""
self._calls.append(
diff --git a/packages/core/src/repowise/core/providers/llm/ollama.py b/packages/core/src/repowise/core/providers/llm/ollama.py
index 0d57f5247..ac89d03ad 100644
--- a/packages/core/src/repowise/core/providers/llm/ollama.py
+++ b/packages/core/src/repowise/core/providers/llm/ollama.py
@@ -103,6 +103,7 @@ class OllamaProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
) -> GeneratedResponse:
ensure_reasoning_supported("ollama", self._model, reasoning)
if self._rate_limiter:
diff --git a/packages/core/src/repowise/core/providers/llm/openai.py b/packages/core/src/repowise/core/providers/llm/openai.py
index b40435a1c..0b85ede03 100644
--- a/packages/core/src/repowise/core/providers/llm/openai.py
+++ b/packages/core/src/repowise/core/providers/llm/openai.py
@@ -29,6 +29,7 @@ from tenacity import (
from repowise.core.providers.llm.base import (
BaseProvider,
+ CacheHint,
ChatStreamEvent,
ChatToolCall,
GeneratedResponse,
@@ -159,7 +160,11 @@ class OpenAIProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple[CacheHint, ...] = (),
) -> GeneratedResponse:
+ # OpenAI auto-caches stable prompt prefixes >= 1024 tokens; hints are
+ # informational only, so we accept and discard them.
+ del cache_hints
reasoning_mode = _resolve_openai_reasoning_mode(
reasoning, model=self._model
)
@@ -225,15 +230,21 @@ class OpenAIProvider(BaseProvider):
) from exc
usage = response.usage
+ cached = 0
+ if usage is not None:
+ details = getattr(usage, "prompt_tokens_details", None)
+ if details is not None:
+ cached = getattr(details, "cached_tokens", 0) or 0
result = GeneratedResponse(
content=response.choices[0].message.content or "",
input_tokens=usage.prompt_tokens if usage else 0,
output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=0,
+ cached_tokens=cached,
usage={
"prompt_tokens": usage.prompt_tokens if usage else 0,
"completion_tokens": usage.completion_tokens if usage else 0,
"total_tokens": usage.total_tokens if usage else 0,
+ "cached_tokens": cached,
},
)
log.debug(
diff --git a/packages/core/src/repowise/core/providers/llm/openrouter.py b/packages/core/src/repowise/core/providers/llm/openrouter.py
index 6579fa31a..3b31f8d67 100644
--- a/packages/core/src/repowise/core/providers/llm/openrouter.py
+++ b/packages/core/src/repowise/core/providers/llm/openrouter.py
@@ -170,6 +170,7 @@ class OpenRouterProvider(BaseProvider):
temperature: float = 0.3,
request_id: str | None = None,
reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
) -> GeneratedResponse:
reasoning_mode = _resolve_openrouter_reasoning_mode(
reasoning, model=self._model
diff --git a/packages/ui/package.json b/packages/ui/package.json
index f0c7e2399..27573ef37 100644
--- a/packages/ui/package.json
+++ b/packages/ui/package.json
@@ -38,6 +38,7 @@
"./coverage/*": "./src/coverage/*.tsx",
"./wiki": "./src/wiki/index.ts",
"./wiki/wiki-mdx-preprocess": "./src/wiki/wiki-mdx-preprocess.ts",
+ "./wiki/wiki-links-types": "./src/wiki/wiki-links-types.ts",
"./wiki/*": "./src/wiki/*.tsx",
"./chat": "./src/chat/index.ts",
"./chat/*": "./src/chat/*.tsx",
diff --git a/packages/ui/src/docs/docs-tree.tsx b/packages/ui/src/docs/docs-tree.tsx
index afe8d23d2..081a60976 100644
--- a/packages/ui/src/docs/docs-tree.tsx
+++ b/packages/ui/src/docs/docs-tree.tsx
@@ -4,16 +4,30 @@ import { useState, useMemo } from "react";
import {
ChevronDown,
ChevronRight,
+ Compass,
FolderOpen,
Folder,
Search,
Filter,
} from "lucide-react";
-import { getPageTypeIcon, getPageTypeLabel, ALL_PAGE_TYPES } from "../lib/page-types";
+import {
+ ALL_PAGE_TYPES,
+ ONBOARDING_ORDER,
+ ONBOARDING_SLOT_TITLES,
+ getOnboardingSlot,
+ getPageTypeIcon,
+ getPageTypeLabel,
+ type OnboardingSlot,
+} from "../lib/page-types";
import { cn } from "../lib/cn";
import { statusBadgeClasses, type FreshnessStatus } from "../lib/confidence";
import type { DocPage } from "@repowise-dev/types/docs";
+// Synthetic path used as the Onboarding folder's tree key. Distinct from any
+// real target_path (which never starts with "@") so directory lookups don't
+// collide with module paths.
+const ONBOARDING_DIR_KEY = "@onboarding";
+
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
@@ -46,14 +60,68 @@ function PageIcon({ pageType, className }: { pageType: string; className?: strin
// Build tree from flat page list
// ---------------------------------------------------------------------------
+function buildOnboardingFolder(pages: DocPage[]): TreeNode | null {
+ // Bucket every page by its onboarding slot. Both promoted pages
+ // (repo_overview / architecture_diagram, tagged via metadata) and dedicated
+ // `page_type === "onboarding"` pages flow into the same bucket.
+ const bySlot = new Map<OnboardingSlot, DocPage>();
+ for (const page of pages) {
+ const slot = getOnboardingSlot(page);
+ if (slot && !bySlot.has(slot)) {
+ bySlot.set(slot, page);
+ }
+ }
+ if (bySlot.size === 0) return null;
+
+ // Render in canonical reading order. Slots without a page (gated out at
+ // generation time) are silently skipped.
+ const children: TreeNode[] = [];
+ for (const slot of ONBOARDING_ORDER) {
+ const page = bySlot.get(slot);
+ if (!page) continue;
+ children.push({
+ name: ONBOARDING_SLOT_TITLES[slot],
+ path: page.id,
+ isDir: false,
+ page,
+ children: [],
+ });
+ }
+
+ return {
+ name: "Onboarding",
+ path: ONBOARDING_DIR_KEY,
+ isDir: true,
+ children,
+ };
+}
+
function buildTree(pages: DocPage[]): TreeNode[] {
const root: TreeNode[] = [];
- // Separate special pages (overview, architecture) from path-based pages
+ // ---- Onboarding folder (always at top when any slot is filled) ----
+ const onboardingFolder = buildOnboardingFolder(pages);
+ if (onboardingFolder) {
+ root.push(onboardingFolder);
+ }
+
+ // Pages already shown inside the Onboarding folder are skipped at the
+ // top level so they don't appear twice.
+ const onboardingPageIds = new Set(
+ onboardingFolder
+ ? onboardingFolder.children.map((c) => c.page?.id).filter((id): id is string => Boolean(id))
+ : [],
+ );
+
+ // ---- Remaining special pages (overview/architecture only when *not*
+ // already promoted into the Onboarding folder) and path-based pages ----
const specialPages: DocPage[] = [];
const pathPages: DocPage[] = [];
for (const page of pages) {
+ if (onboardingPageIds.has(page.id)) continue;
+ // Dedicated onboarding pages without a recognised slot fall through to
+ // path-based grouping under the "onboarding/" prefix.
if (page.page_type === "repo_overview" || page.page_type === "architecture_diagram") {
specialPages.push(page);
} else {
@@ -61,7 +129,7 @@ function buildTree(pages: DocPage[]): TreeNode[] {
}
}
- // Add special pages at top level
+ // Add remaining special pages at top level
for (const page of specialPages) {
root.push({
name: page.title,
@@ -156,6 +224,15 @@ function buildTree(pages: DocPage[]): TreeNode[] {
}
sortChildren(root);
+
+ // The Onboarding folder is a fixed top-level entry and must appear first,
+ // regardless of alphabetical order against other directories.
+ const onbIdx = root.findIndex((n) => n.path === ONBOARDING_DIR_KEY);
+ if (onbIdx > 0) {
+ const [onbNode] = root.splice(onbIdx, 1);
+ if (onbNode) root.unshift(onbNode);
+ }
+
return root;
}
@@ -258,12 +335,21 @@ function TreeItem({
) : (
<span className="w-3 shrink-0" />
)}
- {isExpanded ? (
+ {node.path === ONBOARDING_DIR_KEY ? (
+ <Compass className="h-3.5 w-3.5 shrink-0 text-[var(--color-accent-primary)]" />
+ ) : isExpanded ? (
<FolderOpen className="h-3.5 w-3.5 shrink-0 text-[var(--color-accent-primary)] opacity-70" />
) : (
<Folder className="h-3.5 w-3.5 shrink-0 text-[var(--color-text-tertiary)]" />
)}
- <span className="truncate font-medium">{node.name}</span>
+ <span
+ className={cn(
+ "truncate font-medium",
+ node.path === ONBOARDING_DIR_KEY && "text-[var(--color-text-primary)]",
+ )}
+ >
+ {node.name}
+ </span>
{node.page && (
<FreshnessDot status={node.page.freshness_status as FreshnessStatus} />
)}
@@ -334,8 +420,9 @@ export function DocsTree({ pages, selectedPageId, onSelectPage, className }: Doc
const [typeFilter, setTypeFilter] = useState<TypeFilter>("all");
const [freshnessFilter, setFreshnessFilter] = useState<FreshnessFilter>("all");
const [expandedDirs, setExpandedDirs] = useState<Set<string>>(() => {
- // Auto-expand first two levels
+ // Auto-expand first two levels, plus the Onboarding folder if present.
const dirs = new Set<string>();
+ dirs.add(ONBOARDING_DIR_KEY);
for (const page of pages) {
const parts = page.target_path.split("/");
if (parts.length > 1 && parts[0]) dirs.add(parts[0]);
diff --git a/packages/ui/src/lib/page-types.ts b/packages/ui/src/lib/page-types.ts
index aaac13f7c..1b1a49d6f 100644
--- a/packages/ui/src/lib/page-types.ts
+++ b/packages/ui/src/lib/page-types.ts
@@ -7,10 +7,10 @@ import {
FileText,
FileCode,
RefreshCw,
- ArrowLeftRight,
Server,
- GitCommit,
+ Compass,
} from "lucide-react";
+import type { DocPage } from "@repowise-dev/types/docs";
export interface PageTypeConfig {
label: string;
@@ -25,9 +25,8 @@ export const PAGE_TYPE_CONFIG: Record<string, PageTypeConfig> = {
file_page: { label: "File", icon: FileText },
api_contract: { label: "API Contract", icon: FileCode },
scc_page: { label: "SCC", icon: RefreshCw },
- cross_package: { label: "Cross-Package", icon: ArrowLeftRight },
infra_page: { label: "Infra", icon: Server },
- diff_summary: { label: "Diff Summary", icon: GitCommit },
+ onboarding: { label: "Onboarding", icon: Compass },
};
export const ALL_PAGE_TYPES = Object.keys(PAGE_TYPE_CONFIG);
@@ -39,3 +38,66 @@ export function getPageTypeIcon(pageType: string): ComponentType<{ className?: s
export function getPageTypeLabel(pageType: string): string {
return PAGE_TYPE_CONFIG[pageType]?.label ?? pageType.replace(/_/g, " ");
}
+
+// ---------------------------------------------------------------------------
+// Onboarding collection
+// ---------------------------------------------------------------------------
+//
+// Keep this list in lockstep with the Python side
+// (`packages/core/src/repowise/core/generation/onboarding/slots.py`).
+// Two slots — project_overview and architecture_guide — are *promoted*: their
+// content lives in the existing repo_overview / architecture_diagram pages,
+// tagged via `metadata.onboarding_slot`. The other six are dedicated
+// `page_type === "onboarding"` pages with `metadata.subkind` discriminating
+// them.
+
+export const ONBOARDING_ORDER = [
+ "project_overview",
+ "architecture_guide",
+ "getting_started",
+ "codebase_map",
+ "key_concepts",
+ "how_it_works",
+ "development_guide",
+ "active_landscape",
+] as const;
+
+export type OnboardingSlot = (typeof ONBOARDING_ORDER)[number];
+
+export const ONBOARDING_SLOT_TITLES: Record<OnboardingSlot, string> = {
+ project_overview: "Project Overview",
+ architecture_guide: "Architecture Guide",
+ getting_started: "Getting Started",
+ codebase_map: "Codebase Map",
+ key_concepts: "Key Concepts",
+ how_it_works: "How It Works",
+ development_guide: "Development Guide",
+ active_landscape: "Active Landscape",
+};
+
+/**
+ * Return the onboarding slot a page belongs to, or null if it isn't part of
+ * the Onboarding collection.
+ *
+ * - Promoted pages (repo_overview, architecture_diagram) carry the slot in
+ * `metadata.onboarding_slot`.
+ * - New onboarding pages (page_type === "onboarding") carry the slot in
+ * `metadata.subkind` (and also `metadata.onboarding_slot` as a mirror).
+ */
+export function getOnboardingSlot(page: DocPage): OnboardingSlot | null {
+ const meta = page.metadata ?? {};
+ const fromSlot = meta["onboarding_slot"];
+ if (typeof fromSlot === "string" && (ONBOARDING_ORDER as readonly string[]).includes(fromSlot)) {
+ return fromSlot as OnboardingSlot;
+ }
+ if (page.page_type === "onboarding") {
+ const subkind = meta["subkind"];
+ if (
+ typeof subkind === "string" &&
+ (ONBOARDING_ORDER as readonly string[]).includes(subkind)
+ ) {
+ return subkind as OnboardingSlot;
+ }
+ }
+ return null;
+}
diff --git a/packages/ui/src/wiki/backlinks-panel.tsx b/packages/ui/src/wiki/backlinks-panel.tsx
new file mode 100644
index 000000000..3aa89c895
--- /dev/null
+++ b/packages/ui/src/wiki/backlinks-panel.tsx
@@ -0,0 +1,76 @@
+/**
+ * Sidebar panel that lists pages linking *into* the current page.
+ *
+ * Reads ``page.metadata.backlinks`` populated by the backend's
+ * interlinking post-processor. Empty/missing metadata renders nothing
+ * (graceful fallback for pages generated before interlinking shipped).
+ */
+
+import Link from "next/link";
+import { getPageTypeIcon, getPageTypeLabel } from "../lib/page-types";
+import type { Backlink } from "./wiki-links-types";
+
+export interface BacklinksPanelProps {
+ /** ``page.metadata.backlinks`` from the page response. */
+ backlinks: Backlink[];
+ /** Repo ID for routing the source-page links. */
+ repoId: string;
+ /** Maximum entries to show before "+N more". */
+ limit?: number;
+ /** Optional URL builder; defaults to ``/repos/{repoId}/wiki/{pageId}``. */
+ buildHref?: (repoId: string, pageId: string) => string;
+}
+
+const DEFAULT_LIMIT = 6;
+
+function defaultHref(repoId: string, pageId: string): string {
+ // PageIds are like ``file_page:src/foo.py``. The router resolves them
+ // via the catch-all slug parameter.
+ return `/repos/${repoId}/wiki/${encodeURIComponent(pageId)}`;
+}
+
+export function BacklinksPanel({
+ backlinks,
+ repoId,
+ limit = DEFAULT_LIMIT,
+ buildHref = defaultHref,
+}: BacklinksPanelProps) {
+ if (!backlinks || backlinks.length === 0) {
+ return null;
+ }
+
+ const visible = backlinks.slice(0, limit);
+ const extra = backlinks.length - visible.length;
+
+ return (
+ <div>
+ <p className="text-xs font-medium text-[var(--color-text-tertiary)] uppercase tracking-wider mb-2">
+ Linked by ({backlinks.length})
+ </p>
+ <ul className="space-y-1.5">
+ {visible.map((bl) => {
+ const Icon = getPageTypeIcon(bl.source_page_type);
+ return (
+ <li key={bl.source_page_id} className="text-xs">
+ <Link
+ href={buildHref(repoId, bl.source_page_id)}
+ className="flex items-start gap-1.5 text-[var(--color-text-secondary)] hover:text-[var(--color-accent-primary)] transition-colors"
+ title={getPageTypeLabel(bl.source_page_type)}
+ >
+ <Icon className="h-3 w-3 mt-0.5 shrink-0 text-[var(--color-text-tertiary)]" />
+ <span className="truncate" title={bl.source_title}>
+ {bl.source_title}
+ </span>
+ </Link>
+ </li>
+ );
+ })}
+ </ul>
+ {extra > 0 && (
+ <p className="text-[10px] text-[var(--color-text-tertiary)] mt-1.5">
+ + {extra} more
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/packages/ui/src/wiki/index.ts b/packages/ui/src/wiki/index.ts
index da2c62b26..b322e6df5 100644
--- a/packages/ui/src/wiki/index.ts
+++ b/packages/ui/src/wiki/index.ts
@@ -1,3 +1,4 @@
+export * from "./backlinks-panel.js";
export * from "./code-block.js";
export * from "./confidence-badge.js";
export * from "./git-history-panel.js";
@@ -6,6 +7,8 @@ export * from "./regenerate-button.js";
export * from "./security-panel.js";
export * from "./table-of-contents.js";
export * from "./version-history.js";
+export * from "./wiki-link.js";
+export * from "./wiki-links-types.js";
export * from "./wiki-markdown.js";
export * from "./wiki-mdx-components.js";
export * from "./wiki-mdx-preprocess.js";
diff --git a/packages/ui/src/wiki/wiki-link.tsx b/packages/ui/src/wiki/wiki-link.tsx
new file mode 100644
index 000000000..9096ace73
--- /dev/null
+++ b/packages/ui/src/wiki/wiki-link.tsx
@@ -0,0 +1,28 @@
+/**
+ * Inline anchor for a resolved wiki cross-reference.
+ *
+ * Rendered by the MDX preprocessor wherever a backticked path/symbol
+ * was matched to another wiki page. Styled to look like a `<code>`
+ * inline literal — preserves the dev-doc feel — but with a subtle
+ * dotted underline so users can tell it's clickable.
+ */
+
+import type { ReactNode } from "react";
+
+export interface WikiLinkProps {
+ href: string;
+ kind?: "file" | "symbol" | "page";
+ children?: ReactNode;
+}
+
+export function WikiLink({ href, kind = "page", children }: WikiLinkProps) {
+ return (
+ <a
+ href={href}
+ data-wiki-link-kind={kind}
+ className="rounded bg-[var(--color-bg-elevated)] px-1.5 py-0.5 text-xs font-mono text-[var(--color-accent-primary)] underline decoration-dotted underline-offset-2 hover:opacity-80 transition-opacity"
+ >
+ {children}
+ </a>
+ );
+}
diff --git a/packages/ui/src/wiki/wiki-links-types.ts b/packages/ui/src/wiki/wiki-links-types.ts
new file mode 100644
index 000000000..2789a3de6
--- /dev/null
+++ b/packages/ui/src/wiki/wiki-links-types.ts
@@ -0,0 +1,89 @@
+/**
+ * Typed accessors for page.metadata.wiki_links and page.metadata.backlinks.
+ *
+ * Both fields are populated by the backend's interlinking post-processor
+ * (`packages/core/.../generation/interlinking.py`). They live inside the
+ * generic `metadata: Record<string, unknown>` blob so the API schema
+ * doesn't need a breaking change every time the backend learns to
+ * resolve a new kind of reference.
+ *
+ * The accessors below are defensive — they tolerate missing fields,
+ * legacy pages without metadata, and unexpected shapes.
+ */
+
+export type WikiLinkKind = "file" | "symbol" | "page";
+
+/**
+ * Forward wiki-link entry persisted in ``page.metadata.wiki_links``.
+ *
+ * Note: deliberately named ``WikiLinkRef`` (not ``WikiLink``) to avoid
+ * a name collision with the ``WikiLink`` React component re-exported
+ * from the same package barrel.
+ */
+export interface WikiLinkRef {
+ anchor: string;
+ target_page_id: string;
+ kind: WikiLinkKind;
+}
+
+export interface Backlink {
+ source_page_id: string;
+ source_title: string;
+ source_page_type: string;
+ anchor: string;
+}
+
+function isWikiLinkRef(v: unknown): v is WikiLinkRef {
+ if (typeof v !== "object" || v === null) return false;
+ const r = v as Record<string, unknown>;
+ return (
+ typeof r["anchor"] === "string" &&
+ typeof r["target_page_id"] === "string" &&
+ typeof r["kind"] === "string"
+ );
+}
+
+function isBacklink(v: unknown): v is Backlink {
+ if (typeof v !== "object" || v === null) return false;
+ const r = v as Record<string, unknown>;
+ return (
+ typeof r["source_page_id"] === "string" &&
+ typeof r["source_title"] === "string" &&
+ typeof r["source_page_type"] === "string"
+ );
+}
+
+export function getWikiLinks(
+ metadata: Record<string, unknown> | undefined | null,
+): WikiLinkRef[] {
+ if (!metadata) return [];
+ const raw = metadata["wiki_links"];
+ if (!Array.isArray(raw)) return [];
+ return raw.filter(isWikiLinkRef);
+}
+
+export function getBacklinks(
+ metadata: Record<string, unknown> | undefined | null,
+): Backlink[] {
+ if (!metadata) return [];
+ const raw = metadata["backlinks"];
+ if (!Array.isArray(raw)) return [];
+ return raw.filter(isBacklink);
+}
+
+/**
+ * Build a lookup: anchor → target_page_id for fast O(1) rewrite inside
+ * the MDX preprocessor. Multiple anchors mapping to the same target are
+ * fine; conflicting anchors are resolved first-wins.
+ */
+export function buildAnchorIndex(
+ links: WikiLinkRef[],
+): Map<string, { pageId: string; kind: WikiLinkKind }> {
+ const idx = new Map<string, { pageId: string; kind: WikiLinkKind }>();
+ for (const link of links) {
+ if (!idx.has(link.anchor)) {
+ idx.set(link.anchor, { pageId: link.target_page_id, kind: link.kind });
+ }
+ }
+ return idx;
+}
diff --git a/packages/ui/src/wiki/wiki-mdx-preprocess.ts b/packages/ui/src/wiki/wiki-mdx-preprocess.ts
index d8d7cfad1..66e0f66d3 100644
--- a/packages/ui/src/wiki/wiki-mdx-preprocess.ts
+++ b/packages/ui/src/wiki/wiki-mdx-preprocess.ts
@@ -3,6 +3,8 @@
// is async and provided by the caller so the package never imports `shiki`
// directly — apps decide whether to run it server-side or client-side.
+import type { WikiLinkKind } from "./wiki-links-types";
+
export interface PreprocessedSource {
source: string;
}
@@ -12,10 +14,74 @@ export type ShikiHighlighter = (
lang: string,
) => Promise<string>;
+/**
+ * Resolver function passed into the preprocessor by the host app.
+ *
+ * Given the raw anchor text inside backticks (e.g. ``src/foo.py``),
+ * returns either a ``{ pageId, kind }`` pair when the ref maps to
+ * a known wiki page, or ``null`` when it should render as plain
+ * inline code. The host decides where ``pageId`` becomes a URL.
+ */
+export type WikiLinkResolver = (
+ anchor: string,
+) => { pageId: string; kind: WikiLinkKind } | null;
+
+/**
+ * Build a URL for a resolved wiki link. Default routes under
+ * ``/repos/{repoId}/wiki/{pageId}``; callers can override per app.
+ */
+export type WikiLinkUrlBuilder = (
+ pageId: string,
+ kind: WikiLinkKind,
+) => string;
+
+export interface PreprocessOptions {
+ highlight: ShikiHighlighter;
+ resolveWikiLink?: WikiLinkResolver;
+ buildWikiLinkUrl?: WikiLinkUrlBuilder;
+}
+
+// Inline backtick refs — single-backtick, identifier-ish content. Excludes
+// triple-backtick fences (handled earlier) and ignores refs that already
+// look like part of a URL or contain whitespace.
+const INLINE_BACKTICK_RE = /(?<!`)`([^`\n]+?)`(?!`)/g;
+
+function rewriteInlineBackticks(
+ text: string,
+ resolver: WikiLinkResolver,
+ buildUrl: WikiLinkUrlBuilder,
+): string {
+ return text.replace(INLINE_BACKTICK_RE, (whole, raw: string) => {
+ const ref = raw.trim();
+ if (!ref) return whole;
+ const resolved = resolver(ref);
+ if (!resolved) return whole;
+ const href = buildUrl(resolved.pageId, resolved.kind);
+ // MDX consumes the JSX literal; the host package provides the
+ // ``WikiLink`` component via its ``components`` map.
+ return `<WikiLink href=${JSON.stringify(href)} kind=${JSON.stringify(
+ resolved.kind,
+ )}>${escapeMdxText(ref)}</WikiLink>`;
+ });
+}
+
+function escapeMdxText(s: string): string {
+ // MDX treats `{` and `<` specially. Escape both so a ref like
+ // ``Map<string, int>`` doesn't open a JSX tag.
+ return s.replace(/[<{]/g, (ch) => `{"${ch}"}`);
+}
+
export async function preprocessWikiCodeBlocks(
content: string,
- highlight: ShikiHighlighter,
+ highlightOrOpts: ShikiHighlighter | PreprocessOptions,
): Promise<PreprocessedSource> {
+ // Backwards-compatible signature: previous callers passed the
+ // highlighter directly. New callers pass the options object so they
+ // can include the wiki-link resolver.
+ const opts: PreprocessOptions =
+ typeof highlightOrOpts === "function"
+ ? { highlight: highlightOrOpts }
+ : highlightOrOpts;
const fenceRe = /```(\w*)\n([\s\S]*?)```/g;
const replacements: Array<{ placeholder: string; jsx: string }> = [];
@@ -37,7 +103,7 @@ export async function preprocessWikiCodeBlocks(
let html = "";
try {
- html = await highlight(code, lang);
+ html = await opts.highlight(code, lang);
} catch {
html = `<pre><code>${code.replace(/</g, "&lt;")}</code></pre>`;
}
@@ -55,5 +121,15 @@ export async function preprocessWikiCodeBlocks(
for (const { placeholder, jsx } of replacements) {
source = source.replace(placeholder, jsx);
}
+
+ // Inline backtick rewriting runs AFTER fence replacement so we never
+ // touch the source inside fenced code samples (replaced earlier with
+ // JSX tags that the regex skips).
+ if (opts.resolveWikiLink) {
+ const buildUrl =
+ opts.buildWikiLinkUrl ?? ((pid) => `/wiki/${encodeURIComponent(pid)}`);
+ source = rewriteInlineBackticks(source, opts.resolveWikiLink, buildUrl);
+ }
+
return { source };
}
diff --git a/packages/web/src/app/repos/[id]/wiki/[...slug]/page.tsx b/packages/web/src/app/repos/[id]/wiki/[...slug]/page.tsx
index 278d1d20b..fe7f15d04 100644
--- a/packages/web/src/app/repos/[id]/wiki/[...slug]/page.tsx
+++ b/packages/web/src/app/repos/[id]/wiki/[...slug]/page.tsx
@@ -12,6 +12,11 @@ import { TableOfContents } from "@repowise-dev/ui/wiki/table-of-contents";
import { RegenerateButtonWrapper } from "@/components/wiki/regenerate-button";
import { GitHistoryPanel } from "@repowise-dev/ui/wiki/git-history-panel";
import { SecurityPanelWrapper } from "@/components/wiki/security-panel";
+import { BacklinksPanel } from "@repowise-dev/ui/wiki/backlinks-panel";
+import {
+ getBacklinks,
+ getWikiLinks,
+} from "@repowise-dev/ui/wiki/wiki-links-types";
import { Badge } from "@repowise-dev/ui/ui/badge";
import { Separator } from "@repowise-dev/ui/ui/separator";
import { formatRelativeTime, formatTokens } from "@repowise-dev/ui/lib/format";
@@ -126,7 +131,11 @@ export default async function WikiPageRoute({ params }: Props) {
)}
<article className="prose prose-invert max-w-none leading-relaxed overflow-hidden">
- <WikiRenderer content={page.content} />
+ <WikiRenderer
+ content={page.content}
+ wikiLinks={getWikiLinks(page.metadata)}
+ repoId={id}
+ />
</article>
{/* Hallucination warnings */}
@@ -241,6 +250,14 @@ export default async function WikiPageRoute({ params }: Props) {
</div>
)}
+ {/* Backlinks — pages mentioning this one. Hidden when empty
+ so legacy pages without interlinking metadata don't show
+ an awkward "Linked by (0)" header. */}
+ <BacklinksPanel
+ backlinks={getBacklinks(page.metadata)}
+ repoId={id}
+ />
+
{/* Security findings panel */}
{page.target_path && (
<div>
diff --git a/packages/web/src/components/settings/provider-section.tsx b/packages/web/src/components/settings/provider-section.tsx
index b9ce0ed86..f5a9ac419 100644
--- a/packages/web/src/components/settings/provider-section.tsx
+++ b/packages/web/src/components/settings/provider-section.tsx
@@ -19,7 +19,7 @@ const EMBEDDERS = ["mock", "gemini", "openai"] as const;
const MODEL_PLACEHOLDERS: Record<string, string> = {
gemini: "gemini-3.1-flash-lite-preview",
- openai: "gpt-4.1",
+ openai: "gpt-5.4-nano",
anthropic: "claude-sonnet-4-6",
deepseek: "deepseek-v4-flash",
ollama: "llama3.2",
diff --git a/packages/web/src/components/wiki/wiki-renderer.tsx b/packages/web/src/components/wiki/wiki-renderer.tsx
index 2d580b152..138e5e2f4 100644
--- a/packages/web/src/components/wiki/wiki-renderer.tsx
+++ b/packages/web/src/components/wiki/wiki-renderer.tsx
@@ -4,18 +4,39 @@ import { CodeBlock } from "@repowise-dev/ui/wiki/code-block";
import { MermaidDiagram } from "@repowise-dev/ui/wiki/mermaid-diagram";
import { wikiMdxComponents } from "@repowise-dev/ui/wiki/wiki-mdx-components";
import { preprocessWikiCodeBlocks } from "@repowise-dev/ui/wiki/wiki-mdx-preprocess";
+import { WikiLink } from "@repowise-dev/ui/wiki/wiki-link";
+import {
+ buildAnchorIndex,
+ type WikiLinkRef,
+} from "@repowise-dev/ui/wiki/wiki-links-types";
interface Props {
content: string;
+ /** Pre-resolved wiki links from ``page.metadata.wiki_links``. */
+ wikiLinks?: WikiLinkRef[];
+ /** Repo ID required to build cross-page hrefs. */
+ repoId?: string;
}
-export async function WikiRenderer({ content }: Props) {
- const preprocessed = await preprocessWikiCodeBlocks(content, async (code, lang) =>
- codeToHtml(code, {
- lang: lang as Parameters<typeof codeToHtml>[1]["lang"],
- theme: "vesper",
- }),
- );
+export async function WikiRenderer({ content, wikiLinks, repoId }: Props) {
+ // Build the anchor → page_id index once per render; the preprocessor
+ // does an O(1) Map lookup per backtick match.
+ const index = buildAnchorIndex(wikiLinks ?? []);
+ const resolveWikiLink = repoId
+ ? (anchor: string) => index.get(anchor) ?? null
+ : undefined;
+ const buildWikiLinkUrl = (pageId: string) =>
+ `/repos/${repoId}/wiki/${encodeURIComponent(pageId)}`;
+
+ const preprocessed = await preprocessWikiCodeBlocks(content, {
+ highlight: async (code, lang) =>
+ codeToHtml(code, {
+ lang: lang as Parameters<typeof codeToHtml>[1]["lang"],
+ theme: "vesper",
+ }),
+ resolveWikiLink,
+ buildWikiLinkUrl,
+ });
return (
<MDXRemote
@@ -28,6 +49,7 @@ export async function WikiRenderer({ content }: Props) {
MermaidBlock: ({ chart }: { chart: string }) => (
<MermaidDiagram chart={chart} />
),
+ WikiLink,
}}
options={{
parseFrontmatter: false,
diff --git a/tests/unit/generation/test_api_contract_detector.py b/tests/unit/generation/test_api_contract_detector.py
new file mode 100644
index 000000000..66e9142ad
--- /dev/null
+++ b/tests/unit/generation/test_api_contract_detector.py
@@ -0,0 +1,107 @@
+"""Tests for the framework-aware api_contract detector."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from repowise.core.generation.api_contract_detector import detect_code_api_contracts
+from repowise.core.ingestion.models import FileInfo, Import, ParsedFile, Symbol
+
+
+def _file(path: str, language: str) -> FileInfo:
+ return FileInfo(
+ path=path,
+ abs_path=f"/repo/{path}",
+ language=language, # type: ignore[arg-type]
+ size_bytes=100,
+ git_hash="",
+ last_modified=datetime.now(),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+
+
+def _sym(name: str, kind: str = "function", decorators: list[str] | None = None, signature: str = "") -> Symbol:
+ return Symbol(
+ id=f"x::{name}",
+ name=name,
+ qualified_name=name,
+ kind=kind, # type: ignore[arg-type]
+ signature=signature,
+ start_line=1,
+ end_line=2,
+ docstring=None,
+ decorators=decorators or [],
+ )
+
+
+def _imp(module: str, names: list[str]) -> Import:
+ return Import(
+ raw_statement=f"from {module} import " + ", ".join(names),
+ module_path=module,
+ imported_names=names,
+ is_relative=False,
+ resolved_file=None,
+ )
+
+
+def _parsed(path: str, language: str, *, imports=(), symbols=()) -> ParsedFile:
+ return ParsedFile(
+ file_info=_file(path, language),
+ symbols=list(symbols),
+ imports=list(imports),
+ exports=[],
+ )
+
+
+def test_detects_fastapi_router_via_import():
+ pf = _parsed("app/routes.py", "python", imports=[_imp("fastapi", ["APIRouter"])])
+ assert detect_code_api_contracts([pf]) == 1
+ assert pf.file_info.is_api_contract is True
+
+
+def test_detects_fastapi_via_method_decorator():
+ pf = _parsed(
+ "app/routes.py",
+ "python",
+ imports=[_imp("fastapi", ["Depends"])],
+ symbols=[_sym("list_items", decorators=["@router.get('/items')"])],
+ )
+ assert detect_code_api_contracts([pf]) == 1
+
+
+def test_skips_python_file_without_fastapi():
+ pf = _parsed("app/utils.py", "python", imports=[_imp("os", ["path"])])
+ assert detect_code_api_contracts([pf]) == 0
+ assert pf.file_info.is_api_contract is False
+
+
+def test_detects_aspnet_controller_via_inheritance():
+ pf = _parsed(
+ "Controllers/UsersController.cs",
+ "csharp",
+ symbols=[_sym("UsersController", kind="class", signature="class UsersController : ControllerBase")],
+ )
+ assert detect_code_api_contracts([pf]) == 1
+
+
+def test_detects_aspnet_controller_via_attribute():
+ pf = _parsed(
+ "Controllers/UsersController.cs",
+ "csharp",
+ symbols=[_sym("UsersController", kind="class", decorators=["[ApiController]"])],
+ )
+ assert detect_code_api_contracts([pf]) == 1
+
+
+def test_already_flagged_file_is_untouched():
+ pf = _parsed("api/openapi.yaml", "yaml")
+ pf.file_info.is_api_contract = True
+ assert detect_code_api_contracts([pf]) == 0 # already flagged, not "newly flagged"
+
+
+def test_unknown_language_skipped():
+ pf = _parsed("main.rb", "ruby")
+ assert detect_code_api_contracts([pf]) == 0
diff --git a/tests/unit/generation/test_context_assembler.py b/tests/unit/generation/test_context_assembler.py
index d7ab2ca8c..99648dda5 100644
--- a/tests/unit/generation/test_context_assembler.py
+++ b/tests/unit/generation/test_context_assembler.py
@@ -288,6 +288,76 @@ def test_assemble_repo_overview_circular_dep_count(
# ---------------------------------------------------------------------------
+def test_assemble_file_page_threads_decision_records(
+ sample_config, sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes
+):
+ assembler = ContextAssembler(sample_config)
+ decisions = [{"title": "Use SQLAlchemy", "decision": "ORM", "rationale": "type-safe"}]
+ ctx = assembler.assemble_file_page(
+ sample_parsed_file,
+ sample_graph,
+ graph_metrics["pagerank"],
+ graph_metrics["betweenness"],
+ graph_metrics["community"],
+ sample_source_bytes,
+ decision_records=decisions,
+ )
+ assert ctx.decision_records == decisions
+
+
+def test_assemble_module_page_threads_phase2_signals(
+ sample_config, sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes
+):
+ assembler = ContextAssembler(sample_config)
+ fc = assembler.assemble_file_page(
+ sample_parsed_file,
+ sample_graph,
+ graph_metrics["pagerank"],
+ graph_metrics["betweenness"],
+ graph_metrics["community"],
+ sample_source_bytes,
+ )
+ decisions = [{"title": "X", "decision": "Y", "rationale": ""}]
+ dead = [{"symbol_name": "foo", "reason": "no callers", "confidence": 0.9, "safe_to_delete": True}]
+ externals = [{"name": "fastapi", "category": "framework", "ecosystem": "pypi"}]
+ ctx = assembler.assemble_module_page(
+ "auth/login",
+ "python",
+ [fc],
+ sample_graph,
+ decision_records=decisions,
+ dead_code_findings=dead,
+ external_systems=externals,
+ community_label="auth/login",
+ community_cohesion=0.72,
+ )
+ assert ctx.decision_records == decisions
+ assert ctx.dead_code_findings == dead
+ assert ctx.external_systems == externals
+ assert ctx.community_label == "auth/login"
+ assert ctx.community_cohesion == 0.72
+ # key_files derives from the file contexts we passed in
+ assert ctx.key_files and ctx.key_files[0]["path"] == fc.file_path
+
+
+def test_assemble_repo_overview_threads_external_systems_and_decisions(
+ sample_config, sample_graph, sample_repo_structure
+):
+ assembler = ContextAssembler(sample_config)
+ externals = [{"name": "redis", "category": "datastore", "ecosystem": "pypi"}]
+ decisions = [{"title": "Adopt Redis", "decision": "session cache", "rationale": "low latency"}]
+ ctx = assembler.assemble_repo_overview(
+ sample_repo_structure,
+ {},
+ [],
+ {},
+ external_systems=externals,
+ decision_records=decisions,
+ )
+ assert ctx.external_systems == externals
+ assert ctx.decision_records == decisions
+
+
def test_assemble_api_contract_raw_content_budgeted(sample_config, sample_parsed_file):
assembler = ContextAssembler(sample_config)
huge_bytes = b"x" * 100_000
diff --git a/tests/unit/generation/test_interlinking.py b/tests/unit/generation/test_interlinking.py
new file mode 100644
index 000000000..29ca3bd81
--- /dev/null
+++ b/tests/unit/generation/test_interlinking.py
@@ -0,0 +1,133 @@
+"""Unit tests for the interlinking post-processor."""
+
+from __future__ import annotations
+
+from repowise.core.generation.interlinking import (
+ LinkIndex,
+ attach_wiki_links_and_backlinks,
+)
+from repowise.core.generation.models import GeneratedPage
+
+
+def _make_page(
+ page_type: str,
+ target_path: str,
+ content: str = "",
+ *,
+ title: str = "",
+) -> GeneratedPage:
+ return GeneratedPage(
+ page_id=f"{page_type}:{target_path}",
+ page_type=page_type,
+ title=title or f"{page_type} {target_path}",
+ content=content,
+ source_hash="x",
+ model_name="mock",
+ provider_name="mock",
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ generation_level=0,
+ target_path=target_path,
+ created_at="2026-01-01T00:00:00+00:00",
+ updated_at="2026-01-01T00:00:00+00:00",
+ )
+
+
+def test_resolve_links_for_file_reference():
+ """A backtick-wrapped file path resolves to that file's page_id."""
+ pages = [
+ _make_page(
+ "file_page",
+ "src/auth/service.py",
+ content="See `src/auth/utils.py` for helper functions.",
+ ),
+ _make_page("file_page", "src/auth/utils.py", content="utility functions"),
+ ]
+
+ attach_wiki_links_and_backlinks(pages, parsed_files=None)
+
+ service_links = pages[0].metadata["wiki_links"]
+ assert any(
+ link["target_page_id"] == "file_page:src/auth/utils.py"
+ for link in service_links
+ )
+
+ utils_backlinks = pages[1].metadata["backlinks"]
+ assert any(
+ bl["source_page_id"] == "file_page:src/auth/service.py"
+ for bl in utils_backlinks
+ )
+
+
+def test_dedup_same_target_referenced_multiple_times():
+ """Multiple mentions of one target collapse to one link."""
+ pages = [
+ _make_page(
+ "file_page",
+ "a.py",
+ content="`b.py` does X. Earlier we noted `b.py` and then `b.py`.",
+ ),
+ _make_page("file_page", "b.py", content=""),
+ ]
+
+ attach_wiki_links_and_backlinks(pages)
+
+ assert len(pages[0].metadata["wiki_links"]) == 1
+
+
+def test_self_reference_excluded():
+ """A page referencing itself does not create a self-link."""
+ pages = [
+ _make_page(
+ "file_page",
+ "a.py",
+ content="This file `a.py` documents itself.",
+ ),
+ ]
+
+ attach_wiki_links_and_backlinks(pages)
+
+ assert pages[0].metadata["wiki_links"] == []
+
+
+def test_unresolved_refs_are_dropped_silently():
+ """Refs that point to non-existent pages produce no metadata noise."""
+ pages = [
+ _make_page(
+ "file_page",
+ "a.py",
+ content="See `does/not/exist.py` for context.",
+ ),
+ ]
+
+ attach_wiki_links_and_backlinks(pages)
+
+ assert pages[0].metadata["wiki_links"] == []
+ assert pages[0].metadata["backlinks"] == []
+
+
+def test_index_resolves_basename_when_unique():
+ """`utils.py` alone resolves if there's exactly one matching page."""
+ pages = [
+ _make_page("file_page", "src/lib/utils.py", content=""),
+ _make_page("file_page", "src/app/main.py", content="Calls into `utils.py`."),
+ ]
+
+ attach_wiki_links_and_backlinks(pages)
+
+ main_links = pages[1].metadata["wiki_links"]
+ assert any(
+ link["target_page_id"] == "file_page:src/lib/utils.py" for link in main_links
+ )
+
+
+def test_link_index_build_returns_expected_keys():
+ pages = [
+ _make_page("file_page", "src/x.py"),
+ _make_page("module_page", "community-3"),
+ ]
+ idx = LinkIndex.build(pages, parsed_files=None)
+ assert idx.by_path["src/x.py"] == "file_page:src/x.py"
+ assert idx.by_target["community-3"] == "module_page:community-3"
+ assert idx.by_basename["x.py"] == "file_page:src/x.py"
diff --git a/tests/unit/generation/test_models.py b/tests/unit/generation/test_models.py
index fab450a4d..beab47c53 100644
--- a/tests/unit/generation/test_models.py
+++ b/tests/unit/generation/test_models.py
@@ -144,15 +144,17 @@ def test_decay_confidence_beyond_expiry_is_zero():
def test_generation_config_defaults():
config = GenerationConfig()
- assert config.max_tokens == 16000
+ assert config.max_tokens == 20000
assert config.temperature == 0.3
assert config.token_budget == 48000
- assert config.max_concurrency == 5
- assert config.embed_concurrency == 5
+ assert config.max_concurrency == 12
+ assert config.embed_concurrency == 12
assert config.cache_enabled is True
assert config.staleness_threshold_days == 7
assert config.expiry_threshold_days == 30
- assert config.top_symbol_percentile == 0.10
+ assert config.top_symbol_percentile == 0.20
+ assert config.module_grouping == "community"
+ assert config.min_module_size == 3
assert config.large_file_source_pct == 0.4
assert config.reasoning == "auto"
diff --git a/tests/unit/generation/test_onboarding.py b/tests/unit/generation/test_onboarding.py
new file mode 100644
index 000000000..fb8b862f7
--- /dev/null
+++ b/tests/unit/generation/test_onboarding.py
@@ -0,0 +1,550 @@
+"""Tests for the Phase 3 Onboarding collection.
+
+Covers:
+ - registry contents + canonical order
+ - each subkind's gate (positive + negative cases)
+ - `_tag_promoted_pages` on PageGenerator
+ - templates render against built contexts without StrictUndefined errors
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import jinja2
+import pytest
+
+from repowise.core.generation import onboarding
+from repowise.core.generation.models import GENERATION_LEVELS, GeneratedPage
+from repowise.core.generation.onboarding.signals import OnboardingSignals
+from repowise.core.generation.onboarding.slots import (
+ ONBOARDING_ORDER,
+ PROMOTED_SLOTS,
+ SLOT_ACTIVE_LANDSCAPE,
+ SLOT_CODEBASE_MAP,
+ SLOT_DEVELOPMENT_GUIDE,
+ SLOT_GETTING_STARTED,
+ SLOT_HOW_IT_WORKS,
+ SLOT_KEY_CONCEPTS,
+ target_path,
+)
+from repowise.core.generation.page_generator import PageGenerator
+from repowise.core.ingestion.models import (
+ FileInfo,
+ ParsedFile,
+ RepoStructure,
+ Symbol,
+)
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _file(
+ path: str,
+ *,
+ language: str = "python",
+ is_entry_point: bool = False,
+ symbols: list[str] | None = None,
+) -> ParsedFile:
+ fi = FileInfo(
+ path=path,
+ abs_path=f"/repo/{path}",
+ language=language,
+ size_bytes=512,
+ git_hash="abc",
+ last_modified=datetime(2026, 1, 1, tzinfo=UTC),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=is_entry_point,
+ )
+ syms: list[Symbol] = []
+ for s in symbols or []:
+ syms.append(
+ Symbol(
+ id=f"{path}::{s}",
+ name=s,
+ qualified_name=f"{path.replace('/', '.')}::{s}",
+ kind="class",
+ signature=f"class {s}:",
+ start_line=1,
+ end_line=10,
+ docstring=f"Docstring for {s}.",
+ decorators=[],
+ visibility="public",
+ is_async=False,
+ complexity_estimate=1,
+ language=language,
+ parent_name=None,
+ )
+ )
+ return ParsedFile(
+ file_info=fi,
+ symbols=syms,
+ imports=[],
+ exports=[s for s in (symbols or [])],
+ docstring=None,
+ parse_errors=[],
+ content_hash="abc",
+ )
+
+
+def _signals(
+ *,
+ files: list[ParsedFile],
+ pagerank: dict[str, float] | None = None,
+ source_map: dict[str, bytes] | None = None,
+ git_meta: dict[str, dict] | None = None,
+ external_systems: tuple[dict, ...] = (),
+ decisions: tuple[dict, ...] = (),
+ community: dict[str, int] | None = None,
+ entry_points: list[str] | None = None,
+) -> OnboardingSignals:
+ paths = [f.file_info.path for f in files]
+ pr = pagerank or {p: 0.1 for p in paths}
+ com = community or dict.fromkeys(paths, 0)
+ # Minimal fake graph_builder — community_info / execution_flows return empty.
+ graph_builder = SimpleNamespace(
+ community_info=lambda: {},
+ execution_flows=lambda: SimpleNamespace(flows=[]),
+ )
+ repo_structure = RepoStructure(
+ is_monorepo=False,
+ packages=[],
+ root_language_distribution={"python": 1.0},
+ total_files=len(files),
+ total_loc=len(files) * 50,
+ entry_points=entry_points or [f.file_info.path for f in files if f.file_info.is_entry_point],
+ )
+ return OnboardingSignals(
+ repo_name="testrepo",
+ repo_structure=repo_structure,
+ parsed_files=tuple(files),
+ source_map=source_map or {},
+ graph_builder=graph_builder,
+ pagerank=pr,
+ betweenness={p: 0.0 for p in paths},
+ community=com,
+ sccs=(),
+ git_meta_map=git_meta,
+ dead_code_by_file={},
+ decisions_all=decisions,
+ external_systems=external_systems,
+ completed_page_summaries={},
+ )
+
+
+# ---------------------------------------------------------------------------
+# Registry + ordering
+# ---------------------------------------------------------------------------
+
+
+def test_six_subkinds_registered_in_canonical_order() -> None:
+ specs = onboarding.iter_specs()
+ slots = [s.slot for s in specs]
+ # Promoted slots are excluded from iter_specs even though they're in
+ # ONBOARDING_ORDER.
+ expected = [s for s in ONBOARDING_ORDER if s not in PROMOTED_SLOTS.values()]
+ assert slots == expected
+ assert len(slots) == 6
+
+
+def test_onboarding_level_is_eight() -> None:
+ assert GENERATION_LEVELS["onboarding"] == 8
+
+
+def test_target_path_format() -> None:
+ assert target_path("codebase_map") == "onboarding/codebase_map"
+
+
+def test_promoted_slots_map() -> None:
+ assert PROMOTED_SLOTS == {
+ "repo_overview": "project_overview",
+ "architecture_diagram": "architecture_guide",
+ }
+
+
+# ---------------------------------------------------------------------------
+# Codebase map — always generates
+# ---------------------------------------------------------------------------
+
+
+def test_codebase_map_always_builds() -> None:
+ spec = onboarding.get_spec(SLOT_CODEBASE_MAP)
+ assert spec is not None
+ files = [
+ _file("src/auth/login.py", symbols=["Login"]),
+ _file("src/auth/session.py", symbols=["Session"]),
+ _file("src/auth/middleware.py", symbols=["AuthMiddleware"]),
+ _file("src/api/routes.py", symbols=["Router"]),
+ _file("src/api/handlers.py", symbols=["Handler"]),
+ _file("src/api/serializers.py", symbols=["Serializer"]),
+ ]
+ sig = _signals(files=files)
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ assert len(ctx.directories) >= 1
+ # Largest dir first
+ assert ctx.directories[0].file_count >= ctx.directories[-1].file_count
+
+
+# ---------------------------------------------------------------------------
+# Active landscape — gated on git churn
+# ---------------------------------------------------------------------------
+
+
+def test_active_landscape_skipped_without_git_meta() -> None:
+ spec = onboarding.get_spec(SLOT_ACTIVE_LANDSCAPE)
+ assert spec is not None
+ sig = _signals(files=[_file("src/a.py")], git_meta=None)
+ assert spec.build_context(sig) is None
+
+
+def test_active_landscape_skipped_below_threshold() -> None:
+ spec = onboarding.get_spec(SLOT_ACTIVE_LANDSCAPE)
+ assert spec is not None
+ files = [_file(f"src/f{i}.py") for i in range(20)]
+ # 5 files * 5 commits each = 25 commits total — below 50 floor.
+ git_meta = {f.file_info.path: {"commit_count_90d": 5} for f in files[:5]}
+ sig = _signals(files=files, git_meta=git_meta)
+ assert spec.build_context(sig) is None
+
+
+def test_active_landscape_fires_above_threshold() -> None:
+ spec = onboarding.get_spec(SLOT_ACTIVE_LANDSCAPE)
+ assert spec is not None
+ files = [_file(f"src/f{i}.py") for i in range(20)]
+ # 15 files * 10 commits = 150 commits > 50 floor, 15 files > 10 floor.
+ git_meta = {
+ f.file_info.path: {
+ "commit_count_90d": 10,
+ "is_hotspot": i < 3,
+ "primary_owner_name": "alice",
+ "age_days": 30,
+ }
+ for i, f in enumerate(files[:15])
+ }
+ sig = _signals(files=files, git_meta=git_meta)
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ assert ctx.total_commits_90d == 150
+ assert ctx.files_touched_90d == 15
+ assert len(ctx.hot_files) <= 12
+ # Top hot file should be one of the hotspots.
+ assert ctx.hot_files[0].is_hotspot
+
+
+# ---------------------------------------------------------------------------
+# Getting started — gated on manifest OR readme section
+# ---------------------------------------------------------------------------
+
+
+def test_getting_started_skipped_without_signal() -> None:
+ spec = onboarding.get_spec(SLOT_GETTING_STARTED)
+ assert spec is not None
+ sig = _signals(files=[_file("src/a.py")])
+ assert spec.build_context(sig) is None
+
+
+def test_getting_started_fires_on_manifest() -> None:
+ spec = onboarding.get_spec(SLOT_GETTING_STARTED)
+ assert spec is not None
+ sig = _signals(
+ files=[_file("src/a.py")],
+ external_systems=(
+ {"name": "fastapi", "ecosystem": "pypi", "category": "framework"},
+ {"name": "pytest", "ecosystem": "pypi", "is_dev": True},
+ ),
+ )
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ assert "pypi" in ctx.package_managers
+ assert any(d["name"] == "fastapi" for d in ctx.runtime_dependencies)
+ assert any(d["name"] == "pytest" for d in ctx.dev_dependencies)
+
+
+def test_getting_started_fires_on_readme_install_section() -> None:
+ spec = onboarding.get_spec(SLOT_GETTING_STARTED)
+ assert spec is not None
+ readme = b"""# Cool Project
+
+## Installation
+
+Run `pip install -e .`.
+
+## Running
+
+`python -m cool`.
+"""
+ sig = _signals(files=[_file("src/a.py")], source_map={"README.md": readme})
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ headings = {s.heading for s in ctx.readme_sections}
+ assert "Install" in headings
+
+
+# ---------------------------------------------------------------------------
+# Key concepts — gated on ≥4 high-PageRank public symbols
+# ---------------------------------------------------------------------------
+
+
+def test_key_concepts_skipped_when_too_few_concepts() -> None:
+ spec = onboarding.get_spec(SLOT_KEY_CONCEPTS)
+ assert spec is not None
+ sig = _signals(files=[_file("src/a.py", symbols=["A"])])
+ assert spec.build_context(sig) is None
+
+
+def test_key_concepts_fires_with_enough_concepts() -> None:
+ spec = onboarding.get_spec(SLOT_KEY_CONCEPTS)
+ assert spec is not None
+ files = [
+ _file("src/handler.py", symbols=["Handler"]),
+ _file("src/router.py", symbols=["Router"]),
+ _file("src/middleware.py", symbols=["Middleware"]),
+ _file("src/session.py", symbols=["Session"]),
+ _file("src/util.py", symbols=["Util"]),
+ ]
+ # Boost the first four files into the top decile.
+ pagerank = {
+ "src/handler.py": 0.9,
+ "src/router.py": 0.8,
+ "src/middleware.py": 0.7,
+ "src/session.py": 0.6,
+ "src/util.py": 0.05,
+ }
+ sig = _signals(files=files, pagerank=pagerank)
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ assert len(ctx.concept_symbols) >= 4
+ names = {c.name for c in ctx.concept_symbols}
+ assert {"Handler", "Router", "Middleware", "Session"}.issubset(names)
+
+
+# ---------------------------------------------------------------------------
+# How it works — gated on flow OR archetype
+# ---------------------------------------------------------------------------
+
+
+def test_how_it_works_skipped_for_flat_module_collection() -> None:
+ spec = onboarding.get_spec(SLOT_HOW_IT_WORKS)
+ assert spec is not None
+ sig = _signals(files=[_file("src/util.py"), _file("src/helper.py")])
+ assert spec.build_context(sig) is None
+
+
+def test_how_it_works_fires_on_service_archetype() -> None:
+ spec = onboarding.get_spec(SLOT_HOW_IT_WORKS)
+ assert spec is not None
+ sig = _signals(
+ files=[_file("src/main.py", is_entry_point=True)],
+ external_systems=({"name": "fastapi", "ecosystem": "pypi"},),
+ )
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ assert ctx.archetype == "service"
+
+
+def test_how_it_works_fires_on_cli_archetype_via_entry_point() -> None:
+ spec = onboarding.get_spec(SLOT_HOW_IT_WORKS)
+ assert spec is not None
+ sig = _signals(
+ files=[_file("src/cli/__main__.py", is_entry_point=True)],
+ entry_points=["src/cli/__main__.py"],
+ )
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ assert ctx.archetype == "cli"
+
+
+# ---------------------------------------------------------------------------
+# Development guide — gated on ≥2 structural signals
+# ---------------------------------------------------------------------------
+
+
+def test_development_guide_skipped_for_unconventional_repo() -> None:
+ spec = onboarding.get_spec(SLOT_DEVELOPMENT_GUIDE)
+ assert spec is not None
+ sig = _signals(files=[_file("src/a.py"), _file("src/b.py")])
+ assert spec.build_context(sig) is None
+
+
+def test_development_guide_fires_with_suffix_pattern_and_test_mirror() -> None:
+ spec = onboarding.get_spec(SLOT_DEVELOPMENT_GUIDE)
+ assert spec is not None
+ files = [
+ _file("src/auth_handler.py", symbols=["AuthHandler"]),
+ _file("src/user_handler.py", symbols=["UserHandler"]),
+ _file("src/billing_handler.py", symbols=["BillingHandler"]),
+ _file("src/order_handler.py", symbols=["OrderHandler"]),
+ _file("tests/test_auth_handler.py"),
+ _file("tests/test_user_handler.py"),
+ _file("tests/test_billing_handler.py"),
+ ]
+ sig = _signals(files=files)
+ ctx = spec.build_context(sig)
+ assert ctx is not None
+ suffixes = {p.suffix for p in ctx.suffix_patterns}
+ assert "handler" in suffixes
+ assert ctx.test_mirror is not None
+ assert ctx.test_mirror.test_root == "tests"
+
+
+# ---------------------------------------------------------------------------
+# Promoted-page tagging
+# ---------------------------------------------------------------------------
+
+
+def _make_page(page_type: str, target: str) -> GeneratedPage:
+ return GeneratedPage(
+ page_id=f"{page_type}:{target}",
+ page_type=page_type,
+ title=f"{page_type}: {target}",
+ content="...",
+ source_hash="hash",
+ model_name="mock",
+ provider_name="mock",
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ generation_level=GENERATION_LEVELS.get(page_type, 0),
+ target_path=target,
+ created_at="2026-01-01T00:00:00+00:00",
+ updated_at="2026-01-01T00:00:00+00:00",
+ )
+
+
+def test_tag_promoted_pages_sets_onboarding_slot() -> None:
+ pages = [
+ _make_page("repo_overview", "testrepo"),
+ _make_page("architecture_diagram", "testrepo"),
+ _make_page("file_page", "src/a.py"),
+ ]
+ PageGenerator._tag_promoted_pages(pages)
+ assert pages[0].metadata["onboarding_slot"] == "project_overview"
+ assert pages[1].metadata["onboarding_slot"] == "architecture_guide"
+ # Non-promoted pages stay untouched.
+ assert "onboarding_slot" not in pages[2].metadata
+
+
+# ---------------------------------------------------------------------------
+# Templates render
+# ---------------------------------------------------------------------------
+
+
+def _jinja_env() -> jinja2.Environment:
+ templates_dir = (
+ Path(__file__).resolve().parents[3]
+ / "packages"
+ / "core"
+ / "src"
+ / "repowise"
+ / "core"
+ / "generation"
+ / "templates"
+ )
+ return jinja2.Environment(
+ loader=jinja2.FileSystemLoader(str(templates_dir)),
+ undefined=jinja2.StrictUndefined,
+ autoescape=False,
+ )
+
+
+ "subkind_slot,ctx_factory",
+ [
+ (
+ SLOT_CODEBASE_MAP,
+ lambda: onboarding.get_spec(SLOT_CODEBASE_MAP).build_context(
+ _signals(files=[_file(f"src/m{i}.py", symbols=[f"M{i}"]) for i in range(6)])
+ ),
+ ),
+ (
+ SLOT_ACTIVE_LANDSCAPE,
+ lambda: onboarding.get_spec(SLOT_ACTIVE_LANDSCAPE).build_context(
+ _signals(
+ files=[_file(f"src/f{i}.py") for i in range(20)],
+ git_meta={
+ f"src/f{i}.py": {
+ "commit_count_90d": 10,
+ "is_hotspot": i < 3,
+ "primary_owner_name": "alice",
+ "age_days": 30,
+ }
+ for i in range(15)
+ },
+ )
+ ),
+ ),
+ (
+ SLOT_GETTING_STARTED,
+ lambda: onboarding.get_spec(SLOT_GETTING_STARTED).build_context(
+ _signals(
+ files=[_file("src/a.py")],
+ external_systems=(
+ {"name": "fastapi", "ecosystem": "pypi", "category": "framework"},
+ ),
+ )
+ ),
+ ),
+ (
+ SLOT_KEY_CONCEPTS,
+ lambda: onboarding.get_spec(SLOT_KEY_CONCEPTS).build_context(
+ _signals(
+ files=[
+ _file("src/handler.py", symbols=["Handler"]),
+ _file("src/router.py", symbols=["Router"]),
+ _file("src/middleware.py", symbols=["Middleware"]),
+ _file("src/session.py", symbols=["Session"]),
+ ],
+ pagerank={
+ "src/handler.py": 0.9,
+ "src/router.py": 0.8,
+ "src/middleware.py": 0.7,
+ "src/session.py": 0.6,
+ },
+ )
+ ),
+ ),
+ (
+ SLOT_HOW_IT_WORKS,
+ lambda: onboarding.get_spec(SLOT_HOW_IT_WORKS).build_context(
+ _signals(
+ files=[_file("src/main.py", is_entry_point=True)],
+ external_systems=({"name": "fastapi", "ecosystem": "pypi"},),
+ )
+ ),
+ ),
+ (
+ SLOT_DEVELOPMENT_GUIDE,
+ lambda: onboarding.get_spec(SLOT_DEVELOPMENT_GUIDE).build_context(
+ _signals(
+ files=[
+ _file("src/auth_handler.py"),
+ _file("src/user_handler.py"),
+ _file("src/billing_handler.py"),
+ _file("src/order_handler.py"),
+ _file("tests/test_auth_handler.py"),
+ _file("tests/test_user_handler.py"),
+ _file("tests/test_billing_handler.py"),
+ ]
+ )
+ ),
+ ),
+ ],
+)
+def test_subkind_template_renders(subkind_slot: str, ctx_factory: Any) -> None:
+ spec = onboarding.get_spec(subkind_slot)
+ assert spec is not None
+ ctx = ctx_factory()
+ assert ctx is not None, f"gate failed for {subkind_slot}"
+ env = _jinja_env()
+ template = env.get_template(f"onboarding/{spec.template}")
+ rendered = template.render(ctx=ctx, slot=subkind_slot)
+ assert rendered.strip() # non-empty
+ assert "{{" not in rendered # no unrendered Jinja
diff --git a/tests/unit/generation/test_page_generator.py b/tests/unit/generation/test_page_generator.py
index 8df5f2cb7..6c3cdcaeb 100644
--- a/tests/unit/generation/test_page_generator.py
+++ b/tests/unit/generation/test_page_generator.py
@@ -28,7 +28,6 @@ EXPECTED_PAGE_TYPES = [
"architecture_diagram",
"api_contract",
"infra_page",
- "diff_summary",
]
@@ -397,7 +396,7 @@ async def test_generate_all_level_values_in_range():
[p], {"pkg/main.py": b"def main(): pass"}, builder, repo, "test-repo"
)
for page in pages:
- assert 0 <= page.generation_level <= 7
+ assert 0 <= page.generation_level <= 8
# ---------------------------------------------------------------------------
diff --git a/tests/unit/generation/test_selection_budget.py b/tests/unit/generation/test_selection_budget.py
new file mode 100644
index 000000000..752c50f4d
--- /dev/null
+++ b/tests/unit/generation/test_selection_budget.py
@@ -0,0 +1,235 @@
+"""Budget enforcement regression tests for the selection layer.
+
+These tests directly exercise :func:`select_pages` against synthetic
+parsed-file fixtures and assert that the global page count tracks the
+configured ``coverage_pct``. They are the contract that prevents the
+``529-pages-when-95-was-budgeted`` regression from coming back.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from unittest.mock import MagicMock
+
+import pytest
+
+from repowise.core.generation.models import GenerationConfig
+from repowise.core.generation.selection import (
+ SelectionInputs,
+ select_pages,
+)
+from repowise.core.generation.selection.budget import (
+ SMALL_REPO_THRESHOLD,
+ allocate_budget,
+ compute_budget,
+)
+
+
+# ---------------------------------------------------------------------------
+# Lightweight ParsedFile / Symbol stand-ins
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class FakeFileInfo:
+ path: str
+ language: str = "python"
+ abs_path: str = ""
+ size_bytes: int = 5_000
+ is_test: bool = False
+ is_config: bool = False
+ is_api_contract: bool = False
+ is_entry_point: bool = False
+ git_hash: str = ""
+
+ def __post_init__(self) -> None:
+ if not self.abs_path:
+ self.abs_path = f"/repo/{self.path}"
+
+
+@dataclass
+class FakeSymbol:
+ name: str
+ qualified_name: str = ""
+ kind: str = "function"
+ visibility: str = "public"
+ signature: str = "()"
+ docstring: str | None = None
+ decorators: list[str] = field(default_factory=list)
+ is_async: bool = False
+ complexity_estimate: int = 1
+ parent_name: str | None = None
+
+ def __post_init__(self) -> None:
+ if not self.qualified_name:
+ self.qualified_name = self.name
+
+
+@dataclass
+class FakeParsedFile:
+ file_info: FakeFileInfo
+ symbols: list[FakeSymbol] = field(default_factory=list)
+ imports: list[object] = field(default_factory=list)
+ exports: list[str] = field(default_factory=list)
+ docstring: str | None = None
+ parse_errors: list[str] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Fixture builder
+# ---------------------------------------------------------------------------
+
+
+def _build_synthetic_repo(n_files: int) -> tuple[list[FakeParsedFile], dict, dict, dict]:
+ """Return ``(parsed_files, pagerank, betweenness, community)``.
+
+ PageRank tapers linearly so the top file has 1.0 and the bottom
+ file has ~0.01. Community assignment buckets every 25 files into
+ one community.
+ """
+ parsed: list[FakeParsedFile] = []
+ pagerank: dict[str, float] = {}
+ betweenness: dict[str, float] = {}
+ community: dict[str, int] = {}
+
+ for i in range(n_files):
+ path = f"pkg{i // 25}/module_{i}.py"
+ fi = FakeFileInfo(path=path)
+ syms = [
+ FakeSymbol(name=f"func_{i}_{k}", qualified_name=f"module_{i}.func_{i}_{k}")
+ for k in range(3)
+ ]
+ parsed.append(FakeParsedFile(file_info=fi, symbols=syms))
+ pagerank[path] = 1.0 - (i / max(1, n_files - 1)) * 0.99
+ betweenness[path] = 0.0
+ community[path] = i // 25
+ return parsed, pagerank, betweenness, community
+
+
+# ---------------------------------------------------------------------------
+# Budget computation
+# ---------------------------------------------------------------------------
+
+
+def test_compute_budget_scales_linearly_for_large_repo():
+ assert compute_budget(1_000, 0.10) == 100
+ assert compute_budget(1_000, 0.20) == 200
+ assert compute_budget(1_000, 0.50) == 500
+ assert compute_budget(10_000, 0.20) == 2_000 # no absolute cap
+
+
+def test_compute_budget_floors_for_small_repos():
+ # Below the small-repo threshold, budget never undershoots N_files
+ # so percentage rounding doesn't zero out tiny repos.
+ assert compute_budget(5, 0.20) == 5
+ assert compute_budget(SMALL_REPO_THRESHOLD, 0.20) == SMALL_REPO_THRESHOLD
+
+
+def test_compute_budget_zero_for_empty_repo():
+ assert compute_budget(0, 0.50) == 0
+
+
+def test_allocate_budget_respects_candidate_supply():
+ """A bucket is never asked for more than its available candidates."""
+ alloc = allocate_budget(
+ budget=100,
+ candidates_per_bucket={
+ "file_page": 500,
+ "symbol_spotlight": 10, # capped low
+ "module_page": 20,
+ "api_contract": 0,
+ "infra_page": 0,
+ "scc_page": 2,
+ },
+ shares={
+ "file_page": 0.50,
+ "symbol_spotlight": 0.30,
+ "module_page": 0.10,
+ "api_contract": 0.05,
+ "infra_page": 0.03,
+ "scc_page": 0.02,
+ },
+ n_files=500,
+ )
+ # Capped at supply
+ assert alloc.symbol_spotlight == 10
+ assert alloc.api_contract == 0
+ assert alloc.infra_page == 0
+ # Spill into file_page absorbs the missing share
+ assert alloc.file_page >= 50
+
+
+# ---------------------------------------------------------------------------
+# Full select_pages — integration-shaped
+# ---------------------------------------------------------------------------
+
+
+ "coverage_pct,expected_total",
+ [
+ (0.10, 96),
+ (0.20, 192),
+ (0.50, 479),
+ ],
+)
+def test_select_pages_budget_tracks_coverage(coverage_pct, expected_total):
+ """Total page count tracks coverage_pct within ±10%."""
+ parsed, pagerank, betweenness, community = _build_synthetic_repo(958)
+ cfg = GenerationConfig(coverage_pct=coverage_pct)
+ inputs = SelectionInputs(
+ parsed_files=parsed,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=cfg,
+ )
+
+ selection = select_pages(inputs)
+ total = sum(selection.counts().values())
+
+ tolerance = max(20, int(expected_total * 0.20))
+ assert abs(total - expected_total) <= tolerance, (
+ f"coverage={coverage_pct}: expected ~{expected_total}, got {total}"
+ )
+
+
+def test_select_pages_emits_no_pages_for_empty_repo():
+ cfg = GenerationConfig(coverage_pct=0.20)
+ inputs = SelectionInputs(
+ parsed_files=[],
+ pagerank={},
+ betweenness={},
+ community={},
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=cfg,
+ )
+ selection = select_pages(inputs)
+ assert selection.allocation is not None
+ assert selection.allocation.total == 0
+
+
+def test_higher_coverage_strictly_emits_more_pages():
+ """Bumping coverage_pct strictly increases the total page count."""
+ parsed, pagerank, betweenness, community = _build_synthetic_repo(500)
+ totals: list[int] = []
+ for pct in (0.10, 0.20, 0.30, 0.50):
+ cfg = GenerationConfig(coverage_pct=pct)
+ sel = select_pages(
+ SelectionInputs(
+ parsed_files=parsed,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=cfg,
+ )
+ )
+ totals.append(sum(sel.counts().values()))
+ assert totals == sorted(totals), f"Non-monotonic budget: {totals}"
diff --git a/tests/unit/generation/test_templates.py b/tests/unit/generation/test_templates.py
index 9c0d34f06..6916b0b87 100644
--- a/tests/unit/generation/test_templates.py
+++ b/tests/unit/generation/test_templates.py
@@ -10,7 +10,6 @@ import pytest
from repowise.core.generation.context_assembler import (
ApiContractContext,
ArchitectureDiagramContext,
- DiffSummaryContext,
FilePageContext,
InfraPageContext,
ModulePageContext,
@@ -342,35 +341,3 @@ def test_scc_page_contains_cycle_description(jinja_env, scc_page_ctx):
assert scc_page_ctx.cycle_description in result
-# ---------------------------------------------------------------------------
-# diff_summary.j2
-# ---------------------------------------------------------------------------
-
-
[email protected](scope="module")
-def diff_summary_ctx() -> DiffSummaryContext:
- return DiffSummaryContext(
- from_ref="HEAD~1",
- to_ref="HEAD",
- added_files=["new_file.py"],
- deleted_files=[],
- modified_files=["changed.py"],
- symbol_diffs=[],
- affected_page_ids=["file_page:changed.py"],
- )
-
-
-def test_diff_summary_renders_without_error(jinja_env, diff_summary_ctx):
- result = render(jinja_env, "diff_summary.j2", diff_summary_ctx)
- assert result
-
-
-def test_diff_summary_has_heading(jinja_env, diff_summary_ctx):
- result = render(jinja_env, "diff_summary.j2", diff_summary_ctx)
- assert "##" in result
-
-
-def test_diff_summary_contains_refs(jinja_env, diff_summary_ctx):
- result = render(jinja_env, "diff_summary.j2", diff_summary_ctx)
- assert diff_summary_ctx.from_ref in result
- assert diff_summary_ctx.to_ref in result
diff --git a/tests/unit/ingestion/external_systems/__init__.py b/tests/unit/ingestion/external_systems/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/server/services/__init__.py b/tests/unit/server/services/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/test_providers/test_cost_estimator.py b/tests/unit/test_providers/test_cost_estimator.py
index d5ddd9501..56c7f2fda 100644
--- a/tests/unit/test_providers/test_cost_estimator.py
+++ b/tests/unit/test_providers/test_cost_estimator.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import pytest
from repowise.cli.cost_estimator import PageTypePlan, _lookup_cost, estimate_cost
+from repowise.cli.cost_estimator.heuristics import heuristic_tokens
# ---------------------------------------------------------------------------
# Per-model pricing (input_rate, output_rate) per 1K tokens
@@ -43,25 +44,27 @@ def test_lookup_cost(model, expected_input, expected_output):
def test_estimate_cost_gpt54_nano():
- plans = [PageTypePlan("repo_overview", 1, 6)] # 5000 input, 3000 output tokens
+ plans = [PageTypePlan("repo_overview", 1, 6)]
est = estimate_cost(plans, "openai", "gpt-5.4-nano")
- # 5000 tokens * $0.0002/1K + 3000 tokens * $0.00125/1K
- expected = (5000 / 1000) * 0.0002 + (3000 / 1000) * 0.00125
+ inp, out = heuristic_tokens("repo_overview")
+ expected = (inp / 1000) * 0.0002 + (out / 1000) * 0.00125
assert est.estimated_cost_usd == pytest.approx(expected, rel=1e-6)
assert est.model_name == "gpt-5.4-nano"
def test_estimate_cost_claude_opus():
- plans = [PageTypePlan("repo_overview", 1, 6)] # 5000 input, 3000 output
+ plans = [PageTypePlan("repo_overview", 1, 6)]
est = estimate_cost(plans, "anthropic", "claude-opus-4-6")
- expected = (5000 / 1000) * 0.005 + (3000 / 1000) * 0.025
+ inp, out = heuristic_tokens("repo_overview")
+ expected = (inp / 1000) * 0.005 + (out / 1000) * 0.025
assert est.estimated_cost_usd == pytest.approx(expected, rel=1e-6)
def test_estimate_cost_gemini_lite():
- plans = [PageTypePlan("file_page", 10, 2)] # 4000 input, 2500 output each
+ plans = [PageTypePlan("file_page", 10, 2)]
est = estimate_cost(plans, "gemini", "gemini-3.1-flash-lite-preview")
- expected = (40000 / 1000) * 0.00025 + (25000 / 1000) * 0.0015
+ inp, out = heuristic_tokens("file_page")
+ expected = (inp * 10 / 1000) * 0.00025 + (out * 10 / 1000) * 0.0015
assert est.estimated_cost_usd == pytest.approx(expected, rel=1e-6)
diff --git a/tests/unit/test_providers/test_openai_provider.py b/tests/unit/test_providers/test_openai_provider.py
index 3cf2b49a0..d9da1b8f0 100644
--- a/tests/unit/test_providers/test_openai_provider.py
+++ b/tests/unit/test_providers/test_openai_provider.py
@@ -61,6 +61,7 @@ def _make_mock_chat_response(text: str = "# Doc\nContent.") -> MagicMock:
usage.prompt_tokens = 120
usage.completion_tokens = 60
usage.total_tokens = 180
+ usage.prompt_tokens_details = MagicMock(cached_tokens=0)
choice = MagicMock()
choice.message.content = text