Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(wiki): make the repo overview read like a product, not a directory (#2078)
Raghav Chamadiya0a197b407934
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 416ebfaf3..a736e2693 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -27,8 +27,10 @@ import refrom collections.abc import Callablefrom dataclasses import dataclassfrom pathlib import Path+from typing import Anyfrom repowise.core import fs_walk+from repowise.core.exclusion import build_exclude_spec, is_excludedfrom .grouping import ConceptGroup@@ -235,13 +237,51 @@ class HouseTerm:is_indexed_symbol: bool+def _rel(path: Path, repo_root: Path) -> str:+ """Repo-relative posix path, or the bare name when it is not under root."""+ try:+ return path.relative_to(repo_root).as_posix()+ except ValueError: # pragma: no cover — path came from repo_root+ return path.name+++def _exclude_spec(repo_root: Path) -> Any:+ """The repository's own exclusion rules, or ``None``. Never raises.++ A malformed ``.gitignore`` line makes ``pathspec`` raise, and mining is a+ decoration: a repository with an unparseable ignore file should mine one+ document too many, not fail to generate.+ """+ try:+ return build_exclude_spec(repo_root)+ except Exception as exc:+ logger.warning("vocabulary: could not read exclusion rules (%s); mining unfiltered", exc)+ return None+++def _is_minable(path: Path, repo_root: Path, spec: Any) -> bool:+ """Whether a document may be quoted as where a term is written.++ Two rules, both learned the same way. The repository's own exclusion rules+ are honoured because a path the user told git to ignore is scratch work,+ and the capability table cited ``local-stash/`` harnesses as authoritative+ definitions. Market-facing documents are dropped by name because their+ headings look exactly like subsystem names and nothing downstream can tell+ the difference.+ """+ return not is_excluded(_rel(path, repo_root), spec) and not _NON_NORMATIVE_DOC_NAMES.search(+ path.name+ )++def _doc_paths(repo_root: Path, *, patterns: tuple[str, ...] = ("*.md",)) -> list[Path]:"""The documents worth mining, in a fixed order. Never raises."""files: list[Path] = []+ spec = _exclude_spec(repo_root)try:for name in DOC_FILES:candidate = repo_root / name- if candidate.is_file():+ if candidate.is_file() and _is_minable(candidate, repo_root, spec):files.append(candidate)for rel_dir in DOC_DIRS:directory = repo_root / rel_dir@@ -251,7 +291,11 @@ def _doc_paths(repo_root: Path, *, patterns: tuple[str, ...] = ("*.md",)) -> lisfor pattern in patterns:found.extend(directory.glob(pattern))for candidate in sorted(found):- if candidate.is_file() and candidate not in files:+ if (+ candidate.is_file()+ and candidate not in files+ and _is_minable(candidate, repo_root, spec)+ ):files.append(candidate)if len(files) >= _MAX_DOCS:break@@ -504,6 +548,13 @@ def _join_markdown_wrapped(lines: list[str], start: int) -> str:#: repository and the least useful one to mine: on this repository it consumed#: 142 of the first 200 term slots before the tool guide was ever opened._RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst|txt)$", re.I)+#: Documents about the market rather than about the system. Their headings are+#: product claims, competitor names and score tables, so mining them cites a+#: positioning document as "where this capability is written". Dropped by name+#: because their headings are indistinguishable from a real subsystem's.+_NON_NORMATIVE_DOC_NAMES = re.compile(+ r"(benchmarks?|competitive[_ -]?analysis|competitors?|pricing|roadmap)\.(md|rst|txt)$", re.I+)_VERSION_HEADING = re.compile(r"^v?\d+\.\d+")#: A document has to be overwhelmingly version headings before it is dropped. A#: guide that cites a few release numbers is not release notes, and dropping a@@ -643,10 +694,7 @@ def _harvest(if skip_release_notes and _is_release_notes(path, text):skipped.append(path.name)continue- try:- rel = path.relative_to(repo_root).as_posix()- except ValueError: # pragma: no cover — path came from repo_root- rel = path.name+ rel = _rel(path, repo_root)# Definitions found for terms seen in *this* document, keyed the same# way as candidates so a repeat mention can fill a gap left earlier.@@ -788,6 +836,7 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:dir_names: set[str] = set()unreadable = 0truncated = False+ spec = _exclude_spec(repo_root)for dirpath, dirnames, filenames in fs_walk.walk_repo(repo_root, prune_dirs=_EXTRA_PRUNED_DIRS):# Hidden directories are tool and agent territory — CI definitions,@@ -799,7 +848,16 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:# ``.claude/`` ahead of the original, and the mined definitions cited# paths a reader has no reason to open. Pruning the whole class costs# one string test and no repository keeps its documented source here.- dirnames[:] = [d for d in dirnames if not d.startswith(".")]+ # A tree the repository excludes from its own index is scratch work,+ # not the repository talking about itself. Skipped here rather than+ # filtered later so a large stash costs no walk: the capability table+ # was citing ``local-stash/`` harnesses as where a capability is+ # written, and those files are absent from every other surface.+ dirnames[:] = [+ d+ for d in dirnames+ if not d.startswith(".") and not is_excluded(_rel(dirpath / d, repo_root) + "/", spec)+ ]for name in dirnames:dir_names.add(_singular(name.lower()))if truncated:@@ -820,6 +878,9 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:truncated = Truebreakpath = dirpath / name+ rel = _rel(path, repo_root)+ if is_excluded(rel, spec):+ continuetry:text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_SOURCE_BYTES]except OSError:@@ -829,10 +890,6 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:if not blocks:continuejoined = "\n".join(_JSDOC_GUTTER.sub("", b) for b in blocks)- try:- rel = path.relative_to(repo_root).as_posix()- except ValueError: # pragma: no cover — path came from repo_root- rel = path.nameprose.append((rel, joined))if unreadable:diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.pyindex 2bc2392a3..92a1a9a5d 100644--- a/packages/core/src/repowise/core/generation/context/assembler.py+++ b/packages/core/src/repowise/core/generation/context/assembler.py@@ -762,6 +762,7 @@ class ContextAssembler:external_systems: list[dict] | None = None,decision_records: list[dict] | None = None,parsed_files: list[ParsedFile] | None = None,+ prose_digest: str = "",) -> RepoOverviewContext:"""Assemble context for the repo_overview template."""# Top files sorted by PageRank descending, path breaking ties. Leaf@@ -895,6 +896,7 @@ class ContextAssembler:external_systems=external_systems or [],decision_records=decision_records or [],package_stats=package_stats,+ prose_digest=prose_digest,)# ------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.pyindex 3930cbddb..fed4bc10f 100644--- a/packages/core/src/repowise/core/generation/context/contexts.py+++ b/packages/core/src/repowise/core/generation/context/contexts.py@@ -182,6 +182,10 @@ class RepoOverviewContext:# Phase 2: third-party dependencies + headline architectural decisionsexternal_systems: list[dict] = field(default_factory=list)decision_records: list[dict] = field(default_factory=list)+ # The repository's own headings and section openers, capped. Framing and+ # vocabulary only: every path, count and package name on the page still+ # comes from the structural fields above. See ``readme_digest``.+ prose_digest: str = ""# Per-package file counts and observed languages, largest first. Counted# from the run's own parsed files rather than written by the model, so the# table they feed reads the same on every render. Empty when the repositorydiff --git a/packages/core/src/repowise/core/generation/context/readme_digest.py b/packages/core/src/repowise/core/generation/context/readme_digest.pynew file mode 100644index 000000000..f53a2c946--- /dev/null+++ b/packages/core/src/repowise/core/generation/context/readme_digest.py@@ -0,0 +1,146 @@+"""What the repository says it is, in its own sentences, for the front page.++The repo overview's payload is otherwise entirely structural: pagerank,+communities, package file counts, entry-point paths. A model handed only that+can describe a directory tree accurately and never say what the product is+for, because nothing it was given says so. The words are not missing from the+repository — the intelligence layers, the store architecture, the task-oriented+tools are all under headings in its own README. No generation prompt had ever+read it.++**The measured reason this is a keyhole and not a pipe.**+``concept_tree/vocabulary.py`` records that nine thousand characters of README+handed to the outline *planner* collapsed coverage from 23 pages / 87.5% to 3+pages / 5.4%, and invented a docs path matching the marketing language rather+than the tree. That is a **selection** task: prose outvoted structure at+choosing what exists. The overview is a **writing** task that selects nothing,+so the finding transfers as a bound rather than a veto. Two properties keep it+one:++* **Capped well under the collapse.** ``_MAX_CHARS`` is 2,500 against the 9,000+ that was measured to collapse a planner. The cap is the guardrail, not a+ token budget, which is why it is a constant here and not a config value.+* **Headings and openers only.** A heading names a thing and its first+ paragraph says what the thing is; the rest of a section is instructions,+ tables and examples. Reading only the openers is what keeps 60KB of README+ inside the cap without choosing which 2,500 characters by relevance, which+ would be a selection decision made on prose.++Structure stays the sole authority on paths, counts and package names. This+supplies vocabulary and framing; the template says so in as many words.+"""++from __future__ import annotations++import re+from pathlib import Path++#: Well under the 9,000 characters measured to collapse the planner, and+#: roughly the size of the structural payload it sits beside, so neither one+#: dominates the prompt.+_MAX_CHARS = 2500++#: Root-anchored, and only these two. A ``docs/*.md`` glob matches a path tail,+#: which is how vendored and scratch documents reached the glossary once+#: already; see ``concept_tree/vocabulary.py``.+_DIGEST_DOCS = ("README.md", "docs/README.md")++#: Headings down to level three. Deeper ones are steps and options inside a+#: section, not names of things.+_HEADING = re.compile(r"^\s{0,3}(#{1,3})\s+(.+?)\s*#*\s*$")++#: A line that is only badges, an image, raw HTML, or a link-only banner. These+#: open most READMEs and say nothing, and left in they would spend the cap+#: before the first sentence.+_DECORATION = re.compile(r"^\s*(?:!\[|<[a-zA-Z/!]|\[!\[|\||-{3,}|={3,}|\*{3,})")++#: Inline images and the link syntax around their targets. The link *text* is+#: prose and stays; the URL is not.+_IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)")+_LINK = re.compile(r"\[([^\]]+)\]\([^)]*\)")+#: Named because a backslash cannot appear inside an f-string before 3.12.+_LINK_TEXT = r"\1"+++def _opening_paragraph(lines: list[str], start: int) -> str:+ """The first prose paragraph at or after *start*, before the next heading.++ Skips fenced code, decoration and blank lines. Returns "" when the section+ opens straight onto another heading, which is the common shape of a table+ of contents.+ """+ i = start+ fenced = False+ body: list[str] = []+ while i < len(lines):+ line = lines[i]+ stripped = line.strip()+ # Fence state is read before anything else, because a `#` comment on+ # the first line of a shell block is not a heading and a `---` inside+ # a YAML example is not a rule.+ if stripped.startswith("```"):+ fenced = not fenced+ i += 1+ continue+ if fenced:+ i += 1+ continue+ if _HEADING.match(line):+ break+ if not stripped or _DECORATION.match(line):+ # Blank lines before the paragraph are skipped; one after it ends+ # the paragraph, so that a section contributes one paragraph.+ if body:+ break+ i += 1+ continue+ body.append(stripped)+ i += 1+ text = " ".join(body)+ text = _IMAGE.sub("", text)+ return _LINK.sub(_LINK_TEXT, text).strip()+++def readme_digest(repo_root: Path, *, max_chars: int = _MAX_CHARS) -> str:+ """The repository's headings and section openers, capped. Never raises.++ Returns markdown: each heading kept at its own level with its opening+ paragraph beneath it. Empty when there is no README, so the caller can+ drop the whole block rather than label an absence.+ """+ out: list[str] = []+ budget = max_chars+ for rel in _DIGEST_DOCS:+ path = repo_root / rel+ try:+ text = path.read_text(encoding="utf-8", errors="replace")+ except OSError:+ continue+ lines = text.splitlines()+ fenced = False+ for i, line in enumerate(lines):+ # A `# clone the repo` inside a bash block is a comment, and most+ # READMEs open with one. Read as a heading it becomes a name the+ # project supposedly gives one of its own parts.+ if line.strip().startswith("```"):+ fenced = not fenced+ continue+ if fenced:+ continue+ match = _HEADING.match(line)+ if not match:+ continue+ hashes, title = match.groups()+ entry = f"{hashes} {_LINK.sub(_LINK_TEXT, title).strip()}"+ paragraph = _opening_paragraph(lines, i + 1)+ if paragraph:+ entry = f"{entry}\n{paragraph}"+ cost = len(entry) + 2+ if cost > budget:+ # A cut mid-document, not a ranked selection: the cap has to+ # bind on prose we did not choose, or the choosing becomes the+ # thing the planner's collapse warned about.+ return "\n\n".join(out)+ out.append(entry)+ budget -= cost+ return "\n\n".join(out)diff --git a/packages/core/src/repowise/core/generation/overview_tables.py b/packages/core/src/repowise/core/generation/overview_tables.pyindex 5d8686a7d..af90fa880 100644--- a/packages/core/src/repowise/core/generation/overview_tables.py+++ b/packages/core/src/repowise/core/generation/overview_tables.py@@ -33,8 +33,14 @@ CAPABILITY_TABLE_HEADING = "## What it does"# The heading plus everything up to the next heading of the same or higher# level. Anchored at a line start so a mention inside prose is not a match.+#+# The provenance footer carries no heading, only a horizontal rule, so it is a+# terminator too. Without it a section that is last on the page runs to the end+# and the replacement eats the footer — which is what happened the moment the+# deterministic overview stopped rendering a path table below its packages.+_SECTION_END = r"(?=^#{1,2}[ \t]|^---[ \t]*$|\Z)"_PACKAGE_SECTION_RE = re.compile(- r"^##[ \t]+Packages[ \t]*\n(?:.*?)(?=^#{1,2}[ \t]|\Z)",+ r"^##[ \t]+Packages[ \t]*\n(?:.*?)" + _SECTION_END,re.MULTILINE | re.DOTALL,)@@ -99,7 +105,7 @@ MAX_CAPABILITY_ROWS = 6_MAX_CAPABILITY_DEFINITION = 160_CAPABILITY_SECTION_RE = re.compile(- r"^##[ \t]+What it does[ \t]*\n(?:.*?)(?=^#{1,2}[ \t]|\Z)",+ r"^##[ \t]+What it does[ \t]*\n(?:.*?)" + _SECTION_END,re.MULTILINE | re.DOTALL,)diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.pyindex e48355d60..24c35b950 100644--- a/packages/core/src/repowise/core/generation/page_generator/levels.py+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py@@ -402,6 +402,7 @@ async def build_level6_coros(run: _GenerationRun) -> list[tuple[str, Any]]:uniquely had, so it moved here and the page retired; its id redirects."""from ..architecture_mermaid import build_overview_mermaid+ from ..context.readme_digest import readme_digestfrom ..overview_tables import select_capabilitiesgen = run.gen@@ -457,6 +458,12 @@ async def build_level6_coros(run: _GenerationRun) -> list[tuple[str, Any]]:# going unmentioned.parsed_files=run.parsed_files,capabilities=capabilities,+ # The only natural-language input the front page gets.+ # Framing and vocabulary; the structural fields above stay+ # the authority on paths, counts and package names.+ prose_digest=(+ readme_digest(Path(run.repo_path)) if run.repo_path else ""+ ),),))diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.pyindex a23e5f36f..baf894b36 100644--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py@@ -387,6 +387,7 @@ class PerTypeGenerationMixin:source_map: dict[str, bytes] | None = None,parsed_files: list[ParsedFile] | None = None,capabilities: Sequence[Capability] = (),+ prose_digest: str = "",) -> GeneratedPage:ctx = self._assembler.assemble_repo_overview(repo_structure,@@ -398,6 +399,7 @@ class PerTypeGenerationMixin:external_systems=external_systems,decision_records=decision_records,parsed_files=parsed_files,+ prose_digest=prose_digest,)repo_git_summary = Noneif git_meta_map:@@ -422,7 +424,7 @@ class PerTypeGenerationMixin:ctx, repo_name, f"Repository Overview: {repo_name}", repo_git_summary)page = _with_architecture_map(- _with_capability_table(_with_package_table(stub, ctx.package_stats), capabilities),+ _with_package_table(_with_capability_table(stub, capabilities), ctx.package_stats),overview_mermaid,)selection = self._disabled_source_evidence("repo_overview", "deterministic_generation")@@ -440,11 +442,11 @@ class PerTypeGenerationMixin:ctx, repo_name, f"Repository Overview: {repo_name}", repo_git_summary)page = _with_architecture_map(- _with_capability_table(- _with_package_table(- _stub_fallback(stub, "repo_overview", exc), ctx.package_stats+ _with_package_table(+ _with_capability_table(+ _stub_fallback(stub, "repo_overview", exc), capabilities),- capabilities,+ ctx.package_stats,),overview_mermaid,)@@ -452,19 +454,20 @@ class PerTypeGenerationMixin:# The overview carries its own enumerable facts: the package table and# the KG-derived architecture map are built from the run, not drawn by# the model, and both embeds are idempotent so a reused page picks them- # up too. The table goes in first so it lands above the diagram.- if ctx.package_stats:+ # up too. Appended in reading order, so what the repository does lands+ # above what it is made of, and both above the diagram.+ if capabilities:response = replace(response,- content=embed_package_table(- response.content, build_package_table(ctx.package_stats)+ content=embed_capability_table(+ response.content, build_capability_table(capabilities)),)- if capabilities:+ if ctx.package_stats:response = replace(response,- content=embed_capability_table(- response.content, build_capability_table(capabilities)+ content=embed_package_table(+ response.content, build_package_table(ctx.package_stats)),)if overview_mermaid:diff --git a/packages/core/src/repowise/core/generation/page_generator/prompts.py b/packages/core/src/repowise/core/generation/page_generator/prompts.pyindex 2691d05ed..7550131c6 100644--- a/packages/core/src/repowise/core/generation/page_generator/prompts.py+++ b/packages/core/src/repowise/core/generation/page_generator/prompts.py@@ -75,11 +75,14 @@ SYSTEM_PROMPTS: dict[str, str] = {"Good: 'Repowise is a codebase documentation engine: it indexes a repository ""by traversing files, parsing code into ASTs, analyzing dependencies, and ""generating LLM-synthesised wiki pages served via MCP and a web UI.' "- "Required sections: ## Project Summary, ## Technology Stack, ## Entry Points, ## Architecture. "+ "Required sections, in this order and no others: ## Project Summary, "+ "## Architecture, ## Key concepts. A list of entry-point paths or of the "+ "most-imported files is something the reader can get from ls; say what is "+ "central and why instead. ""\n""The page is a short orientation, not a manual: keep the prose within the "- "word budget the user prompt states, and reach for a table or a list "- "wherever the material is a set of facts rather than an argument."+ "word budget the user prompt states. The enumerable facts are inserted "+ "after you write, from the code as indexed, so do not tabulate them."),"architecture_diagram": ("You are repowise, an expert technical documentation generator. "diff --git a/packages/core/src/repowise/core/generation/templates/repo_overview.j2 b/packages/core/src/repowise/core/generation/templates/repo_overview.j2index 975dd1dd3..4d639549c 100644--- a/packages/core/src/repowise/core/generation/templates/repo_overview.j2+++ b/packages/core/src/repowise/core/generation/templates/repo_overview.j2@@ -1,16 +1,31 @@-You are generating a high-level overview for the repository `{{ ctx.repo_name }}`.+You are generating the front page of the wiki for `{{ ctx.repo_name }}`. It is+the first page anyone opens, so it has to answer "what is this and how is it+put together" before it answers anything else.## Repository Summary**Name:** `{{ ctx.repo_name }}`{% if ctx.is_monorepo %} (monorepo){% endif %} | **Files:** {{ ctx.total_files }} | **LOC:** {{ ctx.total_loc }}{% if ctx.circular_dependency_count > 0 %} | **Circular deps:** {{ ctx.circular_dependency_count }}{% endif %}-The first sentence of your ## Project Summary section is what every downstream-agent (RAG, semantic search, onboarding) anchors on. Make it answer "what does-this repository do, end-to-end?" using concrete pipeline vocabulary — name the-inputs the repo consumes, the transformation stages, and the artifacts produced.-Do NOT open with stats or package counts; those belong further down.+{% if ctx.prose_digest %}+## What the project says about itself-## Packages+The headings and opening sentences of this repository's own README, verbatim.+Use this for **vocabulary, framing and what the product is for**: the names the+project gives its own parts are the names this page should use.++Do **not** take a path, a count, a package name or a file listing from it. Those+arrive below, from the code as it is today, and are the only versions of those+facts that are true. A README says what someone meant to build; everything under+"Structural facts" is what is actually there.++{{ ctx.prose_digest }}+{% endif %}++## Structural facts++Derived from this run. Authoritative for every path, count and name.++### Packages{% for pkg in ctx.package_stats %}- **{{ pkg.name }}** (`{{ pkg.path }}`, {{ pkg.files }} files{% if pkg.languages %}, {{ pkg.languages | join(", ") }}{% endif %}){% else %}@@ -25,23 +40,26 @@ not repeat it and do not write a package list of your own.What that table cannot state is what each package is *for*, which is ajudgement about the code rather than a count. Give each package one clause of-role inside the Architecture section, in the vocabulary this repository uses:-what it owns and who calls it. "The engine." "The bridge to the editor." Not-"holds TypeScript files."+role in the Architecture section, in the vocabulary this repository uses: what+it owns and who calls it. "The engine." "The bridge to the editor." Not "holds+TypeScript files."{% endif %}-## Entry Points-{% for ep in ctx.entry_points %}-- `{{ ep }}`-{% endfor %}--## Top Files by PageRank-{% for item in ctx.top_files_by_pagerank %}-- `{{ item.path }}` (score: {{ "%.4f"|format(item.score) }})-{% endfor %}+{% if ctx.entry_points or ctx.top_files_by_pagerank %}+### Where the weight is+Evidence for what to call central. **Do not print any of these paths.** A table+of file paths is something the reader can get from `ls`; what they cannot get is+which parts matter and why, which is what these are for.+{% if ctx.entry_points %}+Entry points: {% for ep in ctx.entry_points[:10] %}`{{ ep }}`{% if not loop.last %}, {% endif %}{% endfor %}+{% endif %}+{% if ctx.top_files_by_pagerank %}+Most depended-upon, by PageRank over the import graph: {% for item in ctx.top_files_by_pagerank[:10] %}`{{ item.path }}`{% if not loop.last %}, {% endif %}{% endfor %}+{% endif %}+{% endif %}{% if repo_git_summary is defined and repo_git_summary %}-## Codebase health signals+### Codebase health signals- **Hotspots:** {{ repo_git_summary.hotspot_count }} files are high-churn AND high-complexity- **Stable core:** {{ repo_git_summary.stable_count }} files unchanged in 90+ days- **Most changed (90d):** {{ repo_git_summary.top_churn_files[:3] | join(', ') }}@@ -49,7 +67,7 @@ what it owns and who calls it. "The engine." "The bridge to the editor." Not{% endif %}{% if ctx.communities %}-## Architectural Communities+### Architectural CommunitiesThe codebase organizes into these architectural clusters (detected via dependency analysis):{% for c in ctx.communities %}- **{{ c.label or "Community " ~ c.id }}** — {{ c.size }} files (cohesion: {{ c.cohesion }})@@ -57,7 +75,7 @@ The codebase organizes into these architectural clusters (detected via dependenc{% endif %}{% if ctx.external_systems %}-## External Systems+### External SystemsThird-party libraries, services, and tools the repository depends on:{% for sys in ctx.external_systems[:25] %}- `{{ sys.name }}`{% if sys.category and sys.category != 'library' %} *({{ sys.category }})*{% endif %}{% if sys.ecosystem %} — {{ sys.ecosystem }}{% endif %}@@ -65,7 +83,7 @@ Third-party libraries, services, and tools the repository depends on:{% endif %}{% if ctx.decision_records %}-## Key Architectural Decisions+### Key Architectural DecisionsUse these to ground the "why" of the project — do not invent rationale beyond what is listed.{% for dr in ctx.decision_records[:8] %}- **{{ dr.title }}** — {{ dr.decision }}{% if dr.rationale %} *(Why: {{ dr.rationale }})*{% endif %}@@ -73,18 +91,41 @@ Use these to ground the "why" of the project — do not invent rationale beyond{% endif %}{% if ctx.execution_flows %}-## Primary Execution Flows+### Primary Execution FlowsKey entry points and their execution paths:{% for f in ctx.execution_flows %}- `{{ f.entry_point }}` (score: {{ f.score }}{% if f.trace_length %}, {{ f.trace_length }} steps{% endif %}){% endfor %}{% endif %}-Generate a complete wiki page following the sections above.+---++## What to write++Exactly these three sections, in this order, and nothing else. Two tables and a+diagram are inserted after you write — what this repository calls its own+capabilities, the package table, and the architecture map — so leave room for+them by not writing them yourself.++**## Project Summary** — one paragraph. What this is, what it is for, and who+uses it, in the project's own vocabulary. The first sentence is what every+downstream agent (RAG, semantic search, onboarding) anchors on, so make it+answer "what does this repository do, end-to-end?": the inputs it consumes, the+stages it puts them through, the artifacts it produces. Do not open with stats+or package counts.++**## Architecture** — how the parts fit together. One clause of role per+package, per the rule above, plus how work actually moves through them. This is+where the structural facts earn their place: name the clusters, say what the+cycles mean, say where the weight sits. Name no more than a handful of paths,+and only where the path *is* the point.++**## Key concepts** — five to seven of them, each a named idea this project+works in, one sentence each, as a bulleted list. A concept is something the+project has a word for and a reader has to learn: a layer, a store, a scoring+model, a class of artifact. Not a package, not a directory, not a file. Take the+names from the project's own words above wherever it has them.-LENGTH: at most 450 words of prose across the whole page. Tables, lists and-code do not count against that, so put enumerable facts — packages, entry-points, languages — in a table or a list rather than describing them in-sentences. Say each thing once: the reader has the architecture page and the-per-module pages for depth, and a sentence that restates a heading is a-sentence they read twice.+LENGTH: at most 450 words of prose across the whole page. Say each thing once:+the reader has the architecture page and the per-module pages for depth, and a+sentence that restates a heading is a sentence they read twice.diff --git a/packages/core/src/repowise/core/generation/templates/stub/repo_overview.j2 b/packages/core/src/repowise/core/generation/templates/stub/repo_overview.j2index 73604b6de..fae3642b6 100644--- a/packages/core/src/repowise/core/generation/templates/stub/repo_overview.j2+++ b/packages/core/src/repowise/core/generation/templates/stub/repo_overview.j2@@ -35,19 +35,6 @@{% endfor %}{% endif %}-{% if ctx.entry_points %}-## Entry Points-Start here when reading the codebase.-{# Capped: a monorepo where every UI folder has an index.ts produces 75 of- these, and a 75-item wall is not a place to start reading. #}-{% for ep in ctx.entry_points[:15] %}-- `{{ ep }}`-{% endfor %}-{%- if ctx.entry_points | length > 15 %}-- ...and {{ ctx.entry_points | length - 15 }} more.-{% endif %}-{% endif %}-{% if ctx.execution_flows %}## Primary Execution Flows{% for f in ctx.execution_flows %}@@ -63,14 +50,10 @@ Groups of files that depend on each other far more than on the rest of the repo.{% endfor %}{% endif %}-{% if ctx.top_files_by_pagerank %}-## Most Central Files-Ranked by PageRank over the import graph: the files most of the codebase ultimately depends on.-{% for item in ctx.top_files_by_pagerank %}-- `{{ item.path }}` ({{ "%.4f"|format(item.score) }})-{% endfor %}-{% endif %}-+{# Two path tables used to sit above: the entry points and the highest-PageRank+ files. Both are lists a reader can produce with `ls` and neither says what+ the file is for. The summary sentence still names the first few entry points+ in a sentence, which is where a path is worth reading. #}{# A repository-wide percentage breakdown of languages told the reader nothingthey could act on — "python 57.2%" is not a fact anyone navigates by. Thepackage table names the languages per package, which is, and the summarydiff --git a/tests/unit/generation/test_overview_capability_wiring.py b/tests/unit/generation/test_overview_capability_wiring.pyindex e1852a75b..174a574d8 100644--- a/tests/unit/generation/test_overview_capability_wiring.py+++ b/tests/unit/generation/test_overview_capability_wiring.py@@ -362,3 +362,26 @@ async def test_a_run_with_no_glossary_page_does_not_build_the_corpus(tmp_path):assert store.asked == []assert all(s.module_corroboration == () for s in run.gen.onboarding_signals)+++async def test_the_overview_is_handed_the_repositorys_own_prose(tmp_path):+ """The front page's only natural-language input, wired at the level.++ Structure keeps every path and count; this supplies the words. Without it+ the payload is entirely structural and nothing in the prompt says what the+ product is for.+ """+ run = _run(tmp_path)+ await build_level6_coros(run)++ digest = run.gen.overview_kwargs["prose_digest"]+ assert "## Blast radius" in digest+ assert "Blast radius is the set of files a change can reach" in digest+++async def test_a_repository_with_no_path_still_generates_an_overview(tmp_path):+ """``repo_path`` is optional on the run, and an absent one is not a crash."""+ run = _run(tmp_path)+ run.repo_path = None+ assert await build_level6_coros(run)+ assert run.gen.overview_kwargs["prose_digest"] == ""diff --git a/tests/unit/generation/test_overview_front_page.py b/tests/unit/generation/test_overview_front_page.pynew file mode 100644index 000000000..0ab7b99d9--- /dev/null+++ b/tests/unit/generation/test_overview_front_page.py@@ -0,0 +1,191 @@+"""The front page's shape: what it is told to write, and what it stopped printing.++`/repos/[id]/docs` opens the repo overview, and it read like a directory+listing. Two reasons, both fixed here. The prompt's payload had no+natural-language input at all, so nothing in it said what the product was for;+and it rendered an entry-point table and a top-files-by-PageRank table, which+are lists a reader can produce with `ls`.++`repo_overview.j2` is a prompt, not a page, so the tests split accordingly:+what the model is *told* is asserted on the rendered prompt, and what ships is+asserted on the page the generator returns.+"""++from __future__ import annotations++import pytest++from repowise.core.generation.concept_tree.vocabulary import HouseTerm+from repowise.core.generation.context_assembler import ContextAssembler+from repowise.core.generation.models import GenerationConfig+from repowise.core.generation.overview_tables import (+ CAPABILITY_TABLE_HEADING,+ PACKAGE_TABLE_HEADING,+ select_capabilities,+)+from repowise.core.generation.page_generator import PageGenerator+from repowise.core.ingestion.models import PackageInfo, ParsedFile, RepoStructure+from repowise.core.providers.llm.mock import MockProvider++from .conftest import _make_file_info++DIGEST = "## The intelligence layers\nFive of them, over one index."+++def structure():+ return RepoStructure(+ is_monorepo=True,+ packages=[+ PackageInfo(+ name="core",+ path="packages/core",+ language="python",+ manifest_file=None,+ entry_points=[],+ )+ ],+ root_language_distribution={"python": 1.0},+ total_files=2,+ total_loc=200,+ entry_points=["packages/cli/main.py"],+ )+++def parsed_files():+ return [+ ParsedFile(file_info=_make_file_info(p), symbols=[], imports=[], exports=[])+ for p in ("packages/core/a.py", "packages/core/b.py")+ ]+++def capabilities():+ return select_capabilities(+ [+ HouseTerm(+ term="Dead code",+ definition="Dead code is code no import path reaches.",+ definition_source="src/analysis/dead_code.py",+ source_paths=("README.md",),+ doc_frequency=1,+ code_frequency=5,+ is_indexed_symbol=False,+ )+ ],+ ["Dead Code and Reachability Analysis"],+ )+++def prompt(structure, *, pagerank=None, prose_digest="") -> str:+ config = GenerationConfig()+ assembler = ContextAssembler(config)+ gen = PageGenerator(MockProvider(), assembler, config)+ ctx = assembler.assemble_repo_overview(+ structure, pagerank or {}, [], {}, prose_digest=prose_digest+ )+ return gen._render("repo_overview.j2", ctx=ctx, repo_git_summary=None)+++class TestTheProseKeyhole:+ def test_the_digest_reaches_the_model(self, structure):+ assert DIGEST in prompt(structure, prose_digest=DIGEST)++ def test_it_is_scoped_to_vocabulary_and_framing(self, structure):+ """Prose may name things. It may not name paths, counts or packages —+ the guardrail the planner's measured collapse leaves behind."""+ body = prompt(structure, prose_digest=DIGEST)+ opening = body[: body.index("## Structural facts")]+ assert "vocabulary, framing and what the product is for" in opening+ assert "Do **not** take a path, a count, a package name" in opening++ def test_structure_is_still_named_as_the_authority(self, structure):+ assert "Authoritative for every path, count and name." in prompt(structure)++ def test_no_readme_leaves_no_empty_section(self, structure):+ """A repository with a thin or absent README is the common case."""+ assert "What the project says about itself" not in prompt(structure)+++class TestWhatItIsToldToWrite:+ def test_the_shape_is_fixed(self, structure):+ body = prompt(structure)+ assert body.index("## Project Summary") < body.index("## Architecture")+ assert body.index("## Architecture") < body.index("## Key concepts")++ def test_key_concepts_are_ideas_not_directories(self, structure):+ assert "Not a package, not a directory, not a file." in prompt(structure)++ def test_enumerable_facts_are_no_longer_pushed_into_lists(self, structure):+ """The old contract said to put facts in a table or a list, which is+ the instruction that produced a page of paths."""+ assert "in a table or a list" not in prompt(structure)+++class TestPathsAreEvidenceNotContent:+ def test_pagerank_stays_in_the_payload(self, structure):+ """Ranking evidence: it is how the model knows what to call central."""+ body = prompt(structure, pagerank={"packages/core/a.py": 0.9})+ assert "packages/core/a.py" in body+ assert "PageRank" in body++ def test_and_the_model_is_told_not_to_print_it(self, structure):+ assert "**Do not print any of these paths.**" in prompt(structure)++ def test_the_required_sections_no_longer_include_entry_points(self):+ from repowise.core.generation.page_generator.prompts import SYSTEM_PROMPTS++ assert "## Entry Points" not in SYSTEM_PROMPTS["repo_overview"]++ async def test_the_deterministic_page_prints_no_path_tables(+ self, structure, parsed_files+ ) -> None:+ """The keyless page is a real shipped page, not a fallback stub."""+ config = GenerationConfig(deterministic=True)+ gen = PageGenerator(MockProvider(), ContextAssembler(config), config)+ page = await gen.generate_repo_overview(+ structure,+ {"packages/core/a.py": 0.9},+ [],+ {},+ repo_name="testrepo",+ parsed_files=parsed_files,+ )+ assert "## Entry Points" not in page.content+ assert "## Most Central Files" not in page.content++ async def test_it_still_names_the_entry_point_in_a_sentence(+ self, structure, parsed_files+ ) -> None:+ """Where a path is worth reading, it stays. A table of them is not."""+ config = GenerationConfig(deterministic=True)+ gen = PageGenerator(MockProvider(), ContextAssembler(config), config)+ page = await gen.generate_repo_overview(+ structure, {}, [], {}, repo_name="testrepo", parsed_files=parsed_files+ )+ assert "packages/cli/main.py" in page.content+++async def test_what_it_does_lands_above_what_it_is_made_of(+ structure, parsed_files, capabilities+) -> None:+ """Reading order: the capabilities answer the question the packages do not.++ The model path only. Both tables are appended to whatever the model wrote,+ and it is now told to write neither, so the append order is the reading+ order. The deterministic page carries ``## Packages`` in its own template+ and keeps that position.+ """+ config = GenerationConfig()+ gen = PageGenerator(MockProvider(), ContextAssembler(config), config)+ page = await gen.generate_repo_overview(+ structure,+ {},+ [],+ {},+ repo_name="testrepo",+ parsed_files=parsed_files,+ capabilities=capabilities,+ )+ assert page.content.index(CAPABILITY_TABLE_HEADING) < page.content.index(PACKAGE_TABLE_HEADING)diff --git a/tests/unit/generation/test_overview_package_table.py b/tests/unit/generation/test_overview_package_table.pyindex 07271f378..0ee954021 100644--- a/tests/unit/generation/test_overview_package_table.py+++ b/tests/unit/generation/test_overview_package_table.py@@ -341,3 +341,20 @@ def test_table_build_is_logged(sample_config, structure, parsed_files):build_package_table([])events = {e["event"] for e in logs}assert "overview_package_table_empty" in events+++def test_the_provenance_footer_survives_a_table_that_lands_last():+ """The section ends at the footer's rule, not only at the next heading.++ The deterministic page used to render a path table below its packages, so+ the replacement always had a heading to stop at. With those tables gone the+ packages can be the last section, and without this the footer is inside the+ span that gets replaced.+ """+ page = (+ "# Repository Overview: r\n\n## Packages\n- **core** (`packages/core`, python)\n\n"+ "---\n\n*Built from the code's structure.*\n"+ )+ out = embed_package_table(page, "| Package |\n|---|\n| core |")+ assert "| core |" in out+ assert "*Built from the code's structure.*" in outdiff --git a/tests/unit/generation/test_readme_digest.py b/tests/unit/generation/test_readme_digest.pynew file mode 100644index 000000000..e1227419d--- /dev/null+++ b/tests/unit/generation/test_readme_digest.py@@ -0,0 +1,167 @@+"""The front page's one natural-language input, and the bound on it.++The repo overview's payload was entirely structural — pagerank, communities,+package file counts, entry-point paths — so the page could describe a directory+tree accurately and never say what the product was for. This is the keyhole+that fixes that, and the tests below are mostly about the keyhole staying a+keyhole: ``concept_tree/vocabulary.py`` records a measured collapse from+handing 9,000 characters of README to the outline *planner*, and the cap here+is what keeps this the writing task it is rather than a second selection task.+"""++from __future__ import annotations++from pathlib import Path++import pytest++from repowise.core.generation.context.readme_digest import _MAX_CHARS, readme_digest++_README = """<p align="center"><img src="logo.png" alt="logo"></p>++[](https://example.com)++## One index, three ways to use it++These are not disconnected scanners. The graph locates what git history flags.++Instructions nobody needs on the front page.++### Pick your front door++## Know what is dangerous before you merge++Three deterministic signals, all computed from the graph and git history.++```python+this is an example, not a sentence about the product+```++#### An option inside a section++## The [PR bot](https://example.com/bot)++Install the GitHub App and the index shows up where the decision gets made.+"""+++#: The shape almost every README opens its install section with. The `#` line+#: is a shell comment, not a heading, and the paragraph after the fence belongs+#: to the heading above it.+_FENCED_README = """## Install++```bash+# clone the repo+git clone https://x/y+```++Then run it.++## Usage++One command.+"""+++def repo(tmp_path: Path) -> Path:+ (tmp_path / "README.md").write_text(_README, encoding="utf-8")+ return tmp_path+++class TestWhatItLifts:+ def test_headings_keep_their_level(self, repo: Path) -> None:+ out = readme_digest(repo)+ assert "## One index, three ways to use it" in out+ assert "## Know what is dangerous before you merge" in out++ def test_each_heading_carries_its_opening_paragraph(self, repo: Path) -> None:+ assert "The graph locates what git history flags." in readme_digest(repo)++ def test_only_the_opening_paragraph(self, repo: Path) -> None:+ """The rest of a section is instructions, options and examples.++ Reading openers only is what fits a 60KB README inside the cap without+ choosing which parts are relevant, which would be a selection decision+ made on prose.+ """+ assert "Instructions nobody needs" not in readme_digest(repo)++ def test_code_is_not_prose(self, repo: Path) -> None:+ assert "this is an example" not in readme_digest(repo)++ def test_a_comment_in_a_shell_block_is_not_a_heading(self, tmp_path: Path) -> None:+ """Most READMEs open the install section with exactly this.++ Read as a heading it becomes a name the project supposedly gives one of+ its own parts, and it spends the cap to say it.+ """+ (tmp_path / "README.md").write_text(_FENCED_README, encoding="utf-8")+ out = readme_digest(tmp_path)+ assert "clone the repo" not in out+ assert "## Usage" in out++ def test_a_fence_does_not_swallow_the_paragraph_after_it(self, tmp_path: Path) -> None:+ """The same bug, seen from the other side: a mistracked fence hid the+ rest of the document from the heading that owned it."""+ (tmp_path / "README.md").write_text(_FENCED_README, encoding="utf-8")+ assert "Then run it." in readme_digest(tmp_path)++ def test_badges_and_images_do_not_spend_the_cap(self, repo: Path) -> None:+ out = readme_digest(repo)+ assert "logo.png" not in out+ assert "a.svg" not in out++ def test_a_link_keeps_its_words_and_loses_its_url(self, repo: Path) -> None:+ out = readme_digest(repo)+ assert "## The PR bot" in out+ assert "example.com/bot" not in out++ def test_a_heading_with_nothing_under_it_still_names_the_thing(self, repo: Path) -> None:+ """A table of contents heading is a name, and a name is vocabulary."""+ assert "### Pick your front door" in readme_digest(repo)++ def test_deeper_headings_are_not_names_of_things(self, repo: Path) -> None:+ assert "An option inside a section" not in readme_digest(repo)++ def test_the_docs_readme_is_read_too(self, repo: Path) -> None:+ docs = repo / "docs"+ docs.mkdir()+ (docs / "README.md").write_text("## The wiki format\n\nPages, not files.\n", "utf-8")+ assert "## The wiki format" in readme_digest(repo)+++class TestBounds:+ def test_capped(self, tmp_path: Path) -> None:+ body = "\n\n".join(f"## Section {i}\n\n{'word ' * 60}" for i in range(400))+ (tmp_path / "README.md").write_text(body, encoding="utf-8")+ assert len(readme_digest(tmp_path)) <= _MAX_CHARS++ def test_the_cap_is_well_under_the_measured_collapse(self) -> None:+ """9,000 characters is what collapsed the planner. This is the bound."""+ assert _MAX_CHARS <= 3000++ def test_the_cap_cuts_in_document_order(self, tmp_path: Path) -> None:+ """Not by relevance: ranking prose would be the selection this avoids."""+ body = "\n\n".join(f"## Section {i}\n\n{'word ' * 60}" for i in range(400))+ (tmp_path / "README.md").write_text(body, encoding="utf-8")+ out = readme_digest(tmp_path)+ assert "## Section 0" in out+ assert "## Section 399" not in out++ def test_no_readme_yields_nothing_rather_than_a_heading(self, tmp_path: Path) -> None:+ """So the template drops the whole block instead of labelling an absence."""+ assert readme_digest(tmp_path) == ""++ def test_a_readme_that_is_all_prose_yields_nothing(self, tmp_path: Path) -> None:+ (tmp_path / "README.md").write_text("Just a paragraph, no headings.\n", encoding="utf-8")+ assert readme_digest(tmp_path) == ""++ def test_it_reads_only_root_and_docs(self, tmp_path: Path) -> None:+ """Root-anchored for the reason ``DOC_FILES`` is: a ``docs/*.md`` glob+ matches a path tail, and vendored documents reached the glossary once+ already that way."""+ vendored = tmp_path / "node_modules" / "other"+ vendored.mkdir(parents=True)+ (vendored / "README.md").write_text("## Somebody else's product\n\nNo.\n", "utf-8")+ assert readme_digest(tmp_path) == ""diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex 0246f93aa..b921f3a3f 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -17,6 +17,8 @@ from pathlib import Pathimport pytestfrom repowise.core.generation.concept_tree.vocabulary import (+ _doc_paths,+ _scan_tree,extract_house_terms,extract_terms,)@@ -860,3 +862,67 @@ def test_a_hard_wrapped_markdown_sentence_is_read_whole(tmp_path: Path) -> None:)# A sentence that already fits on one line is unaffected.assert by_term["Dead code"].definition == "Dead code is a rule no posting path reaches."+++# ---------------------------------------------------------------------------+# Which files may be cited as where a capability is written+# ---------------------------------------------------------------------------+#+# The capability table's third column is headed "Where it is written", and the+# glossary cites the same path. Both were rendering scratch harnesses under a+# git-excluded ``local-stash/`` and a competitive-positioning document in+# ``docs/`` as the authority for what a term means. Nothing downstream can tell+# a planning document from a subsystem guide by its headings, so the source set+# is decided here.+++def repo_with_scratch(repo: Path) -> Path:+ """The fixture repository plus the two shapes that leaked."""+ (repo / ".gitignore").write_text("local-stash/\n", encoding="utf-8")+ stash = repo / "local-stash" / "harnesses"+ stash.mkdir(parents=True)+ (stash / "_toggle.py").write_text(+ '"""Blast radius is whatever this throwaway harness says it is."""\n',+ encoding="utf-8",+ )+ (repo / "docs" / "BENCHMARKS.md").write_text(+ "# Benchmarks\n\n## Blast radius\n\nWe beat every competitor on blast radius.\n",+ encoding="utf-8",+ )+ return repo+++def test_a_tree_the_repository_excludes_is_not_the_repository_talking(+ repo_with_scratch: Path,+) -> None:+ """A path the user told git to ignore is scratch work, not documentation.++ ``local-stash/`` is absent from the index, the file tree and every other+ surface, so citing it sends the reader to a file they cannot open.+ """+ prose, _ = _scan_tree(repo_with_scratch)+ assert not [rel for rel, _ in prose if rel.startswith("local-stash/")]+++def test_an_excluded_harness_does_not_define_a_term(repo_with_scratch: Path) -> None:+ """The visible failure: the harness supplied the sentence and the citation."""+ term = by_term(repo_with_scratch)["Blast radius"]+ assert "local-stash" not in (term.definition_source or "")+ assert "local-stash" not in " ".join(term.source_paths)+++def test_a_market_facing_document_is_not_mined(repo_with_scratch: Path) -> None:+ """Its headings are product claims, and they look exactly like subsystems."""+ assert "docs/BENCHMARKS.md" not in [+ p.relative_to(repo_with_scratch).as_posix() for p in _doc_paths(repo_with_scratch)+ ]+++def test_excluding_scratch_does_not_cost_a_real_term(repo_with_scratch: Path) -> None:+ """The filter is aimed at the source set, not at the vocabulary.++ "Blast radius" is named by two real documents and spelled by real source,+ so it survives losing both leaked citations.+ """+ assert "Blast radius" in by_term(repo_with_scratch)