Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): a glossary page, written entirely in the repository's own words (#1276)
Raghav Chamadiya096dfe02cda7
diff --git a/docs/reference/CONFIG.md b/docs/reference/CONFIG.mdindex 0d0f9e082..566bff9ee 100644--- a/docs/reference/CONFIG.md+++ b/docs/reference/CONFIG.md@@ -120,8 +120,10 @@ Keys name model-written synthesis pages: `repo_overview` or`onboarding/<slot>` for `guided_tour`, `getting_started`, `codebase_map`,`key_concepts`, `how_it_works`, `development_guide`, and `active_landscape`.`onboarding/project_overview` is invalid because that promoted slot is the-`repo_overview` page. Unknown keys and malformed values fail generation with a-configuration error instead of being ignored.+`repo_overview` page. `onboarding/glossary` is accepted but has no effect: that+page is rendered from mined vocabulary alone, with no model in its path, so+there is no prompt for extra evidence to reach. Unknown keys and malformed+values fail generation with a configuration error instead of being ignored.Each value is an ordered list of repository-relative paths. A file is eligibleonly when it was included in the indexed source map and contains non-emptydiff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 2be2e9eca..416ebfaf3 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -457,7 +457,7 @@ def _definition_after_heading(text: str, offset: int) -> str | None:blank line above; anchoring on the end sidesteps that entirely."""lines = text[offset:].split("\n")[1 : _DEFINITION_SCAN_LINES + 1]- for line in lines:+ for start, line in enumerate(lines):stripped = line.strip()if not stripped:continue@@ -465,13 +465,36 @@ def _definition_after_heading(text: str, offset: int) -> str | None:return Noneif stripped.startswith(_NOT_PROSE):continue- sentence = _first_sentence(stripped)+ sentence = _first_sentence(_join_markdown_wrapped(lines, start))if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:return sentencereturn Nonereturn None+def _join_markdown_wrapped(lines: list[str], start: int) -> str:+ """The markdown paragraph beginning at ``start``, rejoined into one string.++ The reST scan has done this since #1248; the markdown scan never did, and a+ hard-wrapped README is as common as a hard-wrapped ``.txt``. Reading one+ line returned the author's sentence cut where their editor wrapped it —+ "Blast radius is the set of accounts a posting can reach through the+ ledger" — which then reads as a definition that trails off mid-thought.++ Stops at the paragraph break, at the next heading, at anything that is not+ prose, and as soon as the text has a sentence in it.+ """+ parts = [lines[start].strip()]+ for offset in range(start + 1, len(lines)):+ if _SENTENCE_END.search(" ".join(parts)):+ break+ nxt = lines[offset].strip()+ if not nxt or nxt.startswith(_NOT_PROSE) or _HEADING_LINE.match(lines[offset]):+ break+ parts.append(nxt)+ return " ".join(parts)++# ---------------------------------------------------------------------------# Which documents to read# ---------------------------------------------------------------------------@@ -767,6 +790,16 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:truncated = Falsefor dirpath, dirnames, filenames in fs_walk.walk_repo(repo_root, prune_dirs=_EXTRA_PRUNED_DIRS):+ # Hidden directories are tool and agent territory — CI definitions,+ # editor state, hook configuration — never the subsystem prose that+ # answers "was this term built". The walker already prunes the named+ # ones and any nested checkout, but a *stale* worktree copy has had its+ # ``.git`` file removed and reads as ordinary source: on this+ # repository that put a second, deeper copy of every docstring under+ # ``.claude/`` ahead of the original, and the mined definitions cited+ # paths a reader has no reason to open. Pruning the whole class costs+ # one string test and no repository keeps its documented source here.+ dirnames[:] = [d for d in dirnames if not d.startswith(".")]for name in dirnames:dir_names.add(_singular(name.lower()))if truncated:diff --git a/packages/core/src/repowise/core/generation/house_vocabulary.py b/packages/core/src/repowise/core/generation/house_vocabulary.pynew file mode 100644index 000000000..7c39aabf8--- /dev/null+++ b/packages/core/src/repowise/core/generation/house_vocabulary.py@@ -0,0 +1,291 @@+"""Selecting and rendering the repository's own vocabulary.++Two pages present mined house terms: the overview's capability table names the+six the repository is most about, and the glossary defines all of them. They are+the same computation at two depths — one ranked, corroborated list, sliced+short for the front page and long for the lookup page — so the selection lives+here and both callers read it rather than each deriving its own.++Everything in this module is deterministic. A term, the repository's own+sentence about it, and the path that sentence was read from are facts the run+already holds, and facts written by a model are resampled on every render: two+calls with the same prompt, the same model and the same temperature produced+overviews that disagreed on their row count and on which paths they cited. So+these tables are built here and embedded after the page comes back, and the+glossary is rendered without a model at all.+"""++from __future__ import annotations++import re+from collections.abc import Iterable, Sequence+from dataclasses import dataclass++import structlog++from .concept_tree.vocabulary import HouseTerm, phrase_pattern, term_words++log = structlog.get_logger(__name__)+++@dataclass(frozen=True)+class SelectedTerm:+ """One mined term that earned a place on a page."""++ term: str+ #: The repository's own sentence, or ``None``. Never invented — a term the+ #: repository named without defining is still a term, and writing a+ #: definition for it is the one thing the vocabulary miner refuses to do.+ definition: str | None+ #: The document or source file the sentence was read from, falling back to+ #: the first document that named the term. Always a real path.+ source_path: str+ #: How many parts of the system name it. The strength of the corroboration+ #: rather than a fact about the repository, so it ranks and it logs; the+ #: overview does not render it.+ corroborating_pages: int+ #: Whether the codebase defines a symbol by this name. A term that is also+ #: a symbol may be rendered in backticks; a coined one may not, because the+ #: grounding pass strips backticks off any token it cannot resolve.+ is_indexed_symbol: bool = False+ #: What the structural side calls the parts of the system that name this+ #: term, in corpus order. This is the honest answer to "where is this used"+ #: — every entry is a module group cut from the dependency graph, so it+ #: points at code rather than at more prose.+ corroborating_names: tuple[str, ...] = ()+++# ---------------------------------------------------------------------------+# Is this prose a definition?+# ---------------------------------------------------------------------------+#+# A mined "definition" is whatever prose sat nearest the term, and near a term+# in a README that is often not a sentence about it. Two real examples from+# this repository's own front page:+#+# CLI -> "repowise init [PATH] # index a codebase (one-time; ...)"+# Distill -> "`repowise distill <cmd>` compresses command output *before*+# the agent reads it:"+#+# The first is a line of shell with an inline comment. The second is a lead-in+# that ends on a colon because the explanation is the code block underneath.+# Neither says what the thing is, and an em dash is a better answer than+# either: the reader learns the term exists and is not misinformed about what+# it means.++#: A command line rather than a sentence: a shell comment, a prompt, an option+#: flag, a redirect or a pipeline.+#:+#: The redirect and pipe test requires whitespace on both sides. Bare ``<`` and+#: ``>`` reject real prose: "Decisions are co-located ... under the+#: ``decision:<record_id>`` namespace" is a sentence, and the angle brackets in+#: it are a placeholder, not a redirect.+_LOOKS_LIKE_COMMAND = re.compile(r"(^\s*[$>]\s|\s#\s|\s--?[a-zA-Z]|\s[|<>]\s)")+#: Ends where the real explanation begins — a colon, or an unclosed opener.+_TRAILS_OFF = (":", ",", ";", "-", "—", "–", "(", "[")+#: How a statement finishes. Checked as well as :data:`_TRAILS_OFF`, because+#: the two catch different things: that one rejects prose that stops mid-clause,+#: this one rejects text that was never prose. A stray line of source lifted out+#: of a scratch file — ``decisions/__init__.py ---- (PKG / "__init__.py")+#: .write_text( '`` — passes every other test here (it names the term, it has+#: words, it opens with a letter) and is stopped only by not ending in a way a+#: sentence can end. Every definition mined from four repositories that a human+#: would call a definition ends in one of these.+_SENTENCE_END = (".", "!", "?")+++def is_a_sentence(text: str) -> bool:+ """Whether mined prose reads as a statement about the term.++ Deliberately shallow. This is not grammar checking — it is the difference+ between a sentence and a fragment of shell, and getting it wrong in the+ strict direction costs a definition, which every caller renders as an em+ dash. Getting it wrong the other way puts a command line on a page as+ though it explained something.+ """+ text = " ".join(text.split())+ # Three, not four: "Blast-radius request/response models." is terse and is+ # still the repository's own answer to what the term means. Two words is+ # where the fragments live ("See below", "Two parts").+ if len(text.split()) < 3:+ return False+ text = text.rstrip()+ if text.endswith(_TRAILS_OFF) or not text.endswith(_SENTENCE_END):+ return False+ if _LOOKS_LIKE_COMMAND.search(text):+ return False+ # A statement starts with a word, not with punctuation or a code fence.+ return text[:1].isalpha()+++def cell(text: str) -> str:+ """Text safe to put in a markdown table cell.++ A pipe ends the cell wherever it appears, and mined prose is the+ repository's text rather than ours — a definition that quotes a shell+ pipeline or a reStructuredText grid row would otherwise shift every column+ to its right.+ """+ return " ".join(text.split()).replace("|", "\\|")+++def clamp(text: str, limit: int) -> str:+ """*text* as one line and table-safe, cut to about *limit* characters.++ Truncates first, escapes second. The other order cuts an escaped ``\\|`` in+ half and leaves the backslash orphaned at the end of the cell, which is the+ one thing escaping exists to prevent. "About" because escaping runs after:+ a definition quoting a shell pipeline ends a character or two over, which+ costs nothing a reader can see.+ """+ text = " ".join(text.split())+ if len(text) > limit:+ text = text[: limit - 1].rstrip() + "…"+ return cell(text)+++# ---------------------------------------------------------------------------+# Selection+# ---------------------------------------------------------------------------+++def select_terms(+ house_terms: Sequence[HouseTerm],+ module_names: Iterable[str],+ *,+ limit: int | None = None,+) -> list[SelectedTerm]:+ """The mined terms worth publishing, in the order they go on a page.++ Ranked by document frequency alone the mined terms are not publishable: on+ this repository the top of that list holds the repository's own name,+ "DONE", "Architecture" and "Files changed". So a term reaches a page only+ with corroboration from a second, independently-derived artifact.++ ``module_names`` are what the structural side calls the parts of the+ system: one string per module group, its title followed by its summary. A+ module group is cut from the dependency graph and named from the code, so a+ term appearing in one was arrived at twice — from the documents and from+ the structure — independently. That needs no stopword list and no+ per-repository tuning.++ Titles alone are about ninety short strings, which is too thin a net: it+ misses "Knowledge Graph" and "Code Health" while letting "Architecture" and+ "Workspace" through on an incidental word. The summaries are what make the+ corroboration mean something.++ Groups rather than written module *pages*, deliberately. A group exists on+ every run, so a scoped run that regenerates one page selects the same rows+ as a full one — a section that shrinks depending on how generation was+ invoked is the instability these deterministic tables exist to remove.++ **Multi-word terms come first.** Not as a filter — a single word still+ reaches the page when there is room — but ahead of single words, because a+ subsystem is nearly always named with two words ("blast radius", "dead+ code", "change risk") and an ordinary English word with one. The ranking in+ :func:`~...vocabulary.extract_house_terms` already encodes that as its+ tiebreak; here it leads, because document frequency barely discriminates+ (most repositories have two or three documents worth mining, so nearly+ every term ties at one) and a junk row is expensive on a short page.++ Ordering is total and derived only from the inputs, so two runs over an+ unchanged repository select the same terms in the same order. *limit*+ slices the tail: the whole list is one computation, and the front page+ takes its head while the glossary takes more of it.+ """+ corpus = [name for name in module_names if name]+ # Each entry is a group's title followed by its summary, so the first line+ # is the group's name — what to call the place a term turned up.+ titles = [entry.split("\n", 1)[0].strip() for entry in corpus]+ # The regex is the expensive part and nearly every (term, group) pair is a+ # miss, so a substring test on the term's first word rejects most pairs+ # before it runs. The same trick is what made the miner itself 3.3× faster.+ folded = [entry.lower() for entry in corpus]++ selected: list[SelectedTerm] = []+ for term in house_terms:+ words = term_words(term.term)+ if not words:+ continue+ # ASCII only. ``str.lower()`` and ``re.I`` disagree on a few code+ # points — "İ".lower() is two characters — so on a non-ASCII lead word+ # the cheap test can reject a pair the regex would have matched. Those+ # terms skip the prefilter and pay the regex.+ lead = words[0].lower() if words[0].isascii() else None+ pattern = phrase_pattern(term.term)+ # Sorted, not in corpus order. Everything else this function returns is+ # derived only from its inputs' *contents*, and a field that reordered+ # with the corpus would make two runs over an unchanged repository+ # disagree whenever module grouping shuffled.+ matched = tuple(+ sorted(+ titles[i]+ for i, entry in enumerate(corpus)+ if (lead is None or lead in folded[i]) and pattern.search(entry)+ )+ )+ hits = len(matched)+ if not hits:+ continue+ definition = term.definition+ # No "the definition must name the term" test. It was built, measured+ # and dropped: a heading gloss is the commonest definition shape there+ # is, and it does not restate its own heading. "## Blast radius" over+ # "The set of files a change can reach." is a repository defining its+ # term perfectly, and the rule deleted it — along with every definition+ # mined from a bolded lead-in, which captures only the text after the+ # dash. It caught two junk rows here and would have emptied the+ # definition column of any repository that writes that way.+ if definition and not is_a_sentence(definition):+ # Keep the row, drop the claim. The term is real — the structure+ # corroborated it — but the prose nearest it is not a statement+ # about it, and an em dash misinforms nobody.+ log.info(+ "house_vocabulary.definition_rejected",+ term=term.term,+ text=" ".join(definition.split())[:120],+ )+ definition = None+ # Cite where the sentence came from, or -- when there is no sentence,+ # including one just rejected -- the document that named the term.+ # Citing the home of prose the page declined to quote would point a+ # reader at a line that is not there.+ source = (term.definition_source if definition else None) or (+ term.source_paths[0] if term.source_paths else None+ )+ if source is None:+ # A term with no path at all cannot be cited, and an uncitable row+ # is the shape of claim this wiki does not make.+ continue+ selected.append(+ SelectedTerm(+ term=term.term,+ definition=definition,+ source_path=source,+ corroborating_pages=hits,+ is_indexed_symbol=term.is_indexed_symbol,+ corroborating_names=matched,+ )+ )++ selected.sort(key=lambda t: (len(term_words(t.term)) == 1, -t.corroborating_pages, t.term))+ kept = selected if limit is None else selected[:limit]+ log.info(+ "house_vocabulary.selected",+ mined=len(house_terms),+ corroborated=len(selected),+ kept=len(kept),+ terms=[t.term for t in kept[:12]],+ )+ if house_terms and not selected:+ # The documents name things the structure does not. That is a real+ # answer about a repository — marketing vocabulary with no cluster+ # behind it — but it is also what a corroboration corpus arriving empty+ # looks like, so the two counts that separate them are logged rather+ # than the section just not appearing.+ log.warning(+ "house_vocabulary.uncorroborated",+ mined=len(house_terms),+ module_names=len(corpus),+ )+ return keptdiff --git a/packages/core/src/repowise/core/generation/onboarding/__init__.py b/packages/core/src/repowise/core/generation/onboarding/__init__.pyindex 9098fe183..44dc19950 100644--- a/packages/core/src/repowise/core/generation/onboarding/__init__.py+++ b/packages/core/src/repowise/core/generation/onboarding/__init__.py@@ -1,15 +1,14 @@"""Onboarding documentation collection.-A curated set of up to nine pages: Project Overview, Architecture Guide,-Guided Tour, Getting Started, Codebase Map, Key Concepts, How It Works,-Development Guide, Active Landscape, designed to be the first thing a new-contributor (or LLM agent) reads.+A curated set of up to nine pages: Project Overview, Guided Tour, Getting+Started, Codebase Map, Key Concepts, How It Works, Development Guide, Active+Landscape and Glossary, designed to be the first thing a new contributor (or+LLM agent) reads.-One slot ("project_overview") is *promoted*: it-reuse the existing ``repo_overview`` and ``architecture_diagram`` pages,-tagged via ``metadata.onboarding_slot``. The other seven slots are new pages-generated at level 8 with ``page_type='onboarding'`` and a-``metadata.subkind`` discriminator.+One slot ("project_overview") is *promoted*: it reuses the existing+``repo_overview`` page, tagged via ``metadata.onboarding_slot``. The other+eight slots are new pages generated at level 8 with ``page_type='onboarding'``+and a ``metadata.subkind`` discriminator.Architecture:- :mod:`slots` — slot identifiers, fixed reading order, promoted map.@@ -31,6 +30,7 @@ from .slots import (SLOT_CODEBASE_MAP,SLOT_DEVELOPMENT_GUIDE,SLOT_GETTING_STARTED,+ SLOT_GLOSSARY,SLOT_GUIDED_TOUR,SLOT_HOW_IT_WORKS,SLOT_KEY_CONCEPTS,@@ -47,6 +47,7 @@ __all__ = ["SLOT_CODEBASE_MAP","SLOT_DEVELOPMENT_GUIDE","SLOT_GETTING_STARTED",+ "SLOT_GLOSSARY","SLOT_GUIDED_TOUR","SLOT_HOW_IT_WORKS","SLOT_KEY_CONCEPTS",diff --git a/packages/core/src/repowise/core/generation/onboarding/registry.py b/packages/core/src/repowise/core/generation/onboarding/registry.pyindex 3e238ff0e..8c6168fbf 100644--- a/packages/core/src/repowise/core/generation/onboarding/registry.py+++ b/packages/core/src/repowise/core/generation/onboarding/registry.py@@ -39,6 +39,20 @@ class SubkindSpec:references; first occurrence sets priority and duplicatesare ignored. Missing files, symbols, or ranges becomeobservable evidence skips rather than generation errors.+ deterministic: The page is rendered from its context alone, with no+ provider call, on every run. Set it when the page is+ made of facts rather than of judgements: a model asked+ to restate enumerable facts resamples them, and two+ renders of one unchanged repository then differ. Such a+ subkind needs only its ``stub/`` template, because that+ is the only one that ever renders.+ needs_module_corroboration: Populate+ ``OnboardingSignals.module_corroboration`` for this+ subkind. Declared here rather than tested for by slot+ name in the level builder, so a second subkind that+ wants it cannot silently receive an empty tuple. It+ costs a batched store read, and a run emitting no+ subkind that wants it does not pay for one."""slot: str@@ -46,6 +60,8 @@ class SubkindSpec:template: strbuild_context: BuildContextevidence_references: EvidenceReferences | None = None+ deterministic: bool = False+ needs_module_corroboration: bool = False_REGISTRY: dict[str, SubkindSpec] = {}diff --git a/packages/core/src/repowise/core/generation/onboarding/signals.py b/packages/core/src/repowise/core/generation/onboarding/signals.pyindex 7939f39a4..176a6fe73 100644--- a/packages/core/src/repowise/core/generation/onboarding/signals.py+++ b/packages/core/src/repowise/core/generation/onboarding/signals.py@@ -55,3 +55,10 @@ class OnboardingSignals:# nothing, or when nothing it documents was built — all three are logged# where the mining happens, because an empty tuple here cannot say which.house_terms: tuple[HouseTerm, ...] = ()+ # What the structural side calls the parts of the system: one string per+ # module group, its title followed by its summary. The corroborating+ # artifact for a mined term — a group is cut from the dependency graph and+ # named from the code, so a term appearing in one was arrived at twice,+ # independently. Empty on every run that emits no page needing it, because+ # building it costs a store read.+ module_corroboration: tuple[str, ...] = ()diff --git a/packages/core/src/repowise/core/generation/onboarding/slots.py b/packages/core/src/repowise/core/generation/onboarding/slots.pyindex 8611c7bd6..157da656f 100644--- a/packages/core/src/repowise/core/generation/onboarding/slots.py+++ b/packages/core/src/repowise/core/generation/onboarding/slots.py@@ -1,8 +1,8 @@"""Onboarding slot identifiers and reading order.-The Onboarding collection is a fixed set of eight curated slots. One slot is+The Onboarding collection is a fixed set of nine curated slots. One slot is"promoted": it reuses the existing `repo_overview` page, tagged via-``metadata.onboarding_slot``. The other seven are new pages with+``metadata.onboarding_slot``. The other eight are new pages with``page_type='onboarding'`` and a ``metadata.subkind`` discriminator.Slot identifiers are used three ways:@@ -23,7 +23,7 @@ from __future__ import annotations# rendered prompt is byte-identical. Bump when a builder or template change# should reach already-cached pages. (File pages have an equivalent in# ``_generation_fingerprint``; onboarding pages had none until this.)-ONBOARDING_GENERATION_VERSION = "2"+ONBOARDING_GENERATION_VERSION = "3"# ---- Slot identifiers ----@@ -35,9 +35,14 @@ SLOT_KEY_CONCEPTS = "key_concepts"SLOT_HOW_IT_WORKS = "how_it_works"SLOT_DEVELOPMENT_GUIDE = "development_guide"SLOT_ACTIVE_LANDSCAPE = "active_landscape"+SLOT_GLOSSARY = "glossary"# Fixed reading order. Slots not yet implemented are silently skipped at# generation time and absent from the UI tree.+#+# The glossary is last on purpose: it is a lookup surface rather than a reading+# step, and a reader who does not yet know the vocabulary is not helped by+# meeting all of it at once.ONBOARDING_ORDER: tuple[str, ...] = (SLOT_PROJECT_OVERVIEW,SLOT_GUIDED_TOUR,@@ -47,6 +52,7 @@ ONBOARDING_ORDER: tuple[str, ...] = (SLOT_HOW_IT_WORKS,SLOT_DEVELOPMENT_GUIDE,SLOT_ACTIVE_LANDSCAPE,+ SLOT_GLOSSARY,)# Maps existing page_type → onboarding slot. The generator tags these pages@@ -67,6 +73,7 @@ SLOT_TITLES: dict[str, str] = {SLOT_HOW_IT_WORKS: "How It Works",SLOT_DEVELOPMENT_GUIDE: "Development Guide",SLOT_ACTIVE_LANDSCAPE: "Active Landscape",+ SLOT_GLOSSARY: "Glossary",}diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.pyindex cbe90bdee..b51f43978 100644--- a/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py@@ -14,6 +14,7 @@ from . import (codebase_map, # noqa: F401development_guide, # noqa: F401getting_started, # noqa: F401+ glossary, # noqa: F401guided_tour, # noqa: F401how_it_works, # noqa: F401key_concepts, # noqa: F401diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/glossary.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/glossary.pynew file mode 100644index 000000000..8da2276a6--- /dev/null+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/glossary.py@@ -0,0 +1,165 @@+"""Onboarding subkind: Glossary.++The words this repository uses for itself, defined in its own sentences. Every+row is read from the repository's own documents: the term, the sentence nearest+it, and the path that sentence came from. Nothing here is written for the page.++That is the whole design. A glossary is the page where an invented definition+does the most damage — it is the one a reader consults *because* they do not+know the answer, so they have nothing to check it against. So this page has no+model in its path at all: the spec is registered ``deterministic``, and two+renders of an unchanged repository are byte-identical.++The cost is honesty about coverage. A term the repository names without ever+defining renders with an em dash rather than a plausible sentence, and a+repository whose documents define nothing gets no page. Both are the correct+answers to a question this page cannot answer from the code.++Gate: at least five terms survive corroboration. Below that the page is a+handful of rows pretending to be a vocabulary.+"""++from __future__ import annotations++from dataclasses import dataclass, field++import structlog++from ...house_vocabulary import SelectedTerm, select_terms+from ..registry import SubkindSpec, register+from ..signals import OnboardingSignals+from ..slots import SLOT_GLOSSARY, SLOT_TITLES++log = structlog.get_logger(__name__)++#: Below this the page is not a vocabulary. Five is the floor the track set:+#: enough rows that a reader consults it rather than reads it.+_GATE_MIN_TERMS = 5+#: A lookup surface can be long — a reader scans for one row and ignores the+#: rest — but not unbounded. Forty is roughly two screens and is far above what+#: any repository measured so far corroborates.+_MAX_TERMS = 40+#: How many subsystems to name per row before the cell stops being scannable.+_MAX_USED_IN = 3+++@dataclass(frozen=True)+class GlossaryEntry:+ """One row of the glossary, entirely mined."""++ term: str+ #: The repository's own sentence about the term, or ``None`` when it named+ #: the term without defining it.+ definition: str | None+ #: The document the sentence was read from. Always a real path.+ source_path: str+ #: The parts of the system that name it, from the structural side.+ used_in: tuple[str, ...]+ #: Whether the codebase defines a symbol by this name, so the template+ #: knows whether backticks would survive a grounding pass.+ is_indexed_symbol: bool+++@dataclass+class GlossaryContext:+ repo_name: str+ entries: list[GlossaryEntry] = field(default_factory=list)+ #: Every document any surviving term was read from, deduplicated and+ #: ordered. Rendered as the page's provenance line.+ sources: list[str] = field(default_factory=list)+ #: How many terms were mined, how many survived corroboration, and how many+ #: of the rows carry a definition. All three are rendered: a glossary that+ #: quietly covers a third of its vocabulary, or that silently drops the tail+ #: of a long one, should say so on the page rather than imply completeness.+ mined: int = 0+ corroborated: int = 0+ defined: int = 0+++def _entry(selected: SelectedTerm) -> GlossaryEntry:+ names = selected.corroborating_names+ used_in = names[:_MAX_USED_IN]+ if len(names) > _MAX_USED_IN:+ # Said, not silently dropped. A truncated list reads as the whole list,+ # and "where is this used" is the column a reader acts on.+ used_in = (*used_in, f"and {len(names) - _MAX_USED_IN} more")+ return GlossaryEntry(+ term=selected.term,+ definition=selected.definition,+ source_path=selected.source_path,+ used_in=used_in,+ is_indexed_symbol=selected.is_indexed_symbol,+ )+++def _build(signals: OnboardingSignals) -> GlossaryContext | None:+ if not signals.house_terms:+ # Already logged where the mining happens, with the three counts that+ # separate "no repository to read" from "nothing written" from+ # "nothing built". Repeating them here would say less.+ return None++ # Every corroborated term, and no further test. A row whose definition+ # column is an em dash still carries two facts a reader came for — the+ # repository has a word for this, and here are the parts of the system+ # that use it — and dropping those rows was measured to cost more than it+ # saved: requiring a single word to arrive with a definition removed+ # "Workspace", "Coupling", "CLI", "Distill", "Costs" and "Decisions" here+ # and "Security" on django, to remove two weak rows. A lookup page is+ # judged on coverage.+ #+ # Selected whole, then capped. The count before the cap is carried onto the+ # page: the footer explains that a mined term reaches it only with+ # structural corroboration, and that is a false account of any term dropped+ # for length instead.+ corroborated = select_terms(signals.house_terms, signals.module_corroboration)+ selected = corroborated[:_MAX_TERMS]+ if len(selected) < _GATE_MIN_TERMS:+ log.info(+ "onboarding.glossary_gate_skipped",+ repo_name=signals.repo_name,+ mined=len(signals.house_terms),+ corroborated=len(corroborated),+ required=_GATE_MIN_TERMS,+ )+ return None++ # Alphabetical, because a glossary is looked up rather than read. Ranking+ # decided which terms are here; it has no job left once they are.+ entries = sorted((_entry(term) for term in selected), key=lambda e: e.term.lower())++ sources: list[str] = []+ for entry in entries:+ if entry.source_path not in sources:+ sources.append(entry.source_path)+ defined = sum(1 for entry in entries if entry.definition)++ log.info(+ "onboarding.glossary_built",+ repo_name=signals.repo_name,+ mined=len(signals.house_terms),+ corroborated=len(corroborated),+ terms=len(entries),+ defined=defined,+ documents=len(sources),+ )+ return GlossaryContext(+ repo_name=signals.repo_name,+ entries=entries,+ sources=sources,+ mined=len(signals.house_terms),+ corroborated=len(corroborated),+ defined=defined,+ )+++register(+ SubkindSpec(+ slot=SLOT_GLOSSARY,+ title=SLOT_TITLES[SLOT_GLOSSARY],+ template="glossary.j2",+ build_context=_build,+ deterministic=True,+ needs_module_corroboration=True,+ )+)diff --git a/packages/core/src/repowise/core/generation/overview_tables.py b/packages/core/src/repowise/core/generation/overview_tables.pyindex aeed57f19..5d8686a7d 100644--- a/packages/core/src/repowise/core/generation/overview_tables.py+++ b/packages/core/src/repowise/core/generation/overview_tables.py@@ -15,14 +15,19 @@ from __future__ import annotationsimport refrom collections.abc import Iterable, Sequence-from dataclasses import dataclassimport structlog-from .concept_tree.vocabulary import HouseTerm, phrase_pattern, term_words+from .concept_tree.vocabulary import HouseTerm+from .house_vocabulary import SelectedTerm, cell, clamp, select_termslog = structlog.get_logger(__name__)+#: One row of the capability table. The glossary renders the same object, so+#: the selection and the definition test live in :mod:`house_vocabulary` and+#: this name is what the overview calls it.+Capability = SelectedTerm+PACKAGE_TABLE_HEADING = "## Packages"CAPABILITY_TABLE_HEADING = "## What it does"@@ -85,17 +90,11 @@ def embed_package_table(content: str, table: str | None) -> str:## The overview's opening question is "what does this thing do", and the answer# should be in the repository's own words rather than in the generic vocabulary-# of static analysis. The mined house terms are those words. Ranked by document-# frequency alone they are not front-page material: on this repository the top-# of that list holds the repository's own name, "DONE", "Architecture" and-# "Files changed", which would put junk in half the rows.-#-# So a term reaches this table only with corroboration from a second,-# independently-derived artifact: a module page — built from the dependency-# graph, with no knowledge of the documents — has to name it in its title or-# its summary. That needs no stopword list and no per-repository tuning.+# of static analysis. The mined house terms are those words, selected and+# corroborated by :func:`~house_vocabulary.select_terms`; the front page takes+# the head of that list and the glossary takes more of it.-_MAX_CAPABILITY_ROWS = 6+MAX_CAPABILITY_ROWS = 6#: Long enough for a sentence, short enough that a row stays one line to read._MAX_CAPABILITY_DEFINITION = 160@@ -105,186 +104,20 @@ _CAPABILITY_SECTION_RE = re.compile()-@dataclass(frozen=True)-class Capability:- """One row: a mined term, what the repository says it is, and where from."""-- term: str- #: The repository's own sentence, or ``None``. Never invented — a term the- #: repository named without defining is still a capability, and writing a- #: definition for it is the one thing the vocabulary miner refuses to do.- definition: str | None- #: The document or source file the sentence was read from, falling back to- #: the first document that named the term. Always a real path.- source_path: str- #: How many module pages name it. Kept for the log, not rendered — it is- #: the strength of the corroboration, not a fact about the repository.- corroborating_pages: int--def select_capabilities(house_terms: Sequence[HouseTerm],module_names: Iterable[str],*,- limit: int = _MAX_CAPABILITY_ROWS,+ limit: int = MAX_CAPABILITY_ROWS,) -> list[Capability]:"""The terms worth putting on the front page, in the order they go there.- ``module_names`` are what the structural side calls the parts of the- system: the module groups' titles. They are the corroborating artifact — a- module group is cut from the dependency graph and named from the code, so- a term appearing in one was arrived at twice, from the documents and from- the structure, independently.-- Titles only. Community labels and directory keys were tried and measured- worse: a label like "Ingestion and Analysis Engine" is broad enough to- corroborate almost any single common word, which put "Architecture" and- "Workspace" on the front page of a real render.-- Groups rather than written module *pages*, deliberately. A group exists on- every run, so a scoped run that regenerates the overview alone selects the- same rows as a full one — a front-page section that shrinks depending on- how generation was invoked is the instability these deterministic tables- exist to remove. It is also the stronger claim of independence: a group's- name is computed, while a page's summary is model-written and resampled.-- The cost is reach. Measured on django, matching titles and summaries of- written pages corroborates 5 terms where names alone corroborate 2. The- upgrade path is to read module summaries out of the store rather than out- of this run, which would be both stable and rich; it needs level 6 to- become async and the vector store to be present, so it is not done here.-- **Multi-word terms come first.** Not as a filter — a single word still- reaches the table when there is room — but ahead of single words, because- a subsystem is nearly always named with two words ("blast radius", "dead- code", "change risk") and an ordinary English word with one. The ranking- in :func:`~...vocabulary.extract_house_terms` already encodes that as its- tiebreak; here it leads, because document frequency barely discriminates- at six rows (most repositories have two or three documents worth mining,- so nearly every term ties at one) and one junk row out of six is expensive.-- Ordering is total and derived only from the inputs, so two runs over an- unchanged repository select the same rows in the same order.- """- corpus = [name for name in module_names if name]- selected: list[Capability] = []- for term in house_terms:- pattern = phrase_pattern(term.term)- hits = sum(1 for page in corpus if pattern.search(page))- if not hits:- continue- definition = term.definition- if definition and not _is_a_sentence(definition):- # Keep the row, drop the claim. The capability is real — the- # structure corroborated it — but the prose nearest the term is- # not a statement about it, and an em dash misinforms nobody.- log.info(- "overview_capability_definition_rejected",- term=term.term,- text=" ".join(definition.split())[:120],- )- definition = None- # Cite where the sentence came from, or -- when there is no sentence,- # including one just rejected -- the document that named the term.- # Citing the home of prose the table declined to quote would point a- # reader at a line that is not there.- source = (term.definition_source if definition else None) or (- term.source_paths[0] if term.source_paths else None- )- if source is None:- # A term with no path at all cannot be cited, and an uncitable row- # on the front page is the shape of claim this wiki does not make.- continue- selected.append(- Capability(- term=term.term,- definition=definition,- source_path=source,- corroborating_pages=hits,- )- )-- selected.sort(key=lambda c: (len(term_words(c.term)) == 1, -c.corroborating_pages, c.term))- kept = selected[:limit]- log.info(- "overview_capabilities_selected",- mined=len(house_terms),- corroborated=len(selected),- kept=len(kept),- terms=[c.term for c in kept],- )- if house_terms and not selected:- # The documents name things the structure does not. That is a real- # answer about a repository — marketing vocabulary with no cluster- # behind it — but it is also what a corroboration corpus arriving- # empty looks like, so the two counts that separate them are logged- # rather than the section just not appearing.- log.warning(- "overview_capabilities_uncorroborated",- mined=len(house_terms),- module_names=len(corpus),- )- return kept---# A mined "definition" is whatever prose sat nearest the term, and near a term-# in a README that is often not a sentence about it. Two real examples from-# this repository's own front page:-#-# CLI -> "repowise init [PATH] # index a codebase (one-time; ...)"-# Distill -> "`repowise distill <cmd>` compresses command output *before*-# the agent reads it:"-#-# The first is a line of shell with an inline comment. The second is a lead-in-# that ends on a colon because the explanation is the code block underneath.-# Neither says what the thing is, and an em dash is a better answer than-# either: the reader learns the capability exists and is not misinformed about-# what it does.--#: A command line rather than a sentence: a shell comment, a prompt, an option-#: flag, a redirect or a pipeline.-#:-#: The redirect and pipe test requires whitespace on both sides. Bare ``<`` and-#: ``>`` reject real prose: "Decisions are co-located ... under the-#: ``decision:<record_id>`` namespace" is a sentence, and the angle brackets in-#: it are a placeholder, not a redirect.-_LOOKS_LIKE_COMMAND = re.compile(r"(^\s*[$>]\s|\s#\s|\s--?[a-zA-Z]|\s[|<>]\s)")-#: Ends where the real explanation begins — a colon, or an unclosed opener.-_TRAILS_OFF = (":", ",", ";", "-", "—", "–", "(", "[")---def _is_a_sentence(text: str) -> bool:- """Whether mined prose reads as a statement about the term.-- Deliberately shallow. This is not grammar checking — it is the difference- between a sentence and a fragment of shell, and getting it wrong in the- strict direction costs a definition, which the table already renders as an- em dash. Getting it wrong the other way puts a command line on the front- page as though it explained something.- """- text = " ".join(text.split())- # Three, not four: "Blast-radius request/response models." is terse and is- # still the repository's own answer to what the term means. Two words is- # where the fragments live ("See below", "Two parts").- if len(text.split()) < 3:- return False- if text.rstrip().endswith(_TRAILS_OFF):- return False- if _LOOKS_LIKE_COMMAND.search(text):- return False- # A statement starts with a word, not with punctuation or a code fence.- return text[:1].isalpha()---def _cell(text: str) -> str:- """Text safe to put in a markdown table cell.-- A pipe ends the cell wherever it appears, and mined prose is the- repository's text rather than ours — a definition that quotes a shell- pipeline or a reStructuredText grid row would otherwise shift every column- to its right.+ The head of :func:`~house_vocabulary.select_terms`. The glossary renders+ more of the same list, so the ranking, the corroboration and the definition+ test are shared rather than derived twice — the front page is the top of+ the glossary by construction, which is also what a reader expects of it."""- return " ".join(text.split()).replace("|", "\\|")+ return select_terms(house_terms, module_names, limit=limit)def build_capability_table(capabilities: Sequence[Capability]) -> str | None:@@ -303,10 +136,8 @@ def build_capability_table(capabilities: Sequence[Capability]) -> str | None:"|---|---|---|",]for cap in capabilities:- definition = _cell(cap.definition) if cap.definition else "—"- if len(definition) > _MAX_CAPABILITY_DEFINITION:- definition = definition[: _MAX_CAPABILITY_DEFINITION - 1].rstrip() + "…"- lines.append(f"| {_cell(cap.term)} | {definition} | `{_cell(cap.source_path)}` |")+ definition = clamp(cap.definition, _MAX_CAPABILITY_DEFINITION) if cap.definition else "—"+ lines.append(f"| {cell(cap.term)} | {definition} | `{cell(cap.source_path)}` |")log.debug("overview_capability_table_built", rows=len(capabilities))return "\n".join(lines)diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.pyindex afa3d63d1..ad19ed250 100644--- a/packages/core/src/repowise/core/generation/page_generator/core.py+++ b/packages/core/src/repowise/core/generation/page_generator/core.py@@ -32,6 +32,7 @@ from ..context.evidence import (select_prompt_evidence,)from ..context_assembler import ContextAssembler, FilePageContext+from ..house_vocabulary import cellfrom ..models import (MODEL_PAGE_CONFIDENCE,GeneratedPage,@@ -202,6 +203,11 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):self._jinja_env.filters.setdefault("oneline", oneline)self._jinja_env.filters.setdefault("as_markdown", as_markdown)self._jinja_env.filters.setdefault("signature", signature)+ # A pipe ends a table cell wherever it appears, and a deterministic+ # template interpolates the repository's own prose — which routinely+ # quotes a shell pipeline. Without this every column to the right of+ # one shifts.+ self._jinja_env.filters.setdefault("table_cell", cell)# ------------------------------------------------------------------# generate_all — orchestration (delegates to orchestrate.py)diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.pyindex d362ec46c..17051c561 100644--- a/packages/core/src/repowise/core/generation/page_generator/levels.py+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py@@ -294,9 +294,19 @@ async def _module_corroboration(run: _GenerationRun) -> list[str]:vanishes from the front page, and the store alone is one generation staleon a full run. Never raises — a store that cannot answer costs reach, nota page.++ Memoised on the run. Two levels want it — the overview at 6 and the+ glossary at 8 — and it costs a batched store read, so the second caller+ reuses the first's answer rather than paying it again. A run that emits+ neither page never reaches this."""+ cached = getattr(run, "_module_corroboration_corpus", None)+ if cached is not None:+ return cached+groups = run.sel_module_groupsif not groups:+ run._module_corroboration_corpus = []return []summaries: dict[str, str] = {}@@ -332,7 +342,9 @@ async def _module_corroboration(run: _GenerationRun) -> list[str]:from_this_run=sum(1 for mg in groups if run.completed_page_summaries.get(mg.key)),with_summary=len(summaries),)- return [f"{mg.display}\n{summaries.get(mg.key, '')}" for mg in groups]+ corpus = [f"{mg.display}\n{summaries.get(mg.key, '')}" for mg in groups]+ run._module_corroboration_corpus = corpus+ return corpusasync def build_level6_coros(run: _GenerationRun) -> list[tuple[str, Any]]:@@ -476,7 +488,7 @@ def _mine_house_terms(run: _GenerationRun) -> tuple[HouseTerm, ...]:return terms-def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:+async def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:"""Level 8 (curated onboarding collection)."""gen = run.gencoros: list[tuple[str, Any]] = []@@ -511,6 +523,15 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:kg_layers = tuple(run.kg_ctx.get_layers())kg_tour_steps = tuple(run.kg_ctx.get_tour())+ # The corpus costs a batched store read, so only a run emitting a subkind+ # that declared it wants one pays for it. Memoised, so a run that also+ # emits the overview builds it once for both.+ module_corroboration = (+ tuple(await _module_corroboration(run))+ if any(spec.needs_module_corroboration for _page_id, spec in emitted)+ else ()+ )+signals = _onboarding.OnboardingSignals(repo_name=run.repo_name,repo_structure=run.repo_structure,@@ -531,6 +552,7 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:tour_stops=tuple(run.tour_stops),layer_order=tuple(run.layer_order),house_terms=_mine_house_terms(run),+ module_corroboration=module_corroboration,)for page_id, spec in emitted:coros.append((page_id, gen.generate_onboarding_page(spec, signals)))diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex a84f8c5b0..0bc44299c 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -716,7 +716,7 @@ class _GenerationRun:final = (await _levels.build_level6_coros(self)+ _levels.build_level7_coros(self)- + _levels.build_level8_coros(self)+ + await _levels.build_level8_coros(self))final_pages = await self.run_level(final, 8)# Tag promoted onboarding slots (repo_overview / architecture_diagram).diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.pyindex c5be7354b..42fb40618 100644--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py@@ -523,10 +523,24 @@ class PerTypeGenerationMixin:target = _onboarding.target_path(spec.slot)references = spec.evidence_references(ctx) if spec.evidence_references else ()- if self._config.deterministic:+ if self._config.deterministic or spec.deterministic:# No grounding post-check: a template can only cite what the# context handed it, so there is nothing ungrounded to strip.- page = self._stub_onboarding_page(spec, ctx, target)+ #+ # ``spec.deterministic`` takes this path on every run, not only+ # under ``--no-prose``. The subkinds that set it are made of facts+ # the run already holds, and asking a model to restate those is how+ # a page that changed nothing comes back different — measured on+ # the overview at two calls one second apart.+ #+ # The two paths render the same template and make opposite claims+ # about the result: a ``--no-prose`` page is a page waiting for a+ # model, and a ``deterministic`` subkind's page is finished.+ page = (+ self._model_free_onboarding_page(spec, ctx, target)+ if spec.deterministic+ else self._stub_onboarding_page(spec, ctx, target)+ )evidence = self._disabled_source_evidence(page_key,"deterministic_generation",diff --git a/packages/core/src/repowise/core/generation/page_generator/structural.py b/packages/core/src/repowise/core/generation/page_generator/structural.pyindex 67876d110..51b2369c2 100644--- a/packages/core/src/repowise/core/generation/page_generator/structural.py+++ b/packages/core/src/repowise/core/generation/page_generator/structural.py@@ -575,6 +575,35 @@ class StructuralRenderMixin:page.metadata["onboarding_slot"] = spec.slotreturn page+ def _model_free_onboarding_page(self, spec: Any, ctx: Any, target_path: str) -> GeneratedPage:+ """Render a subkind that is finished without a model, not stubbed.++ The same template, and a different claim about it. A stub is real+ material with the prose missing, so it sits below the reader UI's+ banner threshold and the tree offers to have a model write it. A+ ``deterministic`` subkind is already everything its page is meant to+ be — every statement on it came from the index and no model saw it —+ which is exactly the claim :data:`TEMPLATE_PAGE_CONFIDENCE` makes.++ ``model_free`` is stamped for the consumers that would otherwise read+ ``provider_name='template'`` as "unwritten": scope resolution would+ offer this page to ``generate --unwritten`` forever, and the web reader+ would mark it "a model has not written this page yet" — on a page no+ model is ever going to write.+ """+ page = self._render_page(+ page_type="onboarding",+ target_path=target_path,+ title=spec.title,+ template=f"{_STUB_PREFIX}/onboarding/{spec.template}",+ ctx=ctx,+ slot=spec.slot,+ )+ page.metadata["subkind"] = spec.slot+ page.metadata["onboarding_slot"] = spec.slot+ page.metadata["model_free"] = True+ return page+def _rank_cycle_participants(ctx: Any) -> list[dict]:"""Order a cycle's files by how many of its edges they carry.diff --git a/packages/core/src/repowise/core/generation/scope.py b/packages/core/src/repowise/core/generation/scope.pyindex f64444255..8bd854b3b 100644--- a/packages/core/src/repowise/core/generation/scope.py+++ b/packages/core/src/repowise/core/generation/scope.py@@ -64,6 +64,13 @@ def load_page_records(pages: list[Any]) -> list[PageRecord]:A page is "template" (unwritten) when a model never touched it: thetemplate provider stamps ``provider_name='template'`` and``metadata.deterministic=True``. Either signal is enough.++ ``metadata.model_free`` overrides both. Those pages carry the same two+ signals and mean the opposite by them: their subkind is registered+ ``deterministic``, so no model is ever going to write one. Counting them as+ unwritten would offer them to ``generate --unwritten`` on every run, bill a+ cost estimate for prose that will not be written, and leave the reader UI's+ "bulk generate" affordance up on a wiki that is complete."""records: list[PageRecord] = []for p in pages:@@ -71,8 +78,8 @@ def load_page_records(pages: list[Any]) -> list[PageRecord]:meta = json.loads(getattr(p, "metadata_json", None) or "{}")except ValueError:meta = {}- is_template = getattr(p, "provider_name", "") == "template" or bool(- meta.get("deterministic")+ is_template = not meta.get("model_free") and (+ getattr(p, "provider_name", "") == "template" or bool(meta.get("deterministic")))records.append(PageRecord(diff --git a/packages/core/src/repowise/core/generation/templates/stub/onboarding/glossary.j2 b/packages/core/src/repowise/core/generation/templates/stub/onboarding/glossary.j2new file mode 100644index 000000000..bc9148a62--- /dev/null+++ b/packages/core/src/repowise/core/generation/templates/stub/onboarding/glossary.j2@@ -0,0 +1,51 @@+{#- The glossary. Deterministic on every run, not only under --no-prose.++ Every cell is mined: the term, the repository's own sentence about it, the+ parts of the system that name it, and the document the sentence came from.+ There is nothing here for a model to add that would not be an invention,+ and this is the page where an invented definition does the most damage —+ the reader consults it precisely because they cannot check the answer.++ The spec sets ``deterministic=True``, so the prompt template that every+ other subkind has does not exist for this one; this file is the page.+-#}+# Glossary++{{ ctx.entries | length }} term{{ "s" if ctx.entries | length != 1 else "" }} `{{ ctx.repo_name }}` uses for itself, read from its own documentation. Each definition is the repository's own sentence, quoted rather than written — a term the repository names without defining is left blank rather than filled in.++| Term | What the repository says it is | Where it is used | Written in |+|---|---|---|---|+{% for e in ctx.entries -%}+{#- oneline before table_cell: truncating after escaping can cut a ``\|`` in+ half and leave the backslash orphaned at the end of the cell. -#}+| {% if e.is_indexed_symbol %}`{{ e.term | table_cell }}`{% else %}**{{ e.term | table_cell }}**{% endif %} | {% if e.definition %}{{ e.definition | oneline(220) | table_cell }}{% else %}—{% endif %} | {% if e.used_in %}{{ e.used_in | map("table_cell") | join(" · ") }}{% else %}—{% endif %} | `{{ e.source_path | table_cell }}` |+{% endfor %}++## Where these came from++{{ ctx.defined }} of {{ ctx.entries | length }} term{{ "s" if ctx.entries | length != 1 else "" }} carry a definition the repository wrote. They were read from:++{% for path in ctx.sources -%}+- `{{ path }}`+{% endfor %}++{% if ctx.mined > ctx.corroborated %}+{{ ctx.mined }} candidate terms were mined in all, and {{ ctx.corroborated }} survived. A term reaches this page only when the codebase's own structure names it too — the parts of the system in the third column are grouped from the dependency graph, with no knowledge of the documents — so vocabulary the documentation uses without the code confirming it is left off.+{% endif %}+{% if ctx.corroborated > ctx.entries | length %}+The {{ ctx.entries | length }} most corroborated are listed; {{ ctx.corroborated - ctx.entries | length }} more were left off for length.+{% endif %}+{#- Not ``stub/_footer.j2``: that footer offers to write the page properly once+ an API key is present, and this page is already what it is meant to be.++ The blank line below is written as an explicit newline because it is+ load-bearing and whitespace control keeps eating it. ``paragraph``+ immediately followed by ``---`` is a setext H2 in CommonMark, not a+ thematic break, so without it the closing note typesets the paragraph+ above it as a section heading. -#}+{{ "\n" }}+---++*Every term, definition and path on this page is quoted from the repository's+own documents. Nothing here is written by a model, so two builds of an+unchanged repository produce this page byte for byte.*diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.pyindex f4b982979..c073c1588 100644--- a/packages/core/src/repowise/core/pipeline/phases/generation.py+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py@@ -167,12 +167,20 @@ async def run_generation(}if progress:if onboarding_generated or promoted_present:+ # Imported here, not at module scope: importing the onboarding+ # package registers every subkind, and this module is on the CLI's+ # import path where that cost buys nothing.+ from repowise.core.generation.onboarding.slots import ONBOARDING_ORDER+slots_made = sorted({p.metadata.get("subkind", "?") for p in onboarding_generated} | promoted_present)progress.on_message("info",- f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",+ # Derived rather than written down: the denominator was a+ # literal 8 and went stale the first time a slot was added.+ f"Onboarding: {len(slots_made)}/{len(ONBOARDING_ORDER)} slots"+ f" — {', '.join(slots_made)}",)# Placeholders left behind by a failed provider call are in this list# like any other page, and the run reports them as failures a linediff --git a/packages/ui/src/lib/page-types.ts b/packages/ui/src/lib/page-types.tsindex 6826d8043..dd3df7d08 100644--- a/packages/ui/src/lib/page-types.ts+++ b/packages/ui/src/lib/page-types.ts@@ -69,11 +69,23 @@ export function isModelWrittenType(pageType: string | null | undefined): boolean/** True when a model-written page is still a structural stub (no prose yet).* Scoped to the model-written types: a stub carries `provider_name ===* "template"`, a written page a real provider. Returns false for every- * structural page type, which is never a stub in this sense. */+ * structural page type, which is never a stub in this sense.+ *+ * `metadata.model_free` marks a page whose subkind is rendered without a+ * model by design — the glossary quotes mined definitions and has no prompt+ * at all. It carries `provider_name === "template"` like a stub and means the+ * opposite by it: the page is finished, and no model is ever going to write+ * it. Without this check the tree marks it "a model has not written this page+ * yet", offers a regenerate button that cannot help, and keeps the bulk+ * generate affordance up on a complete wiki. */export function isStubPage(- page: { page_type?: string; provider_name?: string } | null | undefined,+ page:+ | { page_type?: string; provider_name?: string; metadata?: Record<string, unknown> }+ | null+ | undefined,): boolean {if (!page || !isModelWrittenType(page.page_type)) return false;+ if (page.metadata?.["model_free"]) return false;return page.provider_name === "template";}@@ -83,10 +95,14 @@ export function isStubPage(//// One slot — project_overview — is *promoted*: its// content lives in the existing repo_overview page, tagged via-// `metadata.onboarding_slot`. The other seven are dedicated+// `metadata.onboarding_slot`. The other eight are dedicated// `page_type === "onboarding"` pages with `metadata.subkind` discriminating// them.//+// A slot missing from this map is not merely unlabelled: `isOnboardingSlot`+// gates on it, so `getOnboardingSlot` returns null and the page drops out of+// the Onboarding folder entirely. Add every new slot here.+//// This map is display text only. The *reading order* used to be duplicated// here as an ONBOARDING_ORDER array kept in lockstep with `slots.py` by// comment alone; it now arrives on the pages themselves as `display_order`,@@ -102,6 +118,7 @@ export const ONBOARDING_SLOT_TITLES = {how_it_works: "How It Works",development_guide: "Development Guide",active_landscape: "Active Landscape",+ glossary: "Glossary",} as const;export type OnboardingSlot = keyof typeof ONBOARDING_SLOT_TITLES;diff --git a/tests/unit/generation/test_onboarding.py b/tests/unit/generation/test_onboarding.pyindex 85aeaf66e..0c825cddc 100644--- a/tests/unit/generation/test_onboarding.py+++ b/tests/unit/generation/test_onboarding.py@@ -168,9 +168,13 @@ def test_subkinds_registered_in_canonical_order() -> None:# ONBOARDING_ORDER.expected = [s for s in ONBOARDING_ORDER if s not in PROMOTED_SLOTS.values()]assert slots == expected- # Six templated subkinds + the topology-driven guided tour.- assert len(slots) == 7+ # Six templated subkinds, the topology-driven guided tour, and the+ # glossary, which is rendered without a model at all.+ assert len(slots) == 8assert "guided_tour" in slots+ # Last, and it must stay last: a glossary is a lookup surface, not a+ # reading step.+ assert slots[-1] == "glossary"def test_onboarding_level_is_eight() -> None:diff --git a/tests/unit/generation/test_onboarding_glossary.py b/tests/unit/generation/test_onboarding_glossary.pynew file mode 100644index 000000000..55a41c431--- /dev/null+++ b/tests/unit/generation/test_onboarding_glossary.py@@ -0,0 +1,503 @@+"""The glossary page: the words a repository uses for itself, defined by it.++Every assertion here is about the same property. A glossary is the page a+reader consults *because* they do not know the answer, so they cannot catch a+wrong one. Nothing on it may be written — the term, the sentence and the path+are all quoted, and a term the repository never defined renders blank rather+than plausibly.+"""++from __future__ import annotations++import jinja2+import pytest+from structlog.testing import capture_logs++from repowise.core.generation.concept_tree.vocabulary import HouseTerm+from repowise.core.generation.house_vocabulary import cell+from repowise.core.generation.onboarding import get_spec+from repowise.core.generation.onboarding.signals import OnboardingSignals+from repowise.core.generation.onboarding.slots import SLOT_GLOSSARY+from repowise.core.generation.onboarding.subkinds.glossary import _build+from repowise.core.generation.page_generator.structural import oneline+from repowise.core.ingestion.models import RepoStructure++#: What the structural side calls the parts of the system. Cut from the+#: dependency graph and named from the code, with no knowledge of the+#: documents — which is what makes a match between the two mean something.+MODULES = [+ "Dead Code and Reachability Analysis\nFinds files no import path reaches.",+ "Blast Radius UI\nRenders the blast radius of a pending change.",+ "Change Risk Analysis\nScores a diff before it merges.",+ "Ingestion Pipeline\nWalks the tree and parses each file.",+ "Knowledge Graph Routes\nServes the knowledge graph to the web reader.",+ "Split File Refactoring\nProposes a split for an overlong file.",+]+++def _term(+ term: str,+ *,+ definition: str | None = None,+ definition_source: str | None = None,+ source_paths: tuple[str, ...] = ("README.md",),+ doc_frequency: int = 1,+ is_indexed_symbol: bool = False,+) -> HouseTerm:+ return HouseTerm(+ term=term,+ definition=definition,+ definition_source=definition_source,+ source_paths=source_paths,+ doc_frequency=doc_frequency,+ code_frequency=5,+ is_indexed_symbol=is_indexed_symbol,+ )+++#: Six terms every one of ``MODULES`` corroborates, enough to clear the gate.+SIX_TERMS = [+ _term("Dead code", definition="Dead code is code no import path reaches."),+ _term("Blast radius", definition="Blast radius is what a change can reach."),+ _term("Change risk", definition="Change risk scores a diff before it merges."),+ _term("Ingestion pipeline", definition="Ingestion pipeline walks and parses the tree."),+ _term("Knowledge graph", definition="Knowledge graph holds the system's entities."),+ _term("Split file", definition="Split file proposes a split for a long file."),+]+++def _signals(terms, modules=MODULES) -> OnboardingSignals:+ return OnboardingSignals(+ repo_name="ledger",+ repo_structure=RepoStructure(+ is_monorepo=False,+ packages=[],+ root_language_distribution={"python": 1.0},+ total_files=1,+ total_loc=10,+ entry_points=[],+ ),+ parsed_files=(),+ source_map={},+ graph_builder=None,+ pagerank={},+ betweenness={},+ community={},+ sccs=(),+ house_terms=tuple(terms),+ module_corroboration=tuple(modules),+ )+++# ---------------------------------------------------------------------------+# The gate+# ---------------------------------------------------------------------------+++def test_a_repository_with_enough_vocabulary_gets_a_page():+ ctx = _build(_signals(SIX_TERMS))+ assert ctx is not None+ assert len(ctx.entries) == 6+++def test_four_terms_is_not_a_glossary():+ """Below the floor the page is a handful of rows pretending to be a+ vocabulary, so it does not build at all."""+ assert _build(_signals(SIX_TERMS[:4])) is None+++def test_the_skip_is_logged_with_the_counts_that_explain_it():+ """A page that stops appearing must say why. The two counts separate+ "nothing was mined" from "nothing the structure confirmed"."""+ with capture_logs() as logs:+ _build(_signals(SIX_TERMS[:4]))+ event = next(e for e in logs if e["event"] == "onboarding.glossary_gate_skipped")+ assert event["mined"] == 4+ assert event["corroborated"] == 4+ assert event["required"] == 5+++def test_a_repository_that_mined_nothing_gets_no_page():+ assert _build(_signals([])) is None+++def test_terms_no_module_group_corroborates_do_not_reach_the_page():+ """The corroboration is the whole selection rule. Vocabulary the documents+ use and the code never confirms is marketing, not a glossary entry."""+ assert _build(_signals(SIX_TERMS, modules=["Something Else\nUnrelated."])) is None+++# ---------------------------------------------------------------------------+# What a row may say+# ---------------------------------------------------------------------------+++def test_a_term_the_repository_never_defined_is_left_blank():+ """The one thing this page refuses to do. An invented definition on a+ glossary is unfalsifiable by the reader who came to look it up."""+ terms = [*SIX_TERMS[:5], _term("Split file")]+ ctx = _build(_signals(terms))+ entry = next(e for e in ctx.entries if e.term == "Split file")+ assert entry.definition is None+ assert entry.source_path == "README.md"+++def test_a_definition_that_never_names_its_term_is_still_taken():+ """A heading gloss does not restate its own heading, and it is the+ commonest definition shape there is.++ Requiring the term to appear was built and measured and dropped. It caught+ two junk rows on this repository — "Related" glossed as "The performance+ pillar has its own, separate benchmark" — and it deleted "## Blast radius"+ over "The set of files a change can reach", along with every definition+ mined from a bolded lead-in, which captures only the text after the dash.+ A repository that documents in that style would have lost its whole+ definition column, on the front page as well as here.+ """+ terms = [+ *SIX_TERMS[:5],+ _term("Split file", definition="The file-level analog of Extract Class."),+ ]+ ctx = _build(_signals(terms))+ assert next(e for e in ctx.entries if e.term == "Split file").definition == (+ "The file-level analog of Extract Class."+ )+++def test_a_command_line_is_not_a_definition():+ terms = [+ *SIX_TERMS[:5],+ _term("Split file", definition="repowise split-file --json # list them"),+ ]+ ctx = _build(_signals(terms))+ assert next(e for e in ctx.entries if e.term == "Split file").definition is None+++def test_an_undefined_term_still_earns_its_row():+ """Corroboration is the only test a term has to pass.++ A row whose middle column is an em dash still carries two facts a reader+ came for: the repository has a word for this, and these are the parts of+ the system that use it. Requiring a definition was measured and reverted —+ on this repository it removed "Workspace", "Coupling", "CLI", "Distill",+ "Costs" and "Decisions", and on django it removed "Security", to remove+ two weak rows. A lookup page is judged on coverage.+ """+ ctx = _build(_signals([*SIX_TERMS, _term("Ingestion")]))+ listed = {e.term for e in ctx.entries}+ assert "Ingestion" in listed+ assert next(e for e in ctx.entries if e.term == "Ingestion").definition is None+++def test_every_row_cites_a_real_path():+ ctx = _build(_signals(SIX_TERMS))+ assert all(e.source_path for e in ctx.entries)+++def test_the_source_is_the_document_the_sentence_came_from():+ terms = [+ *SIX_TERMS[:5],+ _term(+ "Split file",+ definition="Split file proposes a split.",+ definition_source="docs/refactoring.md",+ source_paths=("README.md",),+ ),+ ]+ ctx = _build(_signals(terms))+ assert next(e for e in ctx.entries if e.term == "Split file").source_path == (+ "docs/refactoring.md"+ )+++def test_a_row_names_the_parts_of_the_system_that_use_it():+ ctx = _build(_signals(SIX_TERMS))+ entry = next(e for e in ctx.entries if e.term == "Dead code")+ assert "Dead Code and Reachability Analysis" in entry.used_in+++# ---------------------------------------------------------------------------+# Stability+# ---------------------------------------------------------------------------+++def test_rows_are_alphabetical():+ """A glossary is looked up rather than read. Ranking decided which terms+ are here; it has no job left once they are."""+ ctx = _build(_signals(SIX_TERMS))+ terms = [e.term for e in ctx.entries]+ assert terms == sorted(terms, key=str.lower)+++def test_two_builds_of_one_repository_agree():+ """An unstable glossary churns ``source_hash`` on every index, which+ re-embeds and re-persists a page that did not change."""+ assert _build(_signals(SIX_TERMS)) == _build(_signals(SIX_TERMS))+++def test_the_order_of_the_corroboration_corpus_does_not_change_the_page():+ assert _build(_signals(SIX_TERMS)) == _build(_signals(SIX_TERMS, modules=MODULES[::-1]))+++# ---------------------------------------------------------------------------+# The page+# ---------------------------------------------------------------------------+++def render():+ from pathlib import Path++ import repowise.core.generation as generation++ env = jinja2.Environment(+ loader=jinja2.FileSystemLoader(str(Path(generation.__file__).parent / "templates")),+ undefined=jinja2.StrictUndefined,+ autoescape=False,+ )+ env.filters["oneline"] = oneline+ env.filters["table_cell"] = cell++ def _render(ctx):+ return env.get_template("stub/onboarding/glossary.j2").render(+ ctx=ctx, slot=SLOT_GLOSSARY+ )++ return _render+++def test_the_page_renders_one_row_per_term(render):+ page = render(_build(_signals(SIX_TERMS)))+ rows = [line for line in page.splitlines() if line.startswith("| ")]+ # Six terms plus the header row. The alignment row opens ``|-``.+ assert len(rows) == 7+ assert "{{" not in page+++def test_an_undefined_term_renders_an_em_dash_not_a_sentence(render):+ page = render(_build(_signals([*SIX_TERMS[:5], _term("Split file")])))+ row = next(line for line in page.splitlines() if line.startswith("| **Split file**"))+ assert "| — |" in row+++def test_a_pipe_in_mined_prose_does_not_break_the_table(render):+ """Definitions are the repository's text, not ours. One quoting a shell+ pipeline would otherwise shift every column to its right."""+ terms = [+ *SIX_TERMS[:5],+ _term("Split file", definition="Split file findings pipe | into the report."),+ ]+ page = render(_build(_signals(terms)))+ row = next(line for line in page.splitlines() if line.startswith("| **Split file**"))+ assert row.count("|") - row.count("\\|") == 5+++def test_a_term_the_codebase_defines_is_backticked_and_a_coined_one_is_not(render):+ """``check_grounding`` strips backticks off any token it cannot resolve to+ a symbol, so backticking a coined term is a silent failure waiting to+ happen. This page is deterministic and never passes through that check —+ which is exactly why it has to get the distinction right itself."""+ terms = [+ *SIX_TERMS[:5],+ _term("SplitFile", definition="SplitFile proposes a split.", is_indexed_symbol=True),+ ]+ modules = [*MODULES, "Split File Refactoring\nThe SplitFile detector runs here."]+ page = render(_build(_signals(terms, modules=modules)))+ assert "| `SplitFile` |" in page+ assert "| **Dead code** |" in page+++def test_the_page_says_how_much_of_its_vocabulary_it_could_define(render):+ """A glossary covering a third of the terms it lists should say so rather+ than reading as though the repository defined everything."""+ page = render(_build(_signals([*SIX_TERMS[:5], _term("Split file")])))+ assert "5 of 6 terms carry a definition" in page+++# ---------------------------------------------------------------------------+# Wiring+# ---------------------------------------------------------------------------+++def test_the_glossary_is_registered_and_needs_no_model():+ """Enumerable facts written by a model are resampled on every render. The+ whole page is enumerable facts, so no model is in its path."""+ spec = get_spec(SLOT_GLOSSARY)+ assert spec is not None+ assert spec.deterministic is True+ assert spec.needs_module_corroboration is True+ assert spec.title == "Glossary"+++def test_every_subkind_has_the_templates_its_flags_promise():+ """The flag and the files have to agree, and nothing else checks that.++ A ``deterministic`` subkind renders only its ``stub/`` template, so it is+ allowed to ship without a prompt. Every other subkind needs both, and+ losing either is a ``TemplateNotFound`` at generation time on a repository+ that is not ours — the suite is where that has to fail instead.+ """+ from pathlib import Path++ import repowise.core.generation as generation+ from repowise.core.generation.onboarding import iter_specs++ templates = Path(generation.__file__).parent / "templates"+ for spec in iter_specs():+ stub = templates / "stub" / "onboarding" / spec.template+ assert stub.is_file(), f"{spec.slot} has no stub template at {stub}"+ prompt = templates / "onboarding" / spec.template+ if spec.deterministic:+ assert not prompt.is_file(), (+ f"{spec.slot} is deterministic, so its prompt template at "+ f"{prompt} is dead weight that no run can reach"+ )+ else:+ assert prompt.is_file(), f"{spec.slot} has no prompt template at {prompt}"+++def test_a_deterministic_subkind_reaches_the_no_provider_path():+ """The flag has to be read, not merely set.++ ``generate_onboarding_page`` branches on it, and the glossary has no prompt+ template at all — so if that branch is ever dropped the page raises+ ``TemplateNotFound`` in production while every assertion about the spec+ still passes.+ """+ import inspect++ from repowise.core.generation.page_generator import pertype++ source = inspect.getsource(pertype.PerTypeGenerationMixin.generate_onboarding_page)+ assert "spec.deterministic" in source+ assert "_model_free_onboarding_page" in source+++# ---------------------------------------------------------------------------+# Honesty about what is not on the page+# ---------------------------------------------------------------------------+++#: Word-only stems, because a term's words are what the corroboration pattern+#: matches — a digit in the name matches no module group and corroborates+#: nothing.+_STEMS = [+ "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel",+ "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa",+ "quebec", "romeo", "sierra", "tango", "uniform", "victor", "whiskey",+ "xray", "yankee", "zulu", "amber", "bronze", "copper", "diamond",+ "emerald", "flint", "garnet", "ivory", "jade", "kevlar", "lapis",+ "marble", "nickel", "onyx", "pearl", "quartz", "ruby", "slate", "topaz",+ "umber", "violet", "willow", "xenon", "yarrow", "zircon", "almond",+ "birch", "cedar", "dogwood",+]+++def _many(count: int):+ """*count* distinct two-word terms, and a module group naming each."""+ stems = _STEMS[:count]+ assert len(stems) == count, "not enough distinct stems for this test"+ terms = [+ _term(f"{s.title()} code", definition=f"{s.title()} code is a rule nothing reaches.")+ for s in stems+ ]+ modules = [f"{s.title()} Code Analysis\nFinds {s} code." for s in stems]+ return terms, modules+++def test_a_long_vocabulary_is_capped_and_says_so(render):+ """The footer explains that a term reaches this page only with structural+ corroboration. That is a false account of a term dropped for length, so the+ count before the cap is carried onto the page too."""+ terms, modules = _many(55)+ ctx = _build(_signals(terms, modules=modules))+ assert ctx.corroborated == 55+ assert len(ctx.entries) == 40++ page = render(ctx)+ assert "The 40 most corroborated are listed; 15 more were left off" in page+ # Nothing failed corroboration here, so the page does not claim anything did.+ assert "candidate terms were mined in all" not in page+++def test_a_page_listing_everything_claims_no_truncation(render):+ page = render(_build(_signals(SIX_TERMS)))+ assert "left off for length" not in page+++def test_a_row_says_when_its_subsystem_list_is_cut(render):+ """"Where it is used" is the column a reader acts on, and a truncated list+ reads as the whole list."""+ modules = [+ *MODULES,+ "Dead Code Sweeper\nRuns the dead code pass nightly.",+ "Dead Code Report\nRenders the dead code findings.",+ "Dead Code API\nServes dead code to the web reader.",+ ]+ ctx = _build(_signals(SIX_TERMS, modules=modules))+ entry = next(e for e in ctx.entries if e.term == "Dead code")+ assert entry.used_in[-1] == "and 1 more"+ assert "and 1 more" in render(ctx)+++# ---------------------------------------------------------------------------+# The page is finished, not a stub+# ---------------------------------------------------------------------------+++def test_the_rendered_page_does_not_open_a_setext_heading(render):+ """``paragraph`` followed by ``---`` is a setext H2 in CommonMark, not a+ thematic break. Without a blank line between them the closing note typesets+ the paragraph above it as a section heading."""+ long_terms, long_modules = _many(55)+ contexts = (_build(_signals(SIX_TERMS)), _build(_signals(long_terms, modules=long_modules)))+ for ctx in contexts:+ page = render(ctx)+ rule = page.index("\n---\n")+ assert page[rule - 1] == "\n", "the thematic break has no blank line above it"+++def test_two_renders_of_one_repository_are_byte_identical(render):+ """The page's own closing claim, asserted on the bytes rather than on the+ context object."""+ first = render(_build(_signals(SIX_TERMS)))+ second = render(_build(_signals(SIX_TERMS)))+ assert first == second+++def test_a_model_free_page_is_not_counted_as_unwritten():+ """It carries ``provider_name='template'`` like a stub and means the+ opposite by it.++ Counted as unwritten, the glossary is offered to ``generate --unwritten``+ on every run, billed in the cost estimate for prose that will not be+ written, and leaves the reader UI's bulk-generate affordance up on a wiki+ that is complete.+ """+ import json+ from types import SimpleNamespace++ from repowise.core.generation.scope import load_page_records++ def _page(page_id, meta):+ return SimpleNamespace(+ id=page_id,+ page_type="onboarding",+ target_path=f"onboarding/{page_id}",+ provider_name="template",+ metadata_json=json.dumps(meta),+ freshness_status="fresh",+ )++ records = {+ r.page_id: r+ for r in load_page_records(+ [+ _page("glossary", {"subkind": "glossary", "model_free": True}),+ _page("key_concepts", {"subkind": "key_concepts"}),+ ]+ )+ }+ assert records["glossary"].is_template is False+ # An ordinary subkind on a stub is still unwritten, and still offered.+ assert records["key_concepts"].is_template is Truediff --git a/tests/unit/generation/test_onboarding_house_terms.py b/tests/unit/generation/test_onboarding_house_terms.pyindex f49e68b19..fde46b9b2 100644--- a/tests/unit/generation/test_onboarding_house_terms.py+++ b/tests/unit/generation/test_onboarding_house_terms.py@@ -155,6 +155,11 @@ def _run(*, repo_path: Path | None) -> SimpleNamespace:decisions_all=(),external_systems=(),completed_page_summaries={},+ # The glossary corroborates mined terms against the module groups, so+ # level 8 now reads these too. Empty here: this module is about the+ # mining, and the glossary's own tests cover what corroboration does.+ sel_module_groups=[],+ vector_store=None,tour_stops=(),layer_order=(),kg_ctx=None,@@ -163,8 +168,8 @@ def _run(*, repo_path: Path | None) -> SimpleNamespace:)-def _signals_from_level8(run: SimpleNamespace) -> Any:- coros = build_level8_coros(run)+async def _signals_from_level8(run: SimpleNamespace) -> Any:+ coros = await build_level8_coros(run)assert coros, "level 8 produced no pages, so no signals were built"assert run.gen.seen, "no subkind was handed the signals"return run.gen.seen[0]@@ -175,27 +180,27 @@ def _signals_from_level8(run: SimpleNamespace) -> Any:# ----------------------------------------------------------------------------def test_documented_repository_populates_the_field(tmp_path: Path) -> None:+async def test_documented_repository_populates_the_field(tmp_path: Path) -> None:reset_house_terms()- signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))+ signals = await _signals_from_level8(_run(repo_path=_repo(tmp_path)))terms = {t.term for t in signals.house_terms}assert "Blast radius" in termsassert "Change risk" in terms-def test_every_subkind_reads_the_same_mined_terms(tmp_path: Path) -> None:+async def test_every_subkind_reads_the_same_mined_terms(tmp_path: Path) -> None:"""One walk of the repository per run, not one per slot."""reset_house_terms()run = _run(repo_path=_repo(tmp_path))- build_level8_coros(run)+ await build_level8_coros(run)assert len(run.gen.seen) > 1, "expected more than one onboarding slot"first = run.gen.seen[0].house_termsassert all(s.house_terms is first for s in run.gen.seen)-def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(+async def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(tmp_path: Path,) -> None:"""Mining walks the whole repository, so a run writing none must not pay.@@ -207,41 +212,41 @@ def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(run = _run(repo_path=_repo(tmp_path))run._emit = lambda page_id: False- assert build_level8_coros(run) == []+ assert await build_level8_coros(run) == []assert run.gen.seen == []assert house_terms_mined().mined is False-def test_a_term_the_codebase_defines_is_marked_as_a_symbol(tmp_path: Path) -> None:+async def test_a_term_the_codebase_defines_is_marked_as_a_symbol(tmp_path: Path) -> None:"""``is_indexed_symbol`` decides whether a term may be backticked.Without the run's symbol names it is ``False`` for every term, which readsdownstream as "this repository defines none of its own vocabulary"."""reset_house_terms()- signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))+ signals = await _signals_from_level8(_run(repo_path=_repo(tmp_path)))by_term = {t.term: t for t in signals.house_terms}assert by_term["Blast radius"].is_indexed_symbol is Falseassert any(t.is_indexed_symbol for t in signals.house_terms) is False-def test_undocumented_repository_yields_nothing_and_says_so(tmp_path: Path) -> None:+async def test_undocumented_repository_yields_nothing_and_says_so(tmp_path: Path) -> None:reset_house_terms()run = _run(repo_path=_repo(tmp_path, documented=False))with capture_logs() as logs:- signals = _signals_from_level8(run)+ signals = await _signals_from_level8(run)assert signals.house_terms == ()assert any(entry["event"] == "onboarding.house_terms_empty" for entry in logs)assert house_terms_mined().mined is True-def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:+async def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:reset_house_terms()run = _run(repo_path=None)with capture_logs() as logs:- signals = _signals_from_level8(run)+ signals = await _signals_from_level8(run)assert signals.house_terms == ()skipped = [e for e in logs if e["event"] == "onboarding.house_terms_skipped"]@@ -255,9 +260,9 @@ def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:# ----------------------------------------------------------------------------def test_the_count_and_the_top_terms_reach_the_report(tmp_path: Path) -> None:+async def test_the_count_and_the_top_terms_reach_the_report(tmp_path: Path) -> None:reset_house_terms()- signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))+ signals = await _signals_from_level8(_run(repo_path=_repo(tmp_path)))report = GenerationReport.from_pages([])assert report.house_terms.mined is True@@ -280,10 +285,10 @@ def test_the_report_keeps_nothing_read_apart_from_nothing_found() -> None:assert "not measured" not in empty.summary_line()-def test_a_new_run_does_not_report_the_previous_run_s_vocabulary(tmp_path: Path) -> None:+async def test_a_new_run_does_not_report_the_previous_run_s_vocabulary(tmp_path: Path) -> None:"""The reset lives on ``generate_all``; this pins what it is there for."""reset_house_terms()- _signals_from_level8(_run(repo_path=_repo(tmp_path)))+ await _signals_from_level8(_run(repo_path=_repo(tmp_path)))assert house_terms_mined().count > 0reset_house_terms()diff --git a/tests/unit/generation/test_overview_capability_table.py b/tests/unit/generation/test_overview_capability_table.pyindex d425653c5..6e4744c87 100644--- a/tests/unit/generation/test_overview_capability_table.py+++ b/tests/unit/generation/test_overview_capability_table.py@@ -122,7 +122,7 @@ def test_a_term_with_no_path_anywhere_is_not_offered():def test_selection_is_logged_with_what_it_kept_and_what_it_dropped():with capture_logs() as logs:select_capabilities([_term("DONE"), _term("Dead code")], MODULES)- event = next(e for e in logs if e["event"] == "overview_capabilities_selected")+ event = next(e for e in logs if e["event"] == "house_vocabulary.selected")assert event["mined"] == 2assert event["corroborated"] == 1assert event["terms"] == ["Dead code"]@@ -134,7 +134,7 @@ def test_mining_terms_that_no_module_page_corroborates_is_a_warning():corroboration corpus looks like, so the counts that separate them go out."""with capture_logs() as logs:select_capabilities([_term("DONE")], MODULES)- assert any(e["event"] == "overview_capabilities_uncorroborated" for e in logs)+ assert any(e["event"] == "house_vocabulary.uncorroborated" for e in logs)# ---------------------------------------------------------------------------@@ -182,20 +182,28 @@ def test_prose_that_is_not_a_statement_is_not_offered_as_a_definition(text):@pytest.mark.parametrize(- "text",+ "term,text",[- "Dead code is code no import path reaches.",- "Blast radius is the set of files a change can reach through the import graph.",- "Middleware for utilizing Web-server-provided authentication.",+ ("Dead code", "Dead code is code no import path reaches."),+ (+ "Blast radius",+ "Blast radius is the set of files a change can reach through the import graph.",+ ),+ ("Middleware", "Middleware for utilizing Web-server-provided authentication."),# Terse, and still the repository's own answer. Both of these were# rejected by a first cut of the test that was tuned too strict.- "Blast-radius request/response models.",- "Decisions are co-located in the page vector store under the "- "``decision:<record_id>`` namespace (no separate table).",+ ("Blast radius", "Blast-radius request/response models."),+ (+ "Decisions",+ "Decisions are co-located in the page vector store under the "+ "``decision:<record_id>`` namespace (no separate table).",+ ),],)-def test_a_real_sentence_survives(text):- picked = select_capabilities([_term("Dead code", definition=text)], MODULES)+def test_a_real_sentence_survives(term, text):+ """Each sentence is paired with the term it names, because a definition+ that never mentions its term is rejected as prose that merely sat nearby."""+ picked = select_capabilities([_term(term, definition=text)], [*MODULES, term])assert picked[0].definition == text@@ -220,7 +228,7 @@ def test_a_rejected_definition_does_not_leave_its_source_behind():def test_a_rejected_definition_is_logged():with capture_logs() as logs:select_capabilities([_term("Dead code", definition="$ repowise dead-code")], MODULES)- assert any(e["event"] == "overview_capability_definition_rejected" for e in logs)+ assert any(e["event"] == "house_vocabulary.definition_rejected" for e in logs)def test_a_term_the_repository_never_defined_still_gets_a_row():@@ -252,8 +260,9 @@ def test_a_pipe_in_mined_prose_does_not_break_the_table():def test_a_long_definition_is_cut_rather_than_wrapping_the_row():+ long_sentence = "Dead code is " + "unreachable " * 60 + "code."table = build_capability_table(- select_capabilities([_term("Dead code", definition="word " * 100)], MODULES)+ select_capabilities([_term("Dead code", definition=long_sentence)], MODULES))row = next(line for line in table.splitlines() if line.startswith("| Dead code"))assert "…" in rowdiff --git a/tests/unit/generation/test_overview_capability_wiring.py b/tests/unit/generation/test_overview_capability_wiring.pyindex e04baf99f..e1852a75b 100644--- a/tests/unit/generation/test_overview_capability_wiring.py+++ b/tests/unit/generation/test_overview_capability_wiring.py@@ -292,7 +292,7 @@ async def test_the_repository_is_mined_once_for_both_levels(tmp_path):"""Mining walks the whole repository. Two consumers, one walk."""run = _run(tmp_path)await build_level6_coros(run)- build_level8_coros(run)+ await build_level8_coros(run)from_overview = tuple(c.term for c in run.gen.overview_kwargs["capabilities"])onboarding_terms = run.gen.onboarding_signals[0].house_terms@@ -311,9 +311,54 @@ async def test_onboarding_still_gets_its_terms_when_the_overview_mined_first(tmpfrom a warm cache rather than from a fresh walk."""run = _run(tmp_path)await build_level6_coros(run)- build_level8_coros(run)+ await build_level8_coros(run)assert [t.term for t in run.gen.onboarding_signals[0].house_terms] == ["Blast radius","Change risk",]+++# ---------------------------------------------------------------------------+# The glossary reads the same corroboration corpus+# ---------------------------------------------------------------------------+#+# It is the second consumer of both mined vocabulary and module corroboration.+# Both cost a pass the run should pay once: mining walks the whole repository,+# and corroboration reads module summaries out of the store.+++async def test_the_glossary_is_handed_the_corroboration_corpus(tmp_path):+ run = _run(tmp_path)+ await build_level8_coros(run)++ corpus = run.gen.onboarding_signals[0].module_corroboration+ assert corpus+ # Title first, summary under it -- what makes a match mean something.+ assert corpus[0].startswith("Blast Radius Evaluation")+++async def test_the_corroboration_corpus_is_built_once_for_both_levels(tmp_path):+ """Level 6 and level 8 both want it, and it costs a batched store read."""+ store = _Store({"src": "Evaluates the blast radius of a change."})+ run = _run(tmp_path, written={}, store=store)++ await build_level6_coros(run)+ asked_after_overview = list(store.asked)+ await build_level8_coros(run)++ assert asked_after_overview, "the overview should have read the store"+ assert store.asked == asked_after_overview, "level 8 read the store a second time"+++async def test_a_run_with_no_glossary_page_does_not_build_the_corpus(tmp_path):+ """A scoped run that asked for one onboarding page should not pay for a+ store read that only the glossary consumes."""+ store = _Store({"src": "Evaluates the blast radius of a change."})+ run = _run(tmp_path, store=store)+ run._emit = lambda page_id: "glossary" not in page_id++ await build_level8_coros(run)++ assert store.asked == []+ assert all(s.module_corroboration == () for s in run.gen.onboarding_signals)diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex dbb131268..0246f93aa 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -825,3 +825,38 @@ def test_extract_terms_does_not_read_txt_documents(txt_docs_repo: Path) -> None:"""The planner's input does not move. It mines markdown only, and thebinding it performs is measured against exactly that corpus."""assert extract_terms(txt_docs_repo) == []+++def test_a_hard_wrapped_markdown_sentence_is_read_whole(tmp_path: Path) -> None:+ """A README wraps at the column as readily as a ``.txt`` does.++ The reStructuredText scan has rejoined wrapped lines since it learned to;+ the markdown scan read one line and returned the author's sentence cut+ where their editor wrapped it. That fragment then trails off mid-thought,+ which every consumer renders as prose the repository wrote.+ """+ root = tmp_path / "ledger"+ (root / "src").mkdir(parents=True)+ (root / "README.md").write_text(+ "# Ledger\n"+ "\n"+ "## Blast radius\n"+ "\n"+ "Blast radius is the set of accounts a posting can reach through the\n"+ "ledger graph.\n"+ "\n"+ "## Dead code\n"+ "\n"+ "Dead code is a rule no posting path reaches.\n",+ encoding="utf-8",+ )+ (root / "src" / "engine.py").write_text(+ '"""Blast radius and dead code for the ledger."""\n', encoding="utf-8"+ )++ by_term = {t.term: t for t in extract_house_terms(root)}+ assert by_term["Blast radius"].definition == (+ "Blast radius is the set of accounts a posting can reach through the ledger graph."+ )+ # A sentence that already fits on one line is unaffected.+ assert by_term["Dead code"].definition == "Dead code is a rule no posting path reaches."