Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): a glossary page, written entirely in the repository's own words (#1276)

Raghav Chamadiya096dfe02cda7
diff --git a/docs/reference/CONFIG.md b/docs/reference/CONFIG.md
index 0d0f9e082..566bff9ee 100644
--- a/docs/reference/CONFIG.md
+++ b/docs/reference/CONFIG.md
@@ -120,8 +120,10 @@ Keys name model-written synthesis pages: `repo_overview` or
`onboarding/<slot>` for `guided_tour`, `getting_started`, `codebase_map`,
`key_concepts`, `how_it_works`, `development_guide`, and `active_landscape`.
`onboarding/project_overview` is invalid because that promoted slot is the
-`repo_overview` page. Unknown keys and malformed values fail generation with a
-configuration error instead of being ignored.
+`repo_overview` page. `onboarding/glossary` is accepted but has no effect: that
+page is rendered from mined vocabulary alone, with no model in its path, so
+there is no prompt for extra evidence to reach. Unknown keys and malformed
+values fail generation with a configuration error instead of being ignored.
Each value is an ordered list of repository-relative paths. A file is eligible
only when it was included in the indexed source map and contains non-empty
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
index 2be2e9eca..416ebfaf3 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -457,7 +457,7 @@ def _definition_after_heading(text: str, offset: int) -> str | None:
blank line above; anchoring on the end sidesteps that entirely.
"""
lines = text[offset:].split("\n")[1 : _DEFINITION_SCAN_LINES + 1]
- for line in lines:
+ for start, line in enumerate(lines):
stripped = line.strip()
if not stripped:
continue
@@ -465,13 +465,36 @@ def _definition_after_heading(text: str, offset: int) -> str | None:
return None
if stripped.startswith(_NOT_PROSE):
continue
- sentence = _first_sentence(stripped)
+ sentence = _first_sentence(_join_markdown_wrapped(lines, start))
if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:
return sentence
return None
return None
+def _join_markdown_wrapped(lines: list[str], start: int) -> str:
+ """The markdown paragraph beginning at ``start``, rejoined into one string.
+
+ The reST scan has done this since #1248; the markdown scan never did, and a
+ hard-wrapped README is as common as a hard-wrapped ``.txt``. Reading one
+ line returned the author's sentence cut where their editor wrapped it —
+ "Blast radius is the set of accounts a posting can reach through the
+ ledger" — which then reads as a definition that trails off mid-thought.
+
+ Stops at the paragraph break, at the next heading, at anything that is not
+ prose, and as soon as the text has a sentence in it.
+ """
+ parts = [lines[start].strip()]
+ for offset in range(start + 1, len(lines)):
+ if _SENTENCE_END.search(" ".join(parts)):
+ break
+ nxt = lines[offset].strip()
+ if not nxt or nxt.startswith(_NOT_PROSE) or _HEADING_LINE.match(lines[offset]):
+ break
+ parts.append(nxt)
+ return " ".join(parts)
+
+
# ---------------------------------------------------------------------------
# Which documents to read
# ---------------------------------------------------------------------------
@@ -767,6 +790,16 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:
truncated = False
for dirpath, dirnames, filenames in fs_walk.walk_repo(repo_root, prune_dirs=_EXTRA_PRUNED_DIRS):
+ # Hidden directories are tool and agent territory — CI definitions,
+ # editor state, hook configuration — never the subsystem prose that
+ # answers "was this term built". The walker already prunes the named
+ # ones and any nested checkout, but a *stale* worktree copy has had its
+ # ``.git`` file removed and reads as ordinary source: on this
+ # repository that put a second, deeper copy of every docstring under
+ # ``.claude/`` ahead of the original, and the mined definitions cited
+ # paths a reader has no reason to open. Pruning the whole class costs
+ # one string test and no repository keeps its documented source here.
+ dirnames[:] = [d for d in dirnames if not d.startswith(".")]
for name in dirnames:
dir_names.add(_singular(name.lower()))
if truncated:
diff --git a/packages/core/src/repowise/core/generation/house_vocabulary.py b/packages/core/src/repowise/core/generation/house_vocabulary.py
new file mode 100644
index 000000000..7c39aabf8
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/house_vocabulary.py
@@ -0,0 +1,291 @@
+"""Selecting and rendering the repository's own vocabulary.
+
+Two pages present mined house terms: the overview's capability table names the
+six the repository is most about, and the glossary defines all of them. They are
+the same computation at two depths — one ranked, corroborated list, sliced
+short for the front page and long for the lookup page — so the selection lives
+here and both callers read it rather than each deriving its own.
+
+Everything in this module is deterministic. A term, the repository's own
+sentence about it, and the path that sentence was read from are facts the run
+already holds, and facts written by a model are resampled on every render: two
+calls with the same prompt, the same model and the same temperature produced
+overviews that disagreed on their row count and on which paths they cited. So
+these tables are built here and embedded after the page comes back, and the
+glossary is rendered without a model at all.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+
+import structlog
+
+from .concept_tree.vocabulary import HouseTerm, phrase_pattern, term_words
+
+log = structlog.get_logger(__name__)
+
+
+@dataclass(frozen=True)
+class SelectedTerm:
+ """One mined term that earned a place on a page."""
+
+ term: str
+ #: The repository's own sentence, or ``None``. Never invented — a term the
+ #: repository named without defining is still a term, and writing a
+ #: definition for it is the one thing the vocabulary miner refuses to do.
+ definition: str | None
+ #: The document or source file the sentence was read from, falling back to
+ #: the first document that named the term. Always a real path.
+ source_path: str
+ #: How many parts of the system name it. The strength of the corroboration
+ #: rather than a fact about the repository, so it ranks and it logs; the
+ #: overview does not render it.
+ corroborating_pages: int
+ #: Whether the codebase defines a symbol by this name. A term that is also
+ #: a symbol may be rendered in backticks; a coined one may not, because the
+ #: grounding pass strips backticks off any token it cannot resolve.
+ is_indexed_symbol: bool = False
+ #: What the structural side calls the parts of the system that name this
+ #: term, in corpus order. This is the honest answer to "where is this used"
+ #: — every entry is a module group cut from the dependency graph, so it
+ #: points at code rather than at more prose.
+ corroborating_names: tuple[str, ...] = ()
+
+
+# ---------------------------------------------------------------------------
+# Is this prose a definition?
+# ---------------------------------------------------------------------------
+#
+# A mined "definition" is whatever prose sat nearest the term, and near a term
+# in a README that is often not a sentence about it. Two real examples from
+# this repository's own front page:
+#
+# CLI -> "repowise init [PATH] # index a codebase (one-time; ...)"
+# Distill -> "`repowise distill <cmd>` compresses command output *before*
+# the agent reads it:"
+#
+# The first is a line of shell with an inline comment. The second is a lead-in
+# that ends on a colon because the explanation is the code block underneath.
+# Neither says what the thing is, and an em dash is a better answer than
+# either: the reader learns the term exists and is not misinformed about what
+# it means.
+
+#: A command line rather than a sentence: a shell comment, a prompt, an option
+#: flag, a redirect or a pipeline.
+#:
+#: The redirect and pipe test requires whitespace on both sides. Bare ``<`` and
+#: ``>`` reject real prose: "Decisions are co-located ... under the
+#: ``decision:<record_id>`` namespace" is a sentence, and the angle brackets in
+#: it are a placeholder, not a redirect.
+_LOOKS_LIKE_COMMAND = re.compile(r"(^\s*[$>]\s|\s#\s|\s--?[a-zA-Z]|\s[|<>]\s)")
+#: Ends where the real explanation begins — a colon, or an unclosed opener.
+_TRAILS_OFF = (":", ",", ";", "-", "—", "–", "(", "[")
+#: How a statement finishes. Checked as well as :data:`_TRAILS_OFF`, because
+#: the two catch different things: that one rejects prose that stops mid-clause,
+#: this one rejects text that was never prose. A stray line of source lifted out
+#: of a scratch file — ``decisions/__init__.py ---- (PKG / "__init__.py")
+#: .write_text( '`` — passes every other test here (it names the term, it has
+#: words, it opens with a letter) and is stopped only by not ending in a way a
+#: sentence can end. Every definition mined from four repositories that a human
+#: would call a definition ends in one of these.
+_SENTENCE_END = (".", "!", "?")
+
+
+def is_a_sentence(text: str) -> bool:
+ """Whether mined prose reads as a statement about the term.
+
+ Deliberately shallow. This is not grammar checking — it is the difference
+ between a sentence and a fragment of shell, and getting it wrong in the
+ strict direction costs a definition, which every caller renders as an em
+ dash. Getting it wrong the other way puts a command line on a page as
+ though it explained something.
+ """
+ text = " ".join(text.split())
+ # Three, not four: "Blast-radius request/response models." is terse and is
+ # still the repository's own answer to what the term means. Two words is
+ # where the fragments live ("See below", "Two parts").
+ if len(text.split()) < 3:
+ return False
+ text = text.rstrip()
+ if text.endswith(_TRAILS_OFF) or not text.endswith(_SENTENCE_END):
+ return False
+ if _LOOKS_LIKE_COMMAND.search(text):
+ return False
+ # A statement starts with a word, not with punctuation or a code fence.
+ return text[:1].isalpha()
+
+
+def cell(text: str) -> str:
+ """Text safe to put in a markdown table cell.
+
+ A pipe ends the cell wherever it appears, and mined prose is the
+ repository's text rather than ours — a definition that quotes a shell
+ pipeline or a reStructuredText grid row would otherwise shift every column
+ to its right.
+ """
+ return " ".join(text.split()).replace("|", "\\|")
+
+
+def clamp(text: str, limit: int) -> str:
+ """*text* as one line and table-safe, cut to about *limit* characters.
+
+ Truncates first, escapes second. The other order cuts an escaped ``\\|`` in
+ half and leaves the backslash orphaned at the end of the cell, which is the
+ one thing escaping exists to prevent. "About" because escaping runs after:
+ a definition quoting a shell pipeline ends a character or two over, which
+ costs nothing a reader can see.
+ """
+ text = " ".join(text.split())
+ if len(text) > limit:
+ text = text[: limit - 1].rstrip() + "…"
+ return cell(text)
+
+
+# ---------------------------------------------------------------------------
+# Selection
+# ---------------------------------------------------------------------------
+
+
+def select_terms(
+ house_terms: Sequence[HouseTerm],
+ module_names: Iterable[str],
+ *,
+ limit: int | None = None,
+) -> list[SelectedTerm]:
+ """The mined terms worth publishing, in the order they go on a page.
+
+ Ranked by document frequency alone the mined terms are not publishable: on
+ this repository the top of that list holds the repository's own name,
+ "DONE", "Architecture" and "Files changed". So a term reaches a page only
+ with corroboration from a second, independently-derived artifact.
+
+ ``module_names`` are what the structural side calls the parts of the
+ system: one string per module group, its title followed by its summary. A
+ module group is cut from the dependency graph and named from the code, so a
+ term appearing in one was arrived at twice — from the documents and from
+ the structure — independently. That needs no stopword list and no
+ per-repository tuning.
+
+ Titles alone are about ninety short strings, which is too thin a net: it
+ misses "Knowledge Graph" and "Code Health" while letting "Architecture" and
+ "Workspace" through on an incidental word. The summaries are what make the
+ corroboration mean something.
+
+ Groups rather than written module *pages*, deliberately. A group exists on
+ every run, so a scoped run that regenerates one page selects the same rows
+ as a full one — a section that shrinks depending on how generation was
+ invoked is the instability these deterministic tables exist to remove.
+
+ **Multi-word terms come first.** Not as a filter — a single word still
+ reaches the page when there is room — but ahead of single words, because a
+ subsystem is nearly always named with two words ("blast radius", "dead
+ code", "change risk") and an ordinary English word with one. The ranking in
+ :func:`~...vocabulary.extract_house_terms` already encodes that as its
+ tiebreak; here it leads, because document frequency barely discriminates
+ (most repositories have two or three documents worth mining, so nearly
+ every term ties at one) and a junk row is expensive on a short page.
+
+ Ordering is total and derived only from the inputs, so two runs over an
+ unchanged repository select the same terms in the same order. *limit*
+ slices the tail: the whole list is one computation, and the front page
+ takes its head while the glossary takes more of it.
+ """
+ corpus = [name for name in module_names if name]
+ # Each entry is a group's title followed by its summary, so the first line
+ # is the group's name — what to call the place a term turned up.
+ titles = [entry.split("\n", 1)[0].strip() for entry in corpus]
+ # The regex is the expensive part and nearly every (term, group) pair is a
+ # miss, so a substring test on the term's first word rejects most pairs
+ # before it runs. The same trick is what made the miner itself 3.3× faster.
+ folded = [entry.lower() for entry in corpus]
+
+ selected: list[SelectedTerm] = []
+ for term in house_terms:
+ words = term_words(term.term)
+ if not words:
+ continue
+ # ASCII only. ``str.lower()`` and ``re.I`` disagree on a few code
+ # points — "İ".lower() is two characters — so on a non-ASCII lead word
+ # the cheap test can reject a pair the regex would have matched. Those
+ # terms skip the prefilter and pay the regex.
+ lead = words[0].lower() if words[0].isascii() else None
+ pattern = phrase_pattern(term.term)
+ # Sorted, not in corpus order. Everything else this function returns is
+ # derived only from its inputs' *contents*, and a field that reordered
+ # with the corpus would make two runs over an unchanged repository
+ # disagree whenever module grouping shuffled.
+ matched = tuple(
+ sorted(
+ titles[i]
+ for i, entry in enumerate(corpus)
+ if (lead is None or lead in folded[i]) and pattern.search(entry)
+ )
+ )
+ hits = len(matched)
+ if not hits:
+ continue
+ definition = term.definition
+ # No "the definition must name the term" test. It was built, measured
+ # and dropped: a heading gloss is the commonest definition shape there
+ # is, and it does not restate its own heading. "## Blast radius" over
+ # "The set of files a change can reach." is a repository defining its
+ # term perfectly, and the rule deleted it — along with every definition
+ # mined from a bolded lead-in, which captures only the text after the
+ # dash. It caught two junk rows here and would have emptied the
+ # definition column of any repository that writes that way.
+ if definition and not is_a_sentence(definition):
+ # Keep the row, drop the claim. The term is real — the structure
+ # corroborated it — but the prose nearest it is not a statement
+ # about it, and an em dash misinforms nobody.
+ log.info(
+ "house_vocabulary.definition_rejected",
+ term=term.term,
+ text=" ".join(definition.split())[:120],
+ )
+ definition = None
+ # Cite where the sentence came from, or -- when there is no sentence,
+ # including one just rejected -- the document that named the term.
+ # Citing the home of prose the page declined to quote would point a
+ # reader at a line that is not there.
+ source = (term.definition_source if definition else None) or (
+ term.source_paths[0] if term.source_paths else None
+ )
+ if source is None:
+ # A term with no path at all cannot be cited, and an uncitable row
+ # is the shape of claim this wiki does not make.
+ continue
+ selected.append(
+ SelectedTerm(
+ term=term.term,
+ definition=definition,
+ source_path=source,
+ corroborating_pages=hits,
+ is_indexed_symbol=term.is_indexed_symbol,
+ corroborating_names=matched,
+ )
+ )
+
+ selected.sort(key=lambda t: (len(term_words(t.term)) == 1, -t.corroborating_pages, t.term))
+ kept = selected if limit is None else selected[:limit]
+ log.info(
+ "house_vocabulary.selected",
+ mined=len(house_terms),
+ corroborated=len(selected),
+ kept=len(kept),
+ terms=[t.term for t in kept[:12]],
+ )
+ if house_terms and not selected:
+ # The documents name things the structure does not. That is a real
+ # answer about a repository — marketing vocabulary with no cluster
+ # behind it — but it is also what a corroboration corpus arriving empty
+ # looks like, so the two counts that separate them are logged rather
+ # than the section just not appearing.
+ log.warning(
+ "house_vocabulary.uncorroborated",
+ mined=len(house_terms),
+ module_names=len(corpus),
+ )
+ return kept
diff --git a/packages/core/src/repowise/core/generation/onboarding/__init__.py b/packages/core/src/repowise/core/generation/onboarding/__init__.py
index 9098fe183..44dc19950 100644
--- a/packages/core/src/repowise/core/generation/onboarding/__init__.py
+++ b/packages/core/src/repowise/core/generation/onboarding/__init__.py
@@ -1,15 +1,14 @@
"""Onboarding documentation collection.
-A curated set of up to nine pages: Project Overview, Architecture Guide,
-Guided Tour, Getting Started, Codebase Map, Key Concepts, How It Works,
-Development Guide, Active Landscape, designed to be the first thing a new
-contributor (or LLM agent) reads.
+A curated set of up to nine pages: Project Overview, Guided Tour, Getting
+Started, Codebase Map, Key Concepts, How It Works, Development Guide, Active
+Landscape and Glossary, designed to be the first thing a new contributor (or
+LLM agent) reads.
-One slot ("project_overview") is *promoted*: it
-reuse the existing ``repo_overview`` and ``architecture_diagram`` pages,
-tagged via ``metadata.onboarding_slot``. The other seven slots are new pages
-generated at level 8 with ``page_type='onboarding'`` and a
-``metadata.subkind`` discriminator.
+One slot ("project_overview") is *promoted*: it reuses the existing
+``repo_overview`` page, tagged via ``metadata.onboarding_slot``. The other
+eight slots are new pages generated at level 8 with ``page_type='onboarding'``
+and a ``metadata.subkind`` discriminator.
Architecture:
- :mod:`slots` — slot identifiers, fixed reading order, promoted map.
@@ -31,6 +30,7 @@ from .slots import (
SLOT_CODEBASE_MAP,
SLOT_DEVELOPMENT_GUIDE,
SLOT_GETTING_STARTED,
+ SLOT_GLOSSARY,
SLOT_GUIDED_TOUR,
SLOT_HOW_IT_WORKS,
SLOT_KEY_CONCEPTS,
@@ -47,6 +47,7 @@ __all__ = [
"SLOT_CODEBASE_MAP",
"SLOT_DEVELOPMENT_GUIDE",
"SLOT_GETTING_STARTED",
+ "SLOT_GLOSSARY",
"SLOT_GUIDED_TOUR",
"SLOT_HOW_IT_WORKS",
"SLOT_KEY_CONCEPTS",
diff --git a/packages/core/src/repowise/core/generation/onboarding/registry.py b/packages/core/src/repowise/core/generation/onboarding/registry.py
index 3e238ff0e..8c6168fbf 100644
--- a/packages/core/src/repowise/core/generation/onboarding/registry.py
+++ b/packages/core/src/repowise/core/generation/onboarding/registry.py
@@ -39,6 +39,20 @@ class SubkindSpec:
references; first occurrence sets priority and duplicates
are ignored. Missing files, symbols, or ranges become
observable evidence skips rather than generation errors.
+ deterministic: The page is rendered from its context alone, with no
+ provider call, on every run. Set it when the page is
+ made of facts rather than of judgements: a model asked
+ to restate enumerable facts resamples them, and two
+ renders of one unchanged repository then differ. Such a
+ subkind needs only its ``stub/`` template, because that
+ is the only one that ever renders.
+ needs_module_corroboration: Populate
+ ``OnboardingSignals.module_corroboration`` for this
+ subkind. Declared here rather than tested for by slot
+ name in the level builder, so a second subkind that
+ wants it cannot silently receive an empty tuple. It
+ costs a batched store read, and a run emitting no
+ subkind that wants it does not pay for one.
"""
slot: str
@@ -46,6 +60,8 @@ class SubkindSpec:
template: str
build_context: BuildContext
evidence_references: EvidenceReferences | None = None
+ deterministic: bool = False
+ needs_module_corroboration: bool = False
_REGISTRY: dict[str, SubkindSpec] = {}
diff --git a/packages/core/src/repowise/core/generation/onboarding/signals.py b/packages/core/src/repowise/core/generation/onboarding/signals.py
index 7939f39a4..176a6fe73 100644
--- a/packages/core/src/repowise/core/generation/onboarding/signals.py
+++ b/packages/core/src/repowise/core/generation/onboarding/signals.py
@@ -55,3 +55,10 @@ class OnboardingSignals:
# nothing, or when nothing it documents was built — all three are logged
# where the mining happens, because an empty tuple here cannot say which.
house_terms: tuple[HouseTerm, ...] = ()
+ # What the structural side calls the parts of the system: one string per
+ # module group, its title followed by its summary. The corroborating
+ # artifact for a mined term — a group is cut from the dependency graph and
+ # named from the code, so a term appearing in one was arrived at twice,
+ # independently. Empty on every run that emits no page needing it, because
+ # building it costs a store read.
+ module_corroboration: tuple[str, ...] = ()
diff --git a/packages/core/src/repowise/core/generation/onboarding/slots.py b/packages/core/src/repowise/core/generation/onboarding/slots.py
index 8611c7bd6..157da656f 100644
--- a/packages/core/src/repowise/core/generation/onboarding/slots.py
+++ b/packages/core/src/repowise/core/generation/onboarding/slots.py
@@ -1,8 +1,8 @@
"""Onboarding slot identifiers and reading order.
-The Onboarding collection is a fixed set of eight curated slots. One slot is
+The Onboarding collection is a fixed set of nine curated slots. One slot is
"promoted": it reuses the existing `repo_overview` page, tagged via
-``metadata.onboarding_slot``. The other seven are new pages with
+``metadata.onboarding_slot``. The other eight are new pages with
``page_type='onboarding'`` and a ``metadata.subkind`` discriminator.
Slot identifiers are used three ways:
@@ -23,7 +23,7 @@ from __future__ import annotations
# rendered prompt is byte-identical. Bump when a builder or template change
# should reach already-cached pages. (File pages have an equivalent in
# ``_generation_fingerprint``; onboarding pages had none until this.)
-ONBOARDING_GENERATION_VERSION = "2"
+ONBOARDING_GENERATION_VERSION = "3"
# ---- Slot identifiers ----
@@ -35,9 +35,14 @@ SLOT_KEY_CONCEPTS = "key_concepts"
SLOT_HOW_IT_WORKS = "how_it_works"
SLOT_DEVELOPMENT_GUIDE = "development_guide"
SLOT_ACTIVE_LANDSCAPE = "active_landscape"
+SLOT_GLOSSARY = "glossary"
# Fixed reading order. Slots not yet implemented are silently skipped at
# generation time and absent from the UI tree.
+#
+# The glossary is last on purpose: it is a lookup surface rather than a reading
+# step, and a reader who does not yet know the vocabulary is not helped by
+# meeting all of it at once.
ONBOARDING_ORDER: tuple[str, ...] = (
SLOT_PROJECT_OVERVIEW,
SLOT_GUIDED_TOUR,
@@ -47,6 +52,7 @@ ONBOARDING_ORDER: tuple[str, ...] = (
SLOT_HOW_IT_WORKS,
SLOT_DEVELOPMENT_GUIDE,
SLOT_ACTIVE_LANDSCAPE,
+ SLOT_GLOSSARY,
)
# Maps existing page_type → onboarding slot. The generator tags these pages
@@ -67,6 +73,7 @@ SLOT_TITLES: dict[str, str] = {
SLOT_HOW_IT_WORKS: "How It Works",
SLOT_DEVELOPMENT_GUIDE: "Development Guide",
SLOT_ACTIVE_LANDSCAPE: "Active Landscape",
+ SLOT_GLOSSARY: "Glossary",
}
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py
index cbe90bdee..b51f43978 100644
--- a/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/__init__.py
@@ -14,6 +14,7 @@ from . import (
codebase_map, # noqa: F401
development_guide, # noqa: F401
getting_started, # noqa: F401
+ glossary, # noqa: F401
guided_tour, # noqa: F401
how_it_works, # noqa: F401
key_concepts, # noqa: F401
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/glossary.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/glossary.py
new file mode 100644
index 000000000..8da2276a6
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/glossary.py
@@ -0,0 +1,165 @@
+"""Onboarding subkind: Glossary.
+
+The words this repository uses for itself, defined in its own sentences. Every
+row is read from the repository's own documents: the term, the sentence nearest
+it, and the path that sentence came from. Nothing here is written for the page.
+
+That is the whole design. A glossary is the page where an invented definition
+does the most damage — it is the one a reader consults *because* they do not
+know the answer, so they have nothing to check it against. So this page has no
+model in its path at all: the spec is registered ``deterministic``, and two
+renders of an unchanged repository are byte-identical.
+
+The cost is honesty about coverage. A term the repository names without ever
+defining renders with an em dash rather than a plausible sentence, and a
+repository whose documents define nothing gets no page. Both are the correct
+answers to a question this page cannot answer from the code.
+
+Gate: at least five terms survive corroboration. Below that the page is a
+handful of rows pretending to be a vocabulary.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import structlog
+
+from ...house_vocabulary import SelectedTerm, select_terms
+from ..registry import SubkindSpec, register
+from ..signals import OnboardingSignals
+from ..slots import SLOT_GLOSSARY, SLOT_TITLES
+
+log = structlog.get_logger(__name__)
+
+#: Below this the page is not a vocabulary. Five is the floor the track set:
+#: enough rows that a reader consults it rather than reads it.
+_GATE_MIN_TERMS = 5
+#: A lookup surface can be long — a reader scans for one row and ignores the
+#: rest — but not unbounded. Forty is roughly two screens and is far above what
+#: any repository measured so far corroborates.
+_MAX_TERMS = 40
+#: How many subsystems to name per row before the cell stops being scannable.
+_MAX_USED_IN = 3
+
+
+@dataclass(frozen=True)
+class GlossaryEntry:
+ """One row of the glossary, entirely mined."""
+
+ term: str
+ #: The repository's own sentence about the term, or ``None`` when it named
+ #: the term without defining it.
+ definition: str | None
+ #: The document the sentence was read from. Always a real path.
+ source_path: str
+ #: The parts of the system that name it, from the structural side.
+ used_in: tuple[str, ...]
+ #: Whether the codebase defines a symbol by this name, so the template
+ #: knows whether backticks would survive a grounding pass.
+ is_indexed_symbol: bool
+
+
+@dataclass
+class GlossaryContext:
+ repo_name: str
+ entries: list[GlossaryEntry] = field(default_factory=list)
+ #: Every document any surviving term was read from, deduplicated and
+ #: ordered. Rendered as the page's provenance line.
+ sources: list[str] = field(default_factory=list)
+ #: How many terms were mined, how many survived corroboration, and how many
+ #: of the rows carry a definition. All three are rendered: a glossary that
+ #: quietly covers a third of its vocabulary, or that silently drops the tail
+ #: of a long one, should say so on the page rather than imply completeness.
+ mined: int = 0
+ corroborated: int = 0
+ defined: int = 0
+
+
+def _entry(selected: SelectedTerm) -> GlossaryEntry:
+ names = selected.corroborating_names
+ used_in = names[:_MAX_USED_IN]
+ if len(names) > _MAX_USED_IN:
+ # Said, not silently dropped. A truncated list reads as the whole list,
+ # and "where is this used" is the column a reader acts on.
+ used_in = (*used_in, f"and {len(names) - _MAX_USED_IN} more")
+ return GlossaryEntry(
+ term=selected.term,
+ definition=selected.definition,
+ source_path=selected.source_path,
+ used_in=used_in,
+ is_indexed_symbol=selected.is_indexed_symbol,
+ )
+
+
+def _build(signals: OnboardingSignals) -> GlossaryContext | None:
+ if not signals.house_terms:
+ # Already logged where the mining happens, with the three counts that
+ # separate "no repository to read" from "nothing written" from
+ # "nothing built". Repeating them here would say less.
+ return None
+
+ # Every corroborated term, and no further test. A row whose definition
+ # column is an em dash still carries two facts a reader came for — the
+ # repository has a word for this, and here are the parts of the system
+ # that use it — and dropping those rows was measured to cost more than it
+ # saved: requiring a single word to arrive with a definition removed
+ # "Workspace", "Coupling", "CLI", "Distill", "Costs" and "Decisions" here
+ # and "Security" on django, to remove two weak rows. A lookup page is
+ # judged on coverage.
+ #
+ # Selected whole, then capped. The count before the cap is carried onto the
+ # page: the footer explains that a mined term reaches it only with
+ # structural corroboration, and that is a false account of any term dropped
+ # for length instead.
+ corroborated = select_terms(signals.house_terms, signals.module_corroboration)
+ selected = corroborated[:_MAX_TERMS]
+ if len(selected) < _GATE_MIN_TERMS:
+ log.info(
+ "onboarding.glossary_gate_skipped",
+ repo_name=signals.repo_name,
+ mined=len(signals.house_terms),
+ corroborated=len(corroborated),
+ required=_GATE_MIN_TERMS,
+ )
+ return None
+
+ # Alphabetical, because a glossary is looked up rather than read. Ranking
+ # decided which terms are here; it has no job left once they are.
+ entries = sorted((_entry(term) for term in selected), key=lambda e: e.term.lower())
+
+ sources: list[str] = []
+ for entry in entries:
+ if entry.source_path not in sources:
+ sources.append(entry.source_path)
+ defined = sum(1 for entry in entries if entry.definition)
+
+ log.info(
+ "onboarding.glossary_built",
+ repo_name=signals.repo_name,
+ mined=len(signals.house_terms),
+ corroborated=len(corroborated),
+ terms=len(entries),
+ defined=defined,
+ documents=len(sources),
+ )
+ return GlossaryContext(
+ repo_name=signals.repo_name,
+ entries=entries,
+ sources=sources,
+ mined=len(signals.house_terms),
+ corroborated=len(corroborated),
+ defined=defined,
+ )
+
+
+register(
+ SubkindSpec(
+ slot=SLOT_GLOSSARY,
+ title=SLOT_TITLES[SLOT_GLOSSARY],
+ template="glossary.j2",
+ build_context=_build,
+ deterministic=True,
+ needs_module_corroboration=True,
+ )
+)
diff --git a/packages/core/src/repowise/core/generation/overview_tables.py b/packages/core/src/repowise/core/generation/overview_tables.py
index aeed57f19..5d8686a7d 100644
--- a/packages/core/src/repowise/core/generation/overview_tables.py
+++ b/packages/core/src/repowise/core/generation/overview_tables.py
@@ -15,14 +15,19 @@ from __future__ import annotations
import re
from collections.abc import Iterable, Sequence
-from dataclasses import dataclass
import structlog
-from .concept_tree.vocabulary import HouseTerm, phrase_pattern, term_words
+from .concept_tree.vocabulary import HouseTerm
+from .house_vocabulary import SelectedTerm, cell, clamp, select_terms
log = structlog.get_logger(__name__)
+#: One row of the capability table. The glossary renders the same object, so
+#: the selection and the definition test live in :mod:`house_vocabulary` and
+#: this name is what the overview calls it.
+Capability = SelectedTerm
+
PACKAGE_TABLE_HEADING = "## Packages"
CAPABILITY_TABLE_HEADING = "## What it does"
@@ -85,17 +90,11 @@ def embed_package_table(content: str, table: str | None) -> str:
#
# The overview's opening question is "what does this thing do", and the answer
# should be in the repository's own words rather than in the generic vocabulary
-# of static analysis. The mined house terms are those words. Ranked by document
-# frequency alone they are not front-page material: on this repository the top
-# of that list holds the repository's own name, "DONE", "Architecture" and
-# "Files changed", which would put junk in half the rows.
-#
-# So a term reaches this table only with corroboration from a second,
-# independently-derived artifact: a module page — built from the dependency
-# graph, with no knowledge of the documents — has to name it in its title or
-# its summary. That needs no stopword list and no per-repository tuning.
+# of static analysis. The mined house terms are those words, selected and
+# corroborated by :func:`~house_vocabulary.select_terms`; the front page takes
+# the head of that list and the glossary takes more of it.
-_MAX_CAPABILITY_ROWS = 6
+MAX_CAPABILITY_ROWS = 6
#: Long enough for a sentence, short enough that a row stays one line to read.
_MAX_CAPABILITY_DEFINITION = 160
@@ -105,186 +104,20 @@ _CAPABILITY_SECTION_RE = re.compile(
)
-@dataclass(frozen=True)
-class Capability:
- """One row: a mined term, what the repository says it is, and where from."""
-
- term: str
- #: The repository's own sentence, or ``None``. Never invented — a term the
- #: repository named without defining is still a capability, and writing a
- #: definition for it is the one thing the vocabulary miner refuses to do.
- definition: str | None
- #: The document or source file the sentence was read from, falling back to
- #: the first document that named the term. Always a real path.
- source_path: str
- #: How many module pages name it. Kept for the log, not rendered — it is
- #: the strength of the corroboration, not a fact about the repository.
- corroborating_pages: int
-
-
def select_capabilities(
house_terms: Sequence[HouseTerm],
module_names: Iterable[str],
*,
- limit: int = _MAX_CAPABILITY_ROWS,
+ limit: int = MAX_CAPABILITY_ROWS,
) -> list[Capability]:
"""The terms worth putting on the front page, in the order they go there.
- ``module_names`` are what the structural side calls the parts of the
- system: the module groups' titles. They are the corroborating artifact — a
- module group is cut from the dependency graph and named from the code, so
- a term appearing in one was arrived at twice, from the documents and from
- the structure, independently.
-
- Titles only. Community labels and directory keys were tried and measured
- worse: a label like "Ingestion and Analysis Engine" is broad enough to
- corroborate almost any single common word, which put "Architecture" and
- "Workspace" on the front page of a real render.
-
- Groups rather than written module *pages*, deliberately. A group exists on
- every run, so a scoped run that regenerates the overview alone selects the
- same rows as a full one — a front-page section that shrinks depending on
- how generation was invoked is the instability these deterministic tables
- exist to remove. It is also the stronger claim of independence: a group's
- name is computed, while a page's summary is model-written and resampled.
-
- The cost is reach. Measured on django, matching titles and summaries of
- written pages corroborates 5 terms where names alone corroborate 2. The
- upgrade path is to read module summaries out of the store rather than out
- of this run, which would be both stable and rich; it needs level 6 to
- become async and the vector store to be present, so it is not done here.
-
- **Multi-word terms come first.** Not as a filter — a single word still
- reaches the table when there is room — but ahead of single words, because
- a subsystem is nearly always named with two words ("blast radius", "dead
- code", "change risk") and an ordinary English word with one. The ranking
- in :func:`~...vocabulary.extract_house_terms` already encodes that as its
- tiebreak; here it leads, because document frequency barely discriminates
- at six rows (most repositories have two or three documents worth mining,
- so nearly every term ties at one) and one junk row out of six is expensive.
-
- Ordering is total and derived only from the inputs, so two runs over an
- unchanged repository select the same rows in the same order.
- """
- corpus = [name for name in module_names if name]
- selected: list[Capability] = []
- for term in house_terms:
- pattern = phrase_pattern(term.term)
- hits = sum(1 for page in corpus if pattern.search(page))
- if not hits:
- continue
- definition = term.definition
- if definition and not _is_a_sentence(definition):
- # Keep the row, drop the claim. The capability is real — the
- # structure corroborated it — but the prose nearest the term is
- # not a statement about it, and an em dash misinforms nobody.
- log.info(
- "overview_capability_definition_rejected",
- term=term.term,
- text=" ".join(definition.split())[:120],
- )
- definition = None
- # Cite where the sentence came from, or -- when there is no sentence,
- # including one just rejected -- the document that named the term.
- # Citing the home of prose the table declined to quote would point a
- # reader at a line that is not there.
- source = (term.definition_source if definition else None) or (
- term.source_paths[0] if term.source_paths else None
- )
- if source is None:
- # A term with no path at all cannot be cited, and an uncitable row
- # on the front page is the shape of claim this wiki does not make.
- continue
- selected.append(
- Capability(
- term=term.term,
- definition=definition,
- source_path=source,
- corroborating_pages=hits,
- )
- )
-
- selected.sort(key=lambda c: (len(term_words(c.term)) == 1, -c.corroborating_pages, c.term))
- kept = selected[:limit]
- log.info(
- "overview_capabilities_selected",
- mined=len(house_terms),
- corroborated=len(selected),
- kept=len(kept),
- terms=[c.term for c in kept],
- )
- if house_terms and not selected:
- # The documents name things the structure does not. That is a real
- # answer about a repository — marketing vocabulary with no cluster
- # behind it — but it is also what a corroboration corpus arriving
- # empty looks like, so the two counts that separate them are logged
- # rather than the section just not appearing.
- log.warning(
- "overview_capabilities_uncorroborated",
- mined=len(house_terms),
- module_names=len(corpus),
- )
- return kept
-
-
-# A mined "definition" is whatever prose sat nearest the term, and near a term
-# in a README that is often not a sentence about it. Two real examples from
-# this repository's own front page:
-#
-# CLI -> "repowise init [PATH] # index a codebase (one-time; ...)"
-# Distill -> "`repowise distill <cmd>` compresses command output *before*
-# the agent reads it:"
-#
-# The first is a line of shell with an inline comment. The second is a lead-in
-# that ends on a colon because the explanation is the code block underneath.
-# Neither says what the thing is, and an em dash is a better answer than
-# either: the reader learns the capability exists and is not misinformed about
-# what it does.
-
-#: A command line rather than a sentence: a shell comment, a prompt, an option
-#: flag, a redirect or a pipeline.
-#:
-#: The redirect and pipe test requires whitespace on both sides. Bare ``<`` and
-#: ``>`` reject real prose: "Decisions are co-located ... under the
-#: ``decision:<record_id>`` namespace" is a sentence, and the angle brackets in
-#: it are a placeholder, not a redirect.
-_LOOKS_LIKE_COMMAND = re.compile(r"(^\s*[$>]\s|\s#\s|\s--?[a-zA-Z]|\s[|<>]\s)")
-#: Ends where the real explanation begins — a colon, or an unclosed opener.
-_TRAILS_OFF = (":", ",", ";", "-", "—", "–", "(", "[")
-
-
-def _is_a_sentence(text: str) -> bool:
- """Whether mined prose reads as a statement about the term.
-
- Deliberately shallow. This is not grammar checking — it is the difference
- between a sentence and a fragment of shell, and getting it wrong in the
- strict direction costs a definition, which the table already renders as an
- em dash. Getting it wrong the other way puts a command line on the front
- page as though it explained something.
- """
- text = " ".join(text.split())
- # Three, not four: "Blast-radius request/response models." is terse and is
- # still the repository's own answer to what the term means. Two words is
- # where the fragments live ("See below", "Two parts").
- if len(text.split()) < 3:
- return False
- if text.rstrip().endswith(_TRAILS_OFF):
- return False
- if _LOOKS_LIKE_COMMAND.search(text):
- return False
- # A statement starts with a word, not with punctuation or a code fence.
- return text[:1].isalpha()
-
-
-def _cell(text: str) -> str:
- """Text safe to put in a markdown table cell.
-
- A pipe ends the cell wherever it appears, and mined prose is the
- repository's text rather than ours — a definition that quotes a shell
- pipeline or a reStructuredText grid row would otherwise shift every column
- to its right.
+ The head of :func:`~house_vocabulary.select_terms`. The glossary renders
+ more of the same list, so the ranking, the corroboration and the definition
+ test are shared rather than derived twice — the front page is the top of
+ the glossary by construction, which is also what a reader expects of it.
"""
- return " ".join(text.split()).replace("|", "\\|")
+ return select_terms(house_terms, module_names, limit=limit)
def build_capability_table(capabilities: Sequence[Capability]) -> str | None:
@@ -303,10 +136,8 @@ def build_capability_table(capabilities: Sequence[Capability]) -> str | None:
"|---|---|---|",
]
for cap in capabilities:
- definition = _cell(cap.definition) if cap.definition else "—"
- if len(definition) > _MAX_CAPABILITY_DEFINITION:
- definition = definition[: _MAX_CAPABILITY_DEFINITION - 1].rstrip() + "…"
- lines.append(f"| {_cell(cap.term)} | {definition} | `{_cell(cap.source_path)}` |")
+ definition = clamp(cap.definition, _MAX_CAPABILITY_DEFINITION) if cap.definition else "—"
+ lines.append(f"| {cell(cap.term)} | {definition} | `{cell(cap.source_path)}` |")
log.debug("overview_capability_table_built", rows=len(capabilities))
return "\n".join(lines)
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index afa3d63d1..ad19ed250 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -32,6 +32,7 @@ from ..context.evidence import (
select_prompt_evidence,
)
from ..context_assembler import ContextAssembler, FilePageContext
+from ..house_vocabulary import cell
from ..models import (
MODEL_PAGE_CONFIDENCE,
GeneratedPage,
@@ -202,6 +203,11 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
self._jinja_env.filters.setdefault("oneline", oneline)
self._jinja_env.filters.setdefault("as_markdown", as_markdown)
self._jinja_env.filters.setdefault("signature", signature)
+ # A pipe ends a table cell wherever it appears, and a deterministic
+ # template interpolates the repository's own prose — which routinely
+ # quotes a shell pipeline. Without this every column to the right of
+ # one shifts.
+ self._jinja_env.filters.setdefault("table_cell", cell)
# ------------------------------------------------------------------
# generate_all — orchestration (delegates to orchestrate.py)
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index d362ec46c..17051c561 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -294,9 +294,19 @@ async def _module_corroboration(run: _GenerationRun) -> list[str]:
vanishes from the front page, and the store alone is one generation stale
on a full run. Never raises — a store that cannot answer costs reach, not
a page.
+
+ Memoised on the run. Two levels want it — the overview at 6 and the
+ glossary at 8 — and it costs a batched store read, so the second caller
+ reuses the first's answer rather than paying it again. A run that emits
+ neither page never reaches this.
"""
+ cached = getattr(run, "_module_corroboration_corpus", None)
+ if cached is not None:
+ return cached
+
groups = run.sel_module_groups
if not groups:
+ run._module_corroboration_corpus = []
return []
summaries: dict[str, str] = {}
@@ -332,7 +342,9 @@ async def _module_corroboration(run: _GenerationRun) -> list[str]:
from_this_run=sum(1 for mg in groups if run.completed_page_summaries.get(mg.key)),
with_summary=len(summaries),
)
- return [f"{mg.display}\n{summaries.get(mg.key, '')}" for mg in groups]
+ corpus = [f"{mg.display}\n{summaries.get(mg.key, '')}" for mg in groups]
+ run._module_corroboration_corpus = corpus
+ return corpus
async def build_level6_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
@@ -476,7 +488,7 @@ def _mine_house_terms(run: _GenerationRun) -> tuple[HouseTerm, ...]:
return terms
-def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
+async def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
"""Level 8 (curated onboarding collection)."""
gen = run.gen
coros: list[tuple[str, Any]] = []
@@ -511,6 +523,15 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
kg_layers = tuple(run.kg_ctx.get_layers())
kg_tour_steps = tuple(run.kg_ctx.get_tour())
+ # The corpus costs a batched store read, so only a run emitting a subkind
+ # that declared it wants one pays for it. Memoised, so a run that also
+ # emits the overview builds it once for both.
+ module_corroboration = (
+ tuple(await _module_corroboration(run))
+ if any(spec.needs_module_corroboration for _page_id, spec in emitted)
+ else ()
+ )
+
signals = _onboarding.OnboardingSignals(
repo_name=run.repo_name,
repo_structure=run.repo_structure,
@@ -531,6 +552,7 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
tour_stops=tuple(run.tour_stops),
layer_order=tuple(run.layer_order),
house_terms=_mine_house_terms(run),
+ module_corroboration=module_corroboration,
)
for page_id, spec in emitted:
coros.append((page_id, gen.generate_onboarding_page(spec, signals)))
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index a84f8c5b0..0bc44299c 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -716,7 +716,7 @@ class _GenerationRun:
final = (
await _levels.build_level6_coros(self)
+ _levels.build_level7_coros(self)
- + _levels.build_level8_coros(self)
+ + await _levels.build_level8_coros(self)
)
final_pages = await self.run_level(final, 8)
# Tag promoted onboarding slots (repo_overview / architecture_diagram).
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
index c5be7354b..42fb40618 100644
--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -523,10 +523,24 @@ class PerTypeGenerationMixin:
target = _onboarding.target_path(spec.slot)
references = spec.evidence_references(ctx) if spec.evidence_references else ()
- if self._config.deterministic:
+ if self._config.deterministic or spec.deterministic:
# No grounding post-check: a template can only cite what the
# context handed it, so there is nothing ungrounded to strip.
- page = self._stub_onboarding_page(spec, ctx, target)
+ #
+ # ``spec.deterministic`` takes this path on every run, not only
+ # under ``--no-prose``. The subkinds that set it are made of facts
+ # the run already holds, and asking a model to restate those is how
+ # a page that changed nothing comes back different — measured on
+ # the overview at two calls one second apart.
+ #
+ # The two paths render the same template and make opposite claims
+ # about the result: a ``--no-prose`` page is a page waiting for a
+ # model, and a ``deterministic`` subkind's page is finished.
+ page = (
+ self._model_free_onboarding_page(spec, ctx, target)
+ if spec.deterministic
+ else self._stub_onboarding_page(spec, ctx, target)
+ )
evidence = self._disabled_source_evidence(
page_key,
"deterministic_generation",
diff --git a/packages/core/src/repowise/core/generation/page_generator/structural.py b/packages/core/src/repowise/core/generation/page_generator/structural.py
index 67876d110..51b2369c2 100644
--- a/packages/core/src/repowise/core/generation/page_generator/structural.py
+++ b/packages/core/src/repowise/core/generation/page_generator/structural.py
@@ -575,6 +575,35 @@ class StructuralRenderMixin:
page.metadata["onboarding_slot"] = spec.slot
return page
+ def _model_free_onboarding_page(self, spec: Any, ctx: Any, target_path: str) -> GeneratedPage:
+ """Render a subkind that is finished without a model, not stubbed.
+
+ The same template, and a different claim about it. A stub is real
+ material with the prose missing, so it sits below the reader UI's
+ banner threshold and the tree offers to have a model write it. A
+ ``deterministic`` subkind is already everything its page is meant to
+ be — every statement on it came from the index and no model saw it —
+ which is exactly the claim :data:`TEMPLATE_PAGE_CONFIDENCE` makes.
+
+ ``model_free`` is stamped for the consumers that would otherwise read
+ ``provider_name='template'`` as "unwritten": scope resolution would
+ offer this page to ``generate --unwritten`` forever, and the web reader
+ would mark it "a model has not written this page yet" — on a page no
+ model is ever going to write.
+ """
+ page = self._render_page(
+ page_type="onboarding",
+ target_path=target_path,
+ title=spec.title,
+ template=f"{_STUB_PREFIX}/onboarding/{spec.template}",
+ ctx=ctx,
+ slot=spec.slot,
+ )
+ page.metadata["subkind"] = spec.slot
+ page.metadata["onboarding_slot"] = spec.slot
+ page.metadata["model_free"] = True
+ return page
+
def _rank_cycle_participants(ctx: Any) -> list[dict]:
"""Order a cycle's files by how many of its edges they carry.
diff --git a/packages/core/src/repowise/core/generation/scope.py b/packages/core/src/repowise/core/generation/scope.py
index f64444255..8bd854b3b 100644
--- a/packages/core/src/repowise/core/generation/scope.py
+++ b/packages/core/src/repowise/core/generation/scope.py
@@ -64,6 +64,13 @@ def load_page_records(pages: list[Any]) -> list[PageRecord]:
A page is "template" (unwritten) when a model never touched it: the
template provider stamps ``provider_name='template'`` and
``metadata.deterministic=True``. Either signal is enough.
+
+ ``metadata.model_free`` overrides both. Those pages carry the same two
+ signals and mean the opposite by them: their subkind is registered
+ ``deterministic``, so no model is ever going to write one. Counting them as
+ unwritten would offer them to ``generate --unwritten`` on every run, bill a
+ cost estimate for prose that will not be written, and leave the reader UI's
+ "bulk generate" affordance up on a wiki that is complete.
"""
records: list[PageRecord] = []
for p in pages:
@@ -71,8 +78,8 @@ def load_page_records(pages: list[Any]) -> list[PageRecord]:
meta = json.loads(getattr(p, "metadata_json", None) or "{}")
except ValueError:
meta = {}
- is_template = getattr(p, "provider_name", "") == "template" or bool(
- meta.get("deterministic")
+ is_template = not meta.get("model_free") and (
+ getattr(p, "provider_name", "") == "template" or bool(meta.get("deterministic"))
)
records.append(
PageRecord(
diff --git a/packages/core/src/repowise/core/generation/templates/stub/onboarding/glossary.j2 b/packages/core/src/repowise/core/generation/templates/stub/onboarding/glossary.j2
new file mode 100644
index 000000000..bc9148a62
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/stub/onboarding/glossary.j2
@@ -0,0 +1,51 @@
+{#- The glossary. Deterministic on every run, not only under --no-prose.
+
+ Every cell is mined: the term, the repository's own sentence about it, the
+ parts of the system that name it, and the document the sentence came from.
+ There is nothing here for a model to add that would not be an invention,
+ and this is the page where an invented definition does the most damage —
+ the reader consults it precisely because they cannot check the answer.
+
+ The spec sets ``deterministic=True``, so the prompt template that every
+ other subkind has does not exist for this one; this file is the page.
+-#}
+# Glossary
+
+{{ ctx.entries | length }} term{{ "s" if ctx.entries | length != 1 else "" }} `{{ ctx.repo_name }}` uses for itself, read from its own documentation. Each definition is the repository's own sentence, quoted rather than written — a term the repository names without defining is left blank rather than filled in.
+
+| Term | What the repository says it is | Where it is used | Written in |
+|---|---|---|---|
+{% for e in ctx.entries -%}
+{#- oneline before table_cell: truncating after escaping can cut a ``\|`` in
+ half and leave the backslash orphaned at the end of the cell. -#}
+| {% if e.is_indexed_symbol %}`{{ e.term | table_cell }}`{% else %}**{{ e.term | table_cell }}**{% endif %} | {% if e.definition %}{{ e.definition | oneline(220) | table_cell }}{% else %}—{% endif %} | {% if e.used_in %}{{ e.used_in | map("table_cell") | join(" · ") }}{% else %}—{% endif %} | `{{ e.source_path | table_cell }}` |
+{% endfor %}
+
+## Where these came from
+
+{{ ctx.defined }} of {{ ctx.entries | length }} term{{ "s" if ctx.entries | length != 1 else "" }} carry a definition the repository wrote. They were read from:
+
+{% for path in ctx.sources -%}
+- `{{ path }}`
+{% endfor %}
+
+{% if ctx.mined > ctx.corroborated %}
+{{ ctx.mined }} candidate terms were mined in all, and {{ ctx.corroborated }} survived. A term reaches this page only when the codebase's own structure names it too — the parts of the system in the third column are grouped from the dependency graph, with no knowledge of the documents — so vocabulary the documentation uses without the code confirming it is left off.
+{% endif %}
+{% if ctx.corroborated > ctx.entries | length %}
+The {{ ctx.entries | length }} most corroborated are listed; {{ ctx.corroborated - ctx.entries | length }} more were left off for length.
+{% endif %}
+{#- Not ``stub/_footer.j2``: that footer offers to write the page properly once
+ an API key is present, and this page is already what it is meant to be.
+
+ The blank line below is written as an explicit newline because it is
+ load-bearing and whitespace control keeps eating it. ``paragraph``
+ immediately followed by ``---`` is a setext H2 in CommonMark, not a
+ thematic break, so without it the closing note typesets the paragraph
+ above it as a section heading. -#}
+{{ "\n" }}
+---
+
+*Every term, definition and path on this page is quoted from the repository's
+own documents. Nothing here is written by a model, so two builds of an
+unchanged repository produce this page byte for byte.*
diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.py
index f4b982979..c073c1588 100644
--- a/packages/core/src/repowise/core/pipeline/phases/generation.py
+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py
@@ -167,12 +167,20 @@ async def run_generation(
}
if progress:
if onboarding_generated or promoted_present:
+ # Imported here, not at module scope: importing the onboarding
+ # package registers every subkind, and this module is on the CLI's
+ # import path where that cost buys nothing.
+ from repowise.core.generation.onboarding.slots import ONBOARDING_ORDER
+
slots_made = sorted(
{p.metadata.get("subkind", "?") for p in onboarding_generated} | promoted_present
)
progress.on_message(
"info",
- f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",
+ # Derived rather than written down: the denominator was a
+ # literal 8 and went stale the first time a slot was added.
+ f"Onboarding: {len(slots_made)}/{len(ONBOARDING_ORDER)} slots"
+ f" — {', '.join(slots_made)}",
)
# Placeholders left behind by a failed provider call are in this list
# like any other page, and the run reports them as failures a line
diff --git a/packages/ui/src/lib/page-types.ts b/packages/ui/src/lib/page-types.ts
index 6826d8043..dd3df7d08 100644
--- a/packages/ui/src/lib/page-types.ts
+++ b/packages/ui/src/lib/page-types.ts
@@ -69,11 +69,23 @@ export function isModelWrittenType(pageType: string | null | undefined): boolean
/** True when a model-written page is still a structural stub (no prose yet).
* Scoped to the model-written types: a stub carries `provider_name ===
* "template"`, a written page a real provider. Returns false for every
- * structural page type, which is never a stub in this sense. */
+ * structural page type, which is never a stub in this sense.
+ *
+ * `metadata.model_free` marks a page whose subkind is rendered without a
+ * model by design — the glossary quotes mined definitions and has no prompt
+ * at all. It carries `provider_name === "template"` like a stub and means the
+ * opposite by it: the page is finished, and no model is ever going to write
+ * it. Without this check the tree marks it "a model has not written this page
+ * yet", offers a regenerate button that cannot help, and keeps the bulk
+ * generate affordance up on a complete wiki. */
export function isStubPage(
- page: { page_type?: string; provider_name?: string } | null | undefined,
+ page:
+ | { page_type?: string; provider_name?: string; metadata?: Record<string, unknown> }
+ | null
+ | undefined,
): boolean {
if (!page || !isModelWrittenType(page.page_type)) return false;
+ if (page.metadata?.["model_free"]) return false;
return page.provider_name === "template";
}
@@ -83,10 +95,14 @@ export function isStubPage(
//
// One slot — project_overview — is *promoted*: its
// content lives in the existing repo_overview page, tagged via
-// `metadata.onboarding_slot`. The other seven are dedicated
+// `metadata.onboarding_slot`. The other eight are dedicated
// `page_type === "onboarding"` pages with `metadata.subkind` discriminating
// them.
//
+// A slot missing from this map is not merely unlabelled: `isOnboardingSlot`
+// gates on it, so `getOnboardingSlot` returns null and the page drops out of
+// the Onboarding folder entirely. Add every new slot here.
+//
// This map is display text only. The *reading order* used to be duplicated
// here as an ONBOARDING_ORDER array kept in lockstep with `slots.py` by
// comment alone; it now arrives on the pages themselves as `display_order`,
@@ -102,6 +118,7 @@ export const ONBOARDING_SLOT_TITLES = {
how_it_works: "How It Works",
development_guide: "Development Guide",
active_landscape: "Active Landscape",
+ glossary: "Glossary",
} as const;
export type OnboardingSlot = keyof typeof ONBOARDING_SLOT_TITLES;
diff --git a/tests/unit/generation/test_onboarding.py b/tests/unit/generation/test_onboarding.py
index 85aeaf66e..0c825cddc 100644
--- a/tests/unit/generation/test_onboarding.py
+++ b/tests/unit/generation/test_onboarding.py
@@ -168,9 +168,13 @@ def test_subkinds_registered_in_canonical_order() -> None:
# ONBOARDING_ORDER.
expected = [s for s in ONBOARDING_ORDER if s not in PROMOTED_SLOTS.values()]
assert slots == expected
- # Six templated subkinds + the topology-driven guided tour.
- assert len(slots) == 7
+ # Six templated subkinds, the topology-driven guided tour, and the
+ # glossary, which is rendered without a model at all.
+ assert len(slots) == 8
assert "guided_tour" in slots
+ # Last, and it must stay last: a glossary is a lookup surface, not a
+ # reading step.
+ assert slots[-1] == "glossary"
def test_onboarding_level_is_eight() -> None:
diff --git a/tests/unit/generation/test_onboarding_glossary.py b/tests/unit/generation/test_onboarding_glossary.py
new file mode 100644
index 000000000..55a41c431
--- /dev/null
+++ b/tests/unit/generation/test_onboarding_glossary.py
@@ -0,0 +1,503 @@
+"""The glossary page: the words a repository uses for itself, defined by it.
+
+Every assertion here is about the same property. A glossary is the page a
+reader consults *because* they do not know the answer, so they cannot catch a
+wrong one. Nothing on it may be written — the term, the sentence and the path
+are all quoted, and a term the repository never defined renders blank rather
+than plausibly.
+"""
+
+from __future__ import annotations
+
+import jinja2
+import pytest
+from structlog.testing import capture_logs
+
+from repowise.core.generation.concept_tree.vocabulary import HouseTerm
+from repowise.core.generation.house_vocabulary import cell
+from repowise.core.generation.onboarding import get_spec
+from repowise.core.generation.onboarding.signals import OnboardingSignals
+from repowise.core.generation.onboarding.slots import SLOT_GLOSSARY
+from repowise.core.generation.onboarding.subkinds.glossary import _build
+from repowise.core.generation.page_generator.structural import oneline
+from repowise.core.ingestion.models import RepoStructure
+
+#: What the structural side calls the parts of the system. Cut from the
+#: dependency graph and named from the code, with no knowledge of the
+#: documents — which is what makes a match between the two mean something.
+MODULES = [
+ "Dead Code and Reachability Analysis\nFinds files no import path reaches.",
+ "Blast Radius UI\nRenders the blast radius of a pending change.",
+ "Change Risk Analysis\nScores a diff before it merges.",
+ "Ingestion Pipeline\nWalks the tree and parses each file.",
+ "Knowledge Graph Routes\nServes the knowledge graph to the web reader.",
+ "Split File Refactoring\nProposes a split for an overlong file.",
+]
+
+
+def _term(
+ term: str,
+ *,
+ definition: str | None = None,
+ definition_source: str | None = None,
+ source_paths: tuple[str, ...] = ("README.md",),
+ doc_frequency: int = 1,
+ is_indexed_symbol: bool = False,
+) -> HouseTerm:
+ return HouseTerm(
+ term=term,
+ definition=definition,
+ definition_source=definition_source,
+ source_paths=source_paths,
+ doc_frequency=doc_frequency,
+ code_frequency=5,
+ is_indexed_symbol=is_indexed_symbol,
+ )
+
+
+#: Six terms every one of ``MODULES`` corroborates, enough to clear the gate.
+SIX_TERMS = [
+ _term("Dead code", definition="Dead code is code no import path reaches."),
+ _term("Blast radius", definition="Blast radius is what a change can reach."),
+ _term("Change risk", definition="Change risk scores a diff before it merges."),
+ _term("Ingestion pipeline", definition="Ingestion pipeline walks and parses the tree."),
+ _term("Knowledge graph", definition="Knowledge graph holds the system's entities."),
+ _term("Split file", definition="Split file proposes a split for a long file."),
+]
+
+
+def _signals(terms, modules=MODULES) -> OnboardingSignals:
+ return OnboardingSignals(
+ repo_name="ledger",
+ repo_structure=RepoStructure(
+ is_monorepo=False,
+ packages=[],
+ root_language_distribution={"python": 1.0},
+ total_files=1,
+ total_loc=10,
+ entry_points=[],
+ ),
+ parsed_files=(),
+ source_map={},
+ graph_builder=None,
+ pagerank={},
+ betweenness={},
+ community={},
+ sccs=(),
+ house_terms=tuple(terms),
+ module_corroboration=tuple(modules),
+ )
+
+
+# ---------------------------------------------------------------------------
+# The gate
+# ---------------------------------------------------------------------------
+
+
+def test_a_repository_with_enough_vocabulary_gets_a_page():
+ ctx = _build(_signals(SIX_TERMS))
+ assert ctx is not None
+ assert len(ctx.entries) == 6
+
+
+def test_four_terms_is_not_a_glossary():
+ """Below the floor the page is a handful of rows pretending to be a
+ vocabulary, so it does not build at all."""
+ assert _build(_signals(SIX_TERMS[:4])) is None
+
+
+def test_the_skip_is_logged_with_the_counts_that_explain_it():
+ """A page that stops appearing must say why. The two counts separate
+ "nothing was mined" from "nothing the structure confirmed"."""
+ with capture_logs() as logs:
+ _build(_signals(SIX_TERMS[:4]))
+ event = next(e for e in logs if e["event"] == "onboarding.glossary_gate_skipped")
+ assert event["mined"] == 4
+ assert event["corroborated"] == 4
+ assert event["required"] == 5
+
+
+def test_a_repository_that_mined_nothing_gets_no_page():
+ assert _build(_signals([])) is None
+
+
+def test_terms_no_module_group_corroborates_do_not_reach_the_page():
+ """The corroboration is the whole selection rule. Vocabulary the documents
+ use and the code never confirms is marketing, not a glossary entry."""
+ assert _build(_signals(SIX_TERMS, modules=["Something Else\nUnrelated."])) is None
+
+
+# ---------------------------------------------------------------------------
+# What a row may say
+# ---------------------------------------------------------------------------
+
+
+def test_a_term_the_repository_never_defined_is_left_blank():
+ """The one thing this page refuses to do. An invented definition on a
+ glossary is unfalsifiable by the reader who came to look it up."""
+ terms = [*SIX_TERMS[:5], _term("Split file")]
+ ctx = _build(_signals(terms))
+ entry = next(e for e in ctx.entries if e.term == "Split file")
+ assert entry.definition is None
+ assert entry.source_path == "README.md"
+
+
+def test_a_definition_that_never_names_its_term_is_still_taken():
+ """A heading gloss does not restate its own heading, and it is the
+ commonest definition shape there is.
+
+ Requiring the term to appear was built and measured and dropped. It caught
+ two junk rows on this repository — "Related" glossed as "The performance
+ pillar has its own, separate benchmark" — and it deleted "## Blast radius"
+ over "The set of files a change can reach", along with every definition
+ mined from a bolded lead-in, which captures only the text after the dash.
+ A repository that documents in that style would have lost its whole
+ definition column, on the front page as well as here.
+ """
+ terms = [
+ *SIX_TERMS[:5],
+ _term("Split file", definition="The file-level analog of Extract Class."),
+ ]
+ ctx = _build(_signals(terms))
+ assert next(e for e in ctx.entries if e.term == "Split file").definition == (
+ "The file-level analog of Extract Class."
+ )
+
+
+def test_a_command_line_is_not_a_definition():
+ terms = [
+ *SIX_TERMS[:5],
+ _term("Split file", definition="repowise split-file --json # list them"),
+ ]
+ ctx = _build(_signals(terms))
+ assert next(e for e in ctx.entries if e.term == "Split file").definition is None
+
+
+def test_an_undefined_term_still_earns_its_row():
+ """Corroboration is the only test a term has to pass.
+
+ A row whose middle column is an em dash still carries two facts a reader
+ came for: the repository has a word for this, and these are the parts of
+ the system that use it. Requiring a definition was measured and reverted —
+ on this repository it removed "Workspace", "Coupling", "CLI", "Distill",
+ "Costs" and "Decisions", and on django it removed "Security", to remove
+ two weak rows. A lookup page is judged on coverage.
+ """
+ ctx = _build(_signals([*SIX_TERMS, _term("Ingestion")]))
+ listed = {e.term for e in ctx.entries}
+ assert "Ingestion" in listed
+ assert next(e for e in ctx.entries if e.term == "Ingestion").definition is None
+
+
+def test_every_row_cites_a_real_path():
+ ctx = _build(_signals(SIX_TERMS))
+ assert all(e.source_path for e in ctx.entries)
+
+
+def test_the_source_is_the_document_the_sentence_came_from():
+ terms = [
+ *SIX_TERMS[:5],
+ _term(
+ "Split file",
+ definition="Split file proposes a split.",
+ definition_source="docs/refactoring.md",
+ source_paths=("README.md",),
+ ),
+ ]
+ ctx = _build(_signals(terms))
+ assert next(e for e in ctx.entries if e.term == "Split file").source_path == (
+ "docs/refactoring.md"
+ )
+
+
+def test_a_row_names_the_parts_of_the_system_that_use_it():
+ ctx = _build(_signals(SIX_TERMS))
+ entry = next(e for e in ctx.entries if e.term == "Dead code")
+ assert "Dead Code and Reachability Analysis" in entry.used_in
+
+
+# ---------------------------------------------------------------------------
+# Stability
+# ---------------------------------------------------------------------------
+
+
+def test_rows_are_alphabetical():
+ """A glossary is looked up rather than read. Ranking decided which terms
+ are here; it has no job left once they are."""
+ ctx = _build(_signals(SIX_TERMS))
+ terms = [e.term for e in ctx.entries]
+ assert terms == sorted(terms, key=str.lower)
+
+
+def test_two_builds_of_one_repository_agree():
+ """An unstable glossary churns ``source_hash`` on every index, which
+ re-embeds and re-persists a page that did not change."""
+ assert _build(_signals(SIX_TERMS)) == _build(_signals(SIX_TERMS))
+
+
+def test_the_order_of_the_corroboration_corpus_does_not_change_the_page():
+ assert _build(_signals(SIX_TERMS)) == _build(_signals(SIX_TERMS, modules=MODULES[::-1]))
+
+
+# ---------------------------------------------------------------------------
+# The page
+# ---------------------------------------------------------------------------
+
+
+def render():
+ from pathlib import Path
+
+ import repowise.core.generation as generation
+
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(str(Path(generation.__file__).parent / "templates")),
+ undefined=jinja2.StrictUndefined,
+ autoescape=False,
+ )
+ env.filters["oneline"] = oneline
+ env.filters["table_cell"] = cell
+
+ def _render(ctx):
+ return env.get_template("stub/onboarding/glossary.j2").render(
+ ctx=ctx, slot=SLOT_GLOSSARY
+ )
+
+ return _render
+
+
+def test_the_page_renders_one_row_per_term(render):
+ page = render(_build(_signals(SIX_TERMS)))
+ rows = [line for line in page.splitlines() if line.startswith("| ")]
+ # Six terms plus the header row. The alignment row opens ``|-``.
+ assert len(rows) == 7
+ assert "{{" not in page
+
+
+def test_an_undefined_term_renders_an_em_dash_not_a_sentence(render):
+ page = render(_build(_signals([*SIX_TERMS[:5], _term("Split file")])))
+ row = next(line for line in page.splitlines() if line.startswith("| **Split file**"))
+ assert "| — |" in row
+
+
+def test_a_pipe_in_mined_prose_does_not_break_the_table(render):
+ """Definitions are the repository's text, not ours. One quoting a shell
+ pipeline would otherwise shift every column to its right."""
+ terms = [
+ *SIX_TERMS[:5],
+ _term("Split file", definition="Split file findings pipe | into the report."),
+ ]
+ page = render(_build(_signals(terms)))
+ row = next(line for line in page.splitlines() if line.startswith("| **Split file**"))
+ assert row.count("|") - row.count("\\|") == 5
+
+
+def test_a_term_the_codebase_defines_is_backticked_and_a_coined_one_is_not(render):
+ """``check_grounding`` strips backticks off any token it cannot resolve to
+ a symbol, so backticking a coined term is a silent failure waiting to
+ happen. This page is deterministic and never passes through that check —
+ which is exactly why it has to get the distinction right itself."""
+ terms = [
+ *SIX_TERMS[:5],
+ _term("SplitFile", definition="SplitFile proposes a split.", is_indexed_symbol=True),
+ ]
+ modules = [*MODULES, "Split File Refactoring\nThe SplitFile detector runs here."]
+ page = render(_build(_signals(terms, modules=modules)))
+ assert "| `SplitFile` |" in page
+ assert "| **Dead code** |" in page
+
+
+def test_the_page_says_how_much_of_its_vocabulary_it_could_define(render):
+ """A glossary covering a third of the terms it lists should say so rather
+ than reading as though the repository defined everything."""
+ page = render(_build(_signals([*SIX_TERMS[:5], _term("Split file")])))
+ assert "5 of 6 terms carry a definition" in page
+
+
+# ---------------------------------------------------------------------------
+# Wiring
+# ---------------------------------------------------------------------------
+
+
+def test_the_glossary_is_registered_and_needs_no_model():
+ """Enumerable facts written by a model are resampled on every render. The
+ whole page is enumerable facts, so no model is in its path."""
+ spec = get_spec(SLOT_GLOSSARY)
+ assert spec is not None
+ assert spec.deterministic is True
+ assert spec.needs_module_corroboration is True
+ assert spec.title == "Glossary"
+
+
+def test_every_subkind_has_the_templates_its_flags_promise():
+ """The flag and the files have to agree, and nothing else checks that.
+
+ A ``deterministic`` subkind renders only its ``stub/`` template, so it is
+ allowed to ship without a prompt. Every other subkind needs both, and
+ losing either is a ``TemplateNotFound`` at generation time on a repository
+ that is not ours — the suite is where that has to fail instead.
+ """
+ from pathlib import Path
+
+ import repowise.core.generation as generation
+ from repowise.core.generation.onboarding import iter_specs
+
+ templates = Path(generation.__file__).parent / "templates"
+ for spec in iter_specs():
+ stub = templates / "stub" / "onboarding" / spec.template
+ assert stub.is_file(), f"{spec.slot} has no stub template at {stub}"
+ prompt = templates / "onboarding" / spec.template
+ if spec.deterministic:
+ assert not prompt.is_file(), (
+ f"{spec.slot} is deterministic, so its prompt template at "
+ f"{prompt} is dead weight that no run can reach"
+ )
+ else:
+ assert prompt.is_file(), f"{spec.slot} has no prompt template at {prompt}"
+
+
+def test_a_deterministic_subkind_reaches_the_no_provider_path():
+ """The flag has to be read, not merely set.
+
+ ``generate_onboarding_page`` branches on it, and the glossary has no prompt
+ template at all — so if that branch is ever dropped the page raises
+ ``TemplateNotFound`` in production while every assertion about the spec
+ still passes.
+ """
+ import inspect
+
+ from repowise.core.generation.page_generator import pertype
+
+ source = inspect.getsource(pertype.PerTypeGenerationMixin.generate_onboarding_page)
+ assert "spec.deterministic" in source
+ assert "_model_free_onboarding_page" in source
+
+
+# ---------------------------------------------------------------------------
+# Honesty about what is not on the page
+# ---------------------------------------------------------------------------
+
+
+#: Word-only stems, because a term's words are what the corroboration pattern
+#: matches — a digit in the name matches no module group and corroborates
+#: nothing.
+_STEMS = [
+ "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel",
+ "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa",
+ "quebec", "romeo", "sierra", "tango", "uniform", "victor", "whiskey",
+ "xray", "yankee", "zulu", "amber", "bronze", "copper", "diamond",
+ "emerald", "flint", "garnet", "ivory", "jade", "kevlar", "lapis",
+ "marble", "nickel", "onyx", "pearl", "quartz", "ruby", "slate", "topaz",
+ "umber", "violet", "willow", "xenon", "yarrow", "zircon", "almond",
+ "birch", "cedar", "dogwood",
+]
+
+
+def _many(count: int):
+ """*count* distinct two-word terms, and a module group naming each."""
+ stems = _STEMS[:count]
+ assert len(stems) == count, "not enough distinct stems for this test"
+ terms = [
+ _term(f"{s.title()} code", definition=f"{s.title()} code is a rule nothing reaches.")
+ for s in stems
+ ]
+ modules = [f"{s.title()} Code Analysis\nFinds {s} code." for s in stems]
+ return terms, modules
+
+
+def test_a_long_vocabulary_is_capped_and_says_so(render):
+ """The footer explains that a term reaches this page only with structural
+ corroboration. That is a false account of a term dropped for length, so the
+ count before the cap is carried onto the page too."""
+ terms, modules = _many(55)
+ ctx = _build(_signals(terms, modules=modules))
+ assert ctx.corroborated == 55
+ assert len(ctx.entries) == 40
+
+ page = render(ctx)
+ assert "The 40 most corroborated are listed; 15 more were left off" in page
+ # Nothing failed corroboration here, so the page does not claim anything did.
+ assert "candidate terms were mined in all" not in page
+
+
+def test_a_page_listing_everything_claims_no_truncation(render):
+ page = render(_build(_signals(SIX_TERMS)))
+ assert "left off for length" not in page
+
+
+def test_a_row_says_when_its_subsystem_list_is_cut(render):
+ """"Where it is used" is the column a reader acts on, and a truncated list
+ reads as the whole list."""
+ modules = [
+ *MODULES,
+ "Dead Code Sweeper\nRuns the dead code pass nightly.",
+ "Dead Code Report\nRenders the dead code findings.",
+ "Dead Code API\nServes dead code to the web reader.",
+ ]
+ ctx = _build(_signals(SIX_TERMS, modules=modules))
+ entry = next(e for e in ctx.entries if e.term == "Dead code")
+ assert entry.used_in[-1] == "and 1 more"
+ assert "and 1 more" in render(ctx)
+
+
+# ---------------------------------------------------------------------------
+# The page is finished, not a stub
+# ---------------------------------------------------------------------------
+
+
+def test_the_rendered_page_does_not_open_a_setext_heading(render):
+ """``paragraph`` followed by ``---`` is a setext H2 in CommonMark, not a
+ thematic break. Without a blank line between them the closing note typesets
+ the paragraph above it as a section heading."""
+ long_terms, long_modules = _many(55)
+ contexts = (_build(_signals(SIX_TERMS)), _build(_signals(long_terms, modules=long_modules)))
+ for ctx in contexts:
+ page = render(ctx)
+ rule = page.index("\n---\n")
+ assert page[rule - 1] == "\n", "the thematic break has no blank line above it"
+
+
+def test_two_renders_of_one_repository_are_byte_identical(render):
+ """The page's own closing claim, asserted on the bytes rather than on the
+ context object."""
+ first = render(_build(_signals(SIX_TERMS)))
+ second = render(_build(_signals(SIX_TERMS)))
+ assert first == second
+
+
+def test_a_model_free_page_is_not_counted_as_unwritten():
+ """It carries ``provider_name='template'`` like a stub and means the
+ opposite by it.
+
+ Counted as unwritten, the glossary is offered to ``generate --unwritten``
+ on every run, billed in the cost estimate for prose that will not be
+ written, and leaves the reader UI's bulk-generate affordance up on a wiki
+ that is complete.
+ """
+ import json
+ from types import SimpleNamespace
+
+ from repowise.core.generation.scope import load_page_records
+
+ def _page(page_id, meta):
+ return SimpleNamespace(
+ id=page_id,
+ page_type="onboarding",
+ target_path=f"onboarding/{page_id}",
+ provider_name="template",
+ metadata_json=json.dumps(meta),
+ freshness_status="fresh",
+ )
+
+ records = {
+ r.page_id: r
+ for r in load_page_records(
+ [
+ _page("glossary", {"subkind": "glossary", "model_free": True}),
+ _page("key_concepts", {"subkind": "key_concepts"}),
+ ]
+ )
+ }
+ assert records["glossary"].is_template is False
+ # An ordinary subkind on a stub is still unwritten, and still offered.
+ assert records["key_concepts"].is_template is True
diff --git a/tests/unit/generation/test_onboarding_house_terms.py b/tests/unit/generation/test_onboarding_house_terms.py
index f49e68b19..fde46b9b2 100644
--- a/tests/unit/generation/test_onboarding_house_terms.py
+++ b/tests/unit/generation/test_onboarding_house_terms.py
@@ -155,6 +155,11 @@ def _run(*, repo_path: Path | None) -> SimpleNamespace:
decisions_all=(),
external_systems=(),
completed_page_summaries={},
+ # The glossary corroborates mined terms against the module groups, so
+ # level 8 now reads these too. Empty here: this module is about the
+ # mining, and the glossary's own tests cover what corroboration does.
+ sel_module_groups=[],
+ vector_store=None,
tour_stops=(),
layer_order=(),
kg_ctx=None,
@@ -163,8 +168,8 @@ def _run(*, repo_path: Path | None) -> SimpleNamespace:
)
-def _signals_from_level8(run: SimpleNamespace) -> Any:
- coros = build_level8_coros(run)
+async def _signals_from_level8(run: SimpleNamespace) -> Any:
+ coros = await build_level8_coros(run)
assert coros, "level 8 produced no pages, so no signals were built"
assert run.gen.seen, "no subkind was handed the signals"
return run.gen.seen[0]
@@ -175,27 +180,27 @@ def _signals_from_level8(run: SimpleNamespace) -> Any:
# ---------------------------------------------------------------------------
-def test_documented_repository_populates_the_field(tmp_path: Path) -> None:
+async def test_documented_repository_populates_the_field(tmp_path: Path) -> None:
reset_house_terms()
- signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+ signals = await _signals_from_level8(_run(repo_path=_repo(tmp_path)))
terms = {t.term for t in signals.house_terms}
assert "Blast radius" in terms
assert "Change risk" in terms
-def test_every_subkind_reads_the_same_mined_terms(tmp_path: Path) -> None:
+async def test_every_subkind_reads_the_same_mined_terms(tmp_path: Path) -> None:
"""One walk of the repository per run, not one per slot."""
reset_house_terms()
run = _run(repo_path=_repo(tmp_path))
- build_level8_coros(run)
+ await build_level8_coros(run)
assert len(run.gen.seen) > 1, "expected more than one onboarding slot"
first = run.gen.seen[0].house_terms
assert all(s.house_terms is first for s in run.gen.seen)
-def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(
+async def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(
tmp_path: Path,
) -> None:
"""Mining walks the whole repository, so a run writing none must not pay.
@@ -207,41 +212,41 @@ def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(
run = _run(repo_path=_repo(tmp_path))
run._emit = lambda page_id: False
- assert build_level8_coros(run) == []
+ assert await build_level8_coros(run) == []
assert run.gen.seen == []
assert house_terms_mined().mined is False
-def test_a_term_the_codebase_defines_is_marked_as_a_symbol(tmp_path: Path) -> None:
+async def test_a_term_the_codebase_defines_is_marked_as_a_symbol(tmp_path: Path) -> None:
"""``is_indexed_symbol`` decides whether a term may be backticked.
Without the run's symbol names it is ``False`` for every term, which reads
downstream as "this repository defines none of its own vocabulary".
"""
reset_house_terms()
- signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+ signals = await _signals_from_level8(_run(repo_path=_repo(tmp_path)))
by_term = {t.term: t for t in signals.house_terms}
assert by_term["Blast radius"].is_indexed_symbol is False
assert any(t.is_indexed_symbol for t in signals.house_terms) is False
-def test_undocumented_repository_yields_nothing_and_says_so(tmp_path: Path) -> None:
+async def test_undocumented_repository_yields_nothing_and_says_so(tmp_path: Path) -> None:
reset_house_terms()
run = _run(repo_path=_repo(tmp_path, documented=False))
with capture_logs() as logs:
- signals = _signals_from_level8(run)
+ signals = await _signals_from_level8(run)
assert signals.house_terms == ()
assert any(entry["event"] == "onboarding.house_terms_empty" for entry in logs)
assert house_terms_mined().mined is True
-def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:
+async def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:
reset_house_terms()
run = _run(repo_path=None)
with capture_logs() as logs:
- signals = _signals_from_level8(run)
+ signals = await _signals_from_level8(run)
assert signals.house_terms == ()
skipped = [e for e in logs if e["event"] == "onboarding.house_terms_skipped"]
@@ -255,9 +260,9 @@ def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:
# ---------------------------------------------------------------------------
-def test_the_count_and_the_top_terms_reach_the_report(tmp_path: Path) -> None:
+async def test_the_count_and_the_top_terms_reach_the_report(tmp_path: Path) -> None:
reset_house_terms()
- signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+ signals = await _signals_from_level8(_run(repo_path=_repo(tmp_path)))
report = GenerationReport.from_pages([])
assert report.house_terms.mined is True
@@ -280,10 +285,10 @@ def test_the_report_keeps_nothing_read_apart_from_nothing_found() -> None:
assert "not measured" not in empty.summary_line()
-def test_a_new_run_does_not_report_the_previous_run_s_vocabulary(tmp_path: Path) -> None:
+async def test_a_new_run_does_not_report_the_previous_run_s_vocabulary(tmp_path: Path) -> None:
"""The reset lives on ``generate_all``; this pins what it is there for."""
reset_house_terms()
- _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+ await _signals_from_level8(_run(repo_path=_repo(tmp_path)))
assert house_terms_mined().count > 0
reset_house_terms()
diff --git a/tests/unit/generation/test_overview_capability_table.py b/tests/unit/generation/test_overview_capability_table.py
index d425653c5..6e4744c87 100644
--- a/tests/unit/generation/test_overview_capability_table.py
+++ b/tests/unit/generation/test_overview_capability_table.py
@@ -122,7 +122,7 @@ def test_a_term_with_no_path_anywhere_is_not_offered():
def test_selection_is_logged_with_what_it_kept_and_what_it_dropped():
with capture_logs() as logs:
select_capabilities([_term("DONE"), _term("Dead code")], MODULES)
- event = next(e for e in logs if e["event"] == "overview_capabilities_selected")
+ event = next(e for e in logs if e["event"] == "house_vocabulary.selected")
assert event["mined"] == 2
assert event["corroborated"] == 1
assert event["terms"] == ["Dead code"]
@@ -134,7 +134,7 @@ def test_mining_terms_that_no_module_page_corroborates_is_a_warning():
corroboration corpus looks like, so the counts that separate them go out."""
with capture_logs() as logs:
select_capabilities([_term("DONE")], MODULES)
- assert any(e["event"] == "overview_capabilities_uncorroborated" for e in logs)
+ assert any(e["event"] == "house_vocabulary.uncorroborated" for e in logs)
# ---------------------------------------------------------------------------
@@ -182,20 +182,28 @@ def test_prose_that_is_not_a_statement_is_not_offered_as_a_definition(text):
@pytest.mark.parametrize(
- "text",
+ "term,text",
[
- "Dead code is code no import path reaches.",
- "Blast radius is the set of files a change can reach through the import graph.",
- "Middleware for utilizing Web-server-provided authentication.",
+ ("Dead code", "Dead code is code no import path reaches."),
+ (
+ "Blast radius",
+ "Blast radius is the set of files a change can reach through the import graph.",
+ ),
+ ("Middleware", "Middleware for utilizing Web-server-provided authentication."),
# Terse, and still the repository's own answer. Both of these were
# rejected by a first cut of the test that was tuned too strict.
- "Blast-radius request/response models.",
- "Decisions are co-located in the page vector store under the "
- "``decision:<record_id>`` namespace (no separate table).",
+ ("Blast radius", "Blast-radius request/response models."),
+ (
+ "Decisions",
+ "Decisions are co-located in the page vector store under the "
+ "``decision:<record_id>`` namespace (no separate table).",
+ ),
],
)
-def test_a_real_sentence_survives(text):
- picked = select_capabilities([_term("Dead code", definition=text)], MODULES)
+def test_a_real_sentence_survives(term, text):
+ """Each sentence is paired with the term it names, because a definition
+ that never mentions its term is rejected as prose that merely sat nearby."""
+ picked = select_capabilities([_term(term, definition=text)], [*MODULES, term])
assert picked[0].definition == text
@@ -220,7 +228,7 @@ def test_a_rejected_definition_does_not_leave_its_source_behind():
def test_a_rejected_definition_is_logged():
with capture_logs() as logs:
select_capabilities([_term("Dead code", definition="$ repowise dead-code")], MODULES)
- assert any(e["event"] == "overview_capability_definition_rejected" for e in logs)
+ assert any(e["event"] == "house_vocabulary.definition_rejected" for e in logs)
def test_a_term_the_repository_never_defined_still_gets_a_row():
@@ -252,8 +260,9 @@ def test_a_pipe_in_mined_prose_does_not_break_the_table():
def test_a_long_definition_is_cut_rather_than_wrapping_the_row():
+ long_sentence = "Dead code is " + "unreachable " * 60 + "code."
table = build_capability_table(
- select_capabilities([_term("Dead code", definition="word " * 100)], MODULES)
+ select_capabilities([_term("Dead code", definition=long_sentence)], MODULES)
)
row = next(line for line in table.splitlines() if line.startswith("| Dead code"))
assert "…" in row
diff --git a/tests/unit/generation/test_overview_capability_wiring.py b/tests/unit/generation/test_overview_capability_wiring.py
index e04baf99f..e1852a75b 100644
--- a/tests/unit/generation/test_overview_capability_wiring.py
+++ b/tests/unit/generation/test_overview_capability_wiring.py
@@ -292,7 +292,7 @@ async def test_the_repository_is_mined_once_for_both_levels(tmp_path):
"""Mining walks the whole repository. Two consumers, one walk."""
run = _run(tmp_path)
await build_level6_coros(run)
- build_level8_coros(run)
+ await build_level8_coros(run)
from_overview = tuple(c.term for c in run.gen.overview_kwargs["capabilities"])
onboarding_terms = run.gen.onboarding_signals[0].house_terms
@@ -311,9 +311,54 @@ async def test_onboarding_still_gets_its_terms_when_the_overview_mined_first(tmp
from a warm cache rather than from a fresh walk."""
run = _run(tmp_path)
await build_level6_coros(run)
- build_level8_coros(run)
+ await build_level8_coros(run)
assert [t.term for t in run.gen.onboarding_signals[0].house_terms] == [
"Blast radius",
"Change risk",
]
+
+
+# ---------------------------------------------------------------------------
+# The glossary reads the same corroboration corpus
+# ---------------------------------------------------------------------------
+#
+# It is the second consumer of both mined vocabulary and module corroboration.
+# Both cost a pass the run should pay once: mining walks the whole repository,
+# and corroboration reads module summaries out of the store.
+
+
+async def test_the_glossary_is_handed_the_corroboration_corpus(tmp_path):
+ run = _run(tmp_path)
+ await build_level8_coros(run)
+
+ corpus = run.gen.onboarding_signals[0].module_corroboration
+ assert corpus
+ # Title first, summary under it -- what makes a match mean something.
+ assert corpus[0].startswith("Blast Radius Evaluation")
+
+
+async def test_the_corroboration_corpus_is_built_once_for_both_levels(tmp_path):
+ """Level 6 and level 8 both want it, and it costs a batched store read."""
+ store = _Store({"src": "Evaluates the blast radius of a change."})
+ run = _run(tmp_path, written={}, store=store)
+
+ await build_level6_coros(run)
+ asked_after_overview = list(store.asked)
+ await build_level8_coros(run)
+
+ assert asked_after_overview, "the overview should have read the store"
+ assert store.asked == asked_after_overview, "level 8 read the store a second time"
+
+
+async def test_a_run_with_no_glossary_page_does_not_build_the_corpus(tmp_path):
+ """A scoped run that asked for one onboarding page should not pay for a
+ store read that only the glossary consumes."""
+ store = _Store({"src": "Evaluates the blast radius of a change."})
+ run = _run(tmp_path, store=store)
+ run._emit = lambda page_id: "glossary" not in page_id
+
+ await build_level8_coros(run)
+
+ assert store.asked == []
+ assert all(s.module_corroboration == () for s in run.gen.onboarding_signals)
diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.py
index dbb131268..0246f93aa 100644
--- a/tests/unit/generation/test_vocabulary_house_terms.py
+++ b/tests/unit/generation/test_vocabulary_house_terms.py
@@ -825,3 +825,38 @@ def test_extract_terms_does_not_read_txt_documents(txt_docs_repo: Path) -> None:
"""The planner's input does not move. It mines markdown only, and the
binding it performs is measured against exactly that corpus."""
assert extract_terms(txt_docs_repo) == []
+
+
+def test_a_hard_wrapped_markdown_sentence_is_read_whole(tmp_path: Path) -> None:
+ """A README wraps at the column as readily as a ``.txt`` does.
+
+ The reStructuredText scan has rejoined wrapped lines since it learned to;
+ the markdown scan read one line and returned the author's sentence cut
+ where their editor wrapped it. That fragment then trails off mid-thought,
+ which every consumer renders as prose the repository wrote.
+ """
+ root = tmp_path / "ledger"
+ (root / "src").mkdir(parents=True)
+ (root / "README.md").write_text(
+ "# Ledger\n"
+ "\n"
+ "## Blast radius\n"
+ "\n"
+ "Blast radius is the set of accounts a posting can reach through the\n"
+ "ledger graph.\n"
+ "\n"
+ "## Dead code\n"
+ "\n"
+ "Dead code is a rule no posting path reaches.\n",
+ encoding="utf-8",
+ )
+ (root / "src" / "engine.py").write_text(
+ '"""Blast radius and dead code for the ledger."""\n', encoding="utf-8"
+ )
+
+ by_term = {t.term: t for t in extract_house_terms(root)}
+ assert by_term["Blast radius"].definition == (
+ "Blast radius is the set of accounts a posting can reach through the ledger graph."
+ )
+ # A sentence that already fits on one line is unaffected.
+ assert by_term["Dead code"].definition == "Dead code is a rule no posting path reaches."