Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): mine the repository's own vocabulary, ranked and gated (#1233)

Swati Ahujaf10bc533c722
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
index 71f748a3f..50c4ffe98 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -22,11 +22,18 @@ enough that a model adds cost and variance without adding much.
from __future__ import annotations
+import logging
import re
+from collections.abc import Callable
+from dataclasses import dataclass
from pathlib import Path
+from repowise.core import fs_walk
+
from .grouping import ConceptGroup
+logger = logging.getLogger(__name__)
+
#: Documents worth mining, relative to the repository root. A doc that names a
#: subsystem in a heading is a much stronger signal than one that mentions it
#: in a sentence, so headings are what this reads.
@@ -56,15 +63,69 @@ _BOLD_TERM = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*")
_STOPWORDS = frozenset(
{
- "the", "a", "an", "and", "or", "of", "for", "to", "in", "on", "with",
- "how", "what", "why", "when", "is", "are", "it", "this", "that", "you",
- "your", "we", "our", "using", "use", "usage", "getting", "started",
- "installation", "install", "quickstart", "quick", "start", "license",
- "contributing", "changelog", "table", "contents", "overview",
- "introduction", "intro", "example", "examples", "faq", "notes",
- "requirements", "setup", "configuration", "config", "options", "api",
- "reference", "guide", "tutorial", "docs", "documentation", "features",
- "roadmap", "credits", "acknowledgements", "badges", "support",
+ "the",
+ "a",
+ "an",
+ "and",
+ "or",
+ "of",
+ "for",
+ "to",
+ "in",
+ "on",
+ "with",
+ "how",
+ "what",
+ "why",
+ "when",
+ "is",
+ "are",
+ "it",
+ "this",
+ "that",
+ "you",
+ "your",
+ "we",
+ "our",
+ "using",
+ "use",
+ "usage",
+ "getting",
+ "started",
+ "installation",
+ "install",
+ "quickstart",
+ "quick",
+ "start",
+ "license",
+ "contributing",
+ "changelog",
+ "table",
+ "contents",
+ "overview",
+ "introduction",
+ "intro",
+ "example",
+ "examples",
+ "faq",
+ "notes",
+ "requirements",
+ "setup",
+ "configuration",
+ "config",
+ "options",
+ "api",
+ "reference",
+ "guide",
+ "tutorial",
+ "docs",
+ "documentation",
+ "features",
+ "roadmap",
+ "credits",
+ "acknowledgements",
+ "badges",
+ "support",
}
)
@@ -76,10 +137,39 @@ _MAX_TERM_WORDS = 4
# real headings from this repository's own docs, and neither is a subsystem.
_SENTENCE_VERBS = frozenset(
{
- "is", "are", "was", "were", "has", "have", "had", "do", "does", "did",
- "can", "will", "would", "should", "must", "may", "updated", "added",
- "removed", "fixed", "works", "means", "needs", "gets", "goes", "runs",
- "returns", "uses", "supports", "requires", "shows", "makes", "keeps",
+ "is",
+ "are",
+ "was",
+ "were",
+ "has",
+ "have",
+ "had",
+ "do",
+ "does",
+ "did",
+ "can",
+ "will",
+ "would",
+ "should",
+ "must",
+ "may",
+ "updated",
+ "added",
+ "removed",
+ "fixed",
+ "works",
+ "means",
+ "needs",
+ "gets",
+ "goes",
+ "runs",
+ "returns",
+ "uses",
+ "supports",
+ "requires",
+ "shows",
+ "makes",
+ "keeps",
}
)
@@ -115,15 +205,38 @@ def _is_useful(term: str) -> bool:
return len(words) == 1 or words[0][:1].isupper()
-def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:
- """Lift candidate subsystem names from the repository's own documentation.
+@dataclass(frozen=True)
+class HouseTerm:
+ """A term the repository uses for itself, with what it was read from.
- Returns terms in document order with duplicates removed. Never raises: a
- repository with no docs returns an empty list, which is a supported and
- common outcome, not a degraded one.
+ ``definition`` is the repository's own sentence about the term — the line
+ following the heading that names it, or the clause after a bolded lead-in.
+ It is ``None`` far more often than not; a term with no sentence near it is
+ still a term, and inventing one would be the whole failure this module
+ exists to avoid.
"""
- seen: set[str] = set()
- terms: list[str] = []
+
+ term: str
+ definition: str | None
+ definition_source: str | None
+ #: Repository-relative POSIX paths of every document that names the term,
+ #: in the order they were read.
+ source_paths: tuple[str, ...]
+ #: How many distinct documents name it. This is the primary ranking
+ #: signal: "how often do we write about this".
+ doc_frequency: int
+ #: How many source files use it in their own prose. This is the gate:
+ #: "was it built". It is deliberately not added to ``doc_frequency`` —
+ #: see :func:`extract_house_terms`.
+ code_frequency: int
+ #: Whether the codebase has a symbol by this name. A term that is also a
+ #: symbol can be rendered in backticks; a coined one cannot, because the
+ #: grounding pass strips backticks off tokens it cannot resolve.
+ is_indexed_symbol: bool
+
+
+def _doc_paths(repo_root: Path) -> list[Path]:
+ """The documents worth mining, in a fixed order. Never raises."""
files: list[Path] = []
try:
for name in DOC_FILES:
@@ -142,24 +255,576 @@ def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:
if len(files) >= _MAX_DOCS:
break
except OSError:
+ logger.warning(
+ "vocabulary: could not list documents under %s; mining no terms for this repository",
+ repo_root,
+ )
return []
+ return files
+
+
+# A line that opens a new section. The definition scan stops here: running on
+# into the next section hands back a sentence about a different subject, which
+# reads as authoritative and is wrong.
+_HEADING_LINE = re.compile(r"^\s{0,3}#{1,4}\s+\S")
+# Lines that are structure rather than prose — tables, fences, lists, quotes.
+_NOT_PROSE = ("|", "```", "-", "*", ">", "!")
+# Shorter than this is a lead-in ("See below.", "Two parts:"), not a meaning.
+_MIN_DEFINITION_CHARS = 20
+_MAX_DEFINITION_CHARS = 300
+# How far past a heading to look for the sentence that explains it.
+_DEFINITION_SCAN_LINES = 5
+
+_SENTENCE_END = re.compile(r"(?<=[.!?])(?:\s|$)")
+
+# A bolded lead-in that carries its own definition: "**Blast radius** — ...".
+_BOLD_DEFINITION = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*\s*[—–:-]\s*([^\n]{10,300})")
+
+
+def _first_sentence(line: str) -> str:
+ match = _SENTENCE_END.search(line)
+ return (line[: match.start() + 1] if match else line).strip()
+
+
+def _definition_after_heading(text: str, offset: int) -> str | None:
+ """The first prose sentence between a heading and the next one.
+
+ ``offset`` is the *end* of the heading text, not its start. ``_HEADING``
+ opens with ``\\s{0,3}``, which in multiline mode lets a match begin on the
+ blank line above; anchoring on the end sidesteps that entirely.
+ """
+ lines = text[offset:].split("\n")[1 : _DEFINITION_SCAN_LINES + 1]
+ for line in lines:
+ stripped = line.strip()
+ if not stripped:
+ continue
+ if _HEADING_LINE.match(line):
+ return None
+ if stripped.startswith(_NOT_PROSE):
+ continue
+ sentence = _first_sentence(stripped)
+ if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:
+ return sentence
+ return None
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Which documents to read
+# ---------------------------------------------------------------------------
- for path in files:
+#: Files whose headings are version numbers and shipped-feature blurbs rather
+#: than subsystem names. A changelog is usually the largest document in a
+#: repository and the least useful one to mine: on this repository it consumed
+#: 142 of the first 200 term slots before the tool guide was ever opened.
+_RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst)$", re.I)
+_VERSION_HEADING = re.compile(r"^v?\d+\.\d+")
+#: A document has to be overwhelmingly version headings before it is dropped. A
+#: guide that cites a few release numbers is not release notes, and dropping a
+#: large guide by accident loses more vocabulary than a changelog ever adds.
+_RELEASE_NOTE_HEADING_RATIO = 0.6
+_MIN_HEADINGS_TO_JUDGE = 8
+
+
+def _is_release_notes(path: Path, text: str) -> bool:
+ if _RELEASE_NOTE_NAMES.search(path.name):
+ return True
+ headings = _HEADING.findall(text)
+ if len(headings) < _MIN_HEADINGS_TO_JUDGE:
+ return False
+ versiony = sum(1 for h in headings if _VERSION_HEADING.match(h.strip()))
+ return versiony / len(headings) > _RELEASE_NOTE_HEADING_RATIO
+
+
+# ---------------------------------------------------------------------------
+# Which headings name something
+# ---------------------------------------------------------------------------
+
+#: A pronoun makes a heading a sentence about the reader or about us, never a
+#: name: "Your agent stops guessing", "See all of it", "Who it's for". This is
+#: grammar rather than a list of words we happened to dislike, so it carries
+#: to a repository whose marketing copy reads nothing like ours.
+_PRONOUN = re.compile(
+ r"\b(you|your|yours|i|me|my|we|us|our|ours|it|its|they|them|their|theirs|he|she|him|her|his|hers)\b",
+ re.I,
+)
+
+#: Numbering a heading carries ("2 · Distill: command-output compression",
+#: "3. Ingestion") is document structure, not part of the name.
+_ENUMERATOR = re.compile(r"^\s*\d+[.)]?\s*[·•\-–—]?\s+")
+#: A heading of the form "Name: what it does" names the thing before the colon
+#: and explains it after. Both halves are worth offering as candidates; the
+#: lead is usually the term.
+_COLON_SPLIT = re.compile(r"^([^:]{2,40}):\s+(\S.*)$")
+
+
+def _is_name_like(term: str, *, excluded: frozenset[str]) -> bool:
+ """Whether a heading names a thing, as opposed to claiming something.
+
+ Applied on top of :func:`_is_useful`, which already rejects the obvious
+ sentence shapes. This adds the two tests that matter for a term list that
+ gets rendered: no pronouns, and not the repository's own name — which
+ leads every frequency count in every repository and teaches nobody
+ anything.
+ """
+ if term.lower() in excluded:
+ return False
+ return not _PRONOUN.search(term)
+
+
+def _expand_heading(raw: str) -> list[str]:
+ """The spellings of a heading worth considering as a name.
+
+ A numbered heading yields its text without the number; a "Name: gloss"
+ heading yields the name. Both are offered alongside the original, and
+ :func:`_is_useful` decides which survive — a heading carrying a digit is
+ rejected outright, so without stripping the enumerator a numbered section
+ contributes nothing at all.
+ """
+ spellings = [raw]
+ stripped = _ENUMERATOR.sub("", raw).strip()
+ if stripped and stripped != raw:
+ spellings.append(stripped)
+ for candidate in list(spellings):
+ colon = _COLON_SPLIT.match(candidate)
+ if colon:
+ lead = colon.group(1).strip()
+ if lead and lead not in spellings:
+ spellings.append(lead)
+ return spellings
+
+
+def _repo_own_names(repo_root: Path) -> frozenset[str]:
+ """What this repository calls itself, read from the repository."""
+ names = {repo_root.name.lower()}
+ for name in list(names):
+ names.add(name.replace("-", " "))
+ names.add(name.replace("_", " "))
+ names.add(name.replace("-", ""))
+ return frozenset(n for n in names if n)
+
+
+@dataclass
+class _Candidate:
+ """A term under construction, accumulating across documents."""
+
+ term: str
+ definition: str | None = None
+ definition_source: str | None = None
+ source_paths: list[str] | None = None
+
+
+def _harvest(
+ repo_root: Path,
+ *,
+ limit: int | None = None,
+ expand: Callable[[str], list[str]] | None = None,
+ accept: Callable[[str], bool] | None = None,
+ skip_release_notes: bool = False,
+) -> list[_Candidate]:
+ """Every candidate term in document order, deduplicated, first spelling wins.
+
+ ``limit`` stops the walk early, so a caller wanting the first N terms does
+ not pay to read documents it will discard.
+
+ The three hooks are how the ranked view asks for more than the planner
+ does without moving the planner's input. Called with none of them this is
+ the harvest ``extract_terms`` has always performed, term for term.
+
+ ``expand`` turns one raw heading into several candidate spellings.
+ ``accept`` is an extra test applied after :func:`_is_useful`.
+ ``skip_release_notes`` drops documents whose headings are version numbers.
+ """
+ candidates: dict[str, _Candidate] = {}
+ order: list[str] = []
+ unreadable = 0
+ skipped: list[str] = []
+
+ for path in _doc_paths(repo_root):
try:
text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]
except OSError:
+ unreadable += 1
+ logger.warning("vocabulary: could not read %s; skipping it", path)
+ continue
+ if skip_release_notes and _is_release_notes(path, text):
+ skipped.append(path.name)
+ continue
+ try:
+ rel = path.relative_to(repo_root).as_posix()
+ except ValueError: # pragma: no cover — path came from repo_root
+ rel = path.name
+
+ # Definitions found for terms seen in *this* document, keyed the same
+ # way as candidates so a repeat mention can fill a gap left earlier.
+ definitions = {
+ _normalise(m.group(1)).lower(): m.group(2).strip()
+ for m in _BOLD_DEFINITION.finditer(text)
+ }
+
+ # Headings first, then bolded lead-ins — the order the planner's
+ # binding was measured against. Which of the two a term came from
+ # decides where its definition may be looked for: the sentence under a
+ # heading belongs to that heading, but the lines under a bolded term
+ # mid-paragraph belong to the paragraph, not to the term.
+ matches = [(True, m) for m in _HEADING.finditer(text)]
+ matches += [(False, m) for m in _BOLD_TERM.finditer(text)]
+ for is_heading, match in matches:
+ # Expansion runs on the raw heading, before normalisation:
+ # normalising strips the punctuation — the colon, the bullet —
+ # that says where the name ends and the gloss begins.
+ raw = match.group(1)
+ spellings = expand(raw) if expand is not None else [raw]
+ for spelling in spellings:
+ term = _normalise(spelling)
+ if not _is_useful(term):
+ continue
+ if accept is not None and not accept(term):
+ continue
+ key = term.lower()
+ candidate = candidates.get(key)
+ if candidate is None:
+ if limit is not None and len(order) >= limit:
+ return [candidates[k] for k in order]
+ candidate = _Candidate(term=term, source_paths=[])
+ candidates[key] = candidate
+ order.append(key)
+ assert candidate.source_paths is not None
+ if rel not in candidate.source_paths:
+ candidate.source_paths.append(rel)
+ if candidate.definition is None:
+ definition = definitions.get(key)
+ if not definition and is_heading:
+ definition = _definition_after_heading(text, match.end())
+ if definition:
+ candidate.definition = definition
+ candidate.definition_source = rel
+
+ if unreadable:
+ logger.warning(
+ "vocabulary: %d of the repository's documents could not be read",
+ unreadable,
+ )
+ if skipped:
+ logger.info(
+ "vocabulary: skipped %d release-note document(s): %s",
+ len(skipped),
+ ", ".join(sorted(skipped)),
+ )
+ return [candidates[k] for k in order]
+
+
+# ---------------------------------------------------------------------------
+# What the code says about itself
+# ---------------------------------------------------------------------------
+
+#: Directories that hold somebody else's code on top of what the shared
+#: walker already prunes. Their prose is about their subsystems, not ours.
+_EXTRA_PRUNED_DIRS = fs_walk.PRUNED_DIRS_DERIVED | frozenset(
+ {"vendor", "third_party", "thirdparty", "site-packages"}
+)
+
+#: Python docstrings, and the block comments JSDoc and TSDoc are written in.
+#: Both are prose a maintainer wrote about a unit of code, which is what makes
+#: them the right place to ask whether a term names something that was built.
+#: Reading only Python here would score every TypeScript-first repository zero
+#: and reject its entire vocabulary.
+_SOURCE_PROSE = {
+ ".py": re.compile(r'(?:^|\n)\s*(?:[rubfRUBF]{0,2})"""(.*?)"""', re.S),
+ ".js": re.compile(r"/\*\*(.*?)\*/", re.S),
+}
+for _ext in (".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts"):
+ _SOURCE_PROSE[_ext] = _SOURCE_PROSE[".js"]
+
+_MAX_SOURCE_FILES = 6_000
+_MAX_SOURCE_BYTES = 60_000
+_MAX_PROSE_BLOCKS = 8
+#: Leading ``*`` on every line of a JSDoc block is layout, not text.
+_JSDOC_GUTTER = re.compile(r"^\s*\*[ \t]?", re.M)
+
+
+def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:
+ """One pruned pass over the repository, answering both tree questions.
+
+ Returns ``(prose, directory_names)`` — ``(path, prose)`` for every source
+ file that documents itself, and every name the repository has given a
+ directory. Both come from the same walk because both need the whole tree
+ and the walk is the expensive part.
+
+ Never raises: an unreadable file is one fewer signal, not a failed
+ generation.
+ """
+ prose: list[tuple[str, str]] = []
+ dir_names: set[str] = set()
+ unreadable = 0
+ truncated = False
+
+ for dirpath, dirnames, filenames in fs_walk.walk_repo(repo_root, prune_dirs=_EXTRA_PRUNED_DIRS):
+ for name in dirnames:
+ dir_names.add(_singular(name.lower()))
+ if truncated:
continue
- for match in list(_HEADING.finditer(text)) + list(_BOLD_TERM.finditer(text)):
- term = _normalise(match.group(1))
- if not _is_useful(term):
+ for name in sorted(filenames):
+ pattern = _SOURCE_PROSE.get(Path(name).suffix)
+ if pattern is None:
+ continue
+ if len(prose) >= _MAX_SOURCE_FILES:
+ # A floor rather than a count. Said out loud, because a
+ # silently truncated corpus reads downstream as "the code
+ # does not use this term" — which is the gate's reject.
+ logger.info(
+ "vocabulary: stopped reading source prose at %d files; "
+ "term code frequencies are a floor, not a count",
+ _MAX_SOURCE_FILES,
+ )
+ truncated = True
+ break
+ path = dirpath / name
+ try:
+ text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_SOURCE_BYTES]
+ except OSError:
+ unreadable += 1
continue
- key = term.lower()
- if key in seen:
+ blocks = pattern.findall(text)[:_MAX_PROSE_BLOCKS]
+ if not blocks:
continue
- seen.add(key)
- terms.append(term)
- if len(terms) >= max_terms:
- return terms
+ joined = "\n".join(_JSDOC_GUTTER.sub("", b) for b in blocks)
+ try:
+ rel = path.relative_to(repo_root).as_posix()
+ except ValueError: # pragma: no cover — path came from repo_root
+ rel = path.name
+ prose.append((rel, joined))
+
+ if unreadable:
+ logger.warning("vocabulary: %d source files could not be read for prose", unreadable)
+ return prose, frozenset(dir_names)
+
+
+#: The gaps between a term's words. Prose writes "co-change", an identifier
+#: writes "co_change" and a heading writes "Co change"; they are one term.
+_WORD_GAP = re.compile(r"[\s_\-]+")
+
+
+def _term_words(term: str) -> list[str]:
+ return [w for w in _WORD_GAP.split(term) if w]
+
+
+def _phrase_pattern(term: str) -> re.Pattern[str]:
+ """Match a term however the code spells the gaps between its words.
+
+ ``bug magnet`` has to find ``bug magnet``, ``bug-magnet`` and
+ ``bug_magnet``, because the docs write it one way and the code the other.
+ """
+ words = [re.escape(w) for w in _term_words(term)]
+ return re.compile(r"\b" + r"[\s_\-]+".join(words) + r"\b", re.I)
+
+
+def _definition_from_prose(prose: str, pattern: re.Pattern[str]) -> str | None:
+ """A sentence in code prose that opens by naming the term.
+
+ A sentence leading with the term is usually the best definition in the
+ repository — better than a heading's follow-on line, which is often a
+ lead-in rather than a meaning. A sentence that merely mentions it is not a
+ definition and is not taken.
+ """
+ for raw in re.split(r"(?<=[.!?])\s+", prose):
+ sentence = " ".join(raw.split())
+ if not (_MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS):
+ continue
+ if pattern.match(sentence):
+ return sentence
+ return None
+
+
+def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:
+ """Lift candidate subsystem names from the repository's own documentation.
+
+ Returns terms in document order with duplicates removed. Never raises: a
+ repository with no docs returns an empty list, which is a supported and
+ common outcome, not a degraded one.
+ """
+ return [c.term for c in _harvest(repo_root, limit=max_terms)]
+
+
+#: How many source paths to keep on a term. A term used by 300 files does not
+#: need 300 citations; the count is kept in full on ``code_frequency``.
+_MAX_CODE_PATHS = 40
+
+
+def _warn_empty(repo_root: Path) -> None:
+ """Nothing was read at all — as opposed to read and rejected."""
+ logger.warning(
+ "vocabulary: no house terms found under %s — no readable %s and no "
+ "top-level docs directory",
+ repo_root,
+ "/".join(DOC_FILES),
+ )
+
+
+def _survives_single_word_test(term: str, dir_names: frozenset[str]) -> bool:
+ """Whether a one-word term is a name rather than an ordinary word.
+
+ A single common English word is never a house term however often it
+ appears — "Code", "Response", "Query" and "Bare" all rank near the top on
+ raw frequency and none of them names a subsystem. A single word earns its
+ place when its *shape* marks it as a name (an acronym like MCP or FTS, an
+ internal capital like PageRank), or when the codebase has named a whole
+ directory after it.
+
+ The directory test is deliberately a match against the **whole** directory
+ name rather than against the words inside one. Matching word-by-word looks
+ stricter than it is: a repository of a few thousand files spells almost
+ every common English word somewhere inside some path, so the test passes
+ everything and stops nothing. "dead_code" must not be what lets "Code"
+ through.
+ """
+ words = _term_words(term)
+ if len(words) != 1:
+ return True
+ word = words[0]
+ acronym = word.isupper() and 2 <= len(word) <= 5
+ internal_capital = any(c.isupper() for c in word[1:])
+ return acronym or internal_capital or _singular(word.lower()) in dir_names
+
+
+def _singular(word: str) -> str:
+ if word.endswith("ies") and len(word) > 4:
+ return word[:-3] + "y"
+ if word.endswith("es") and len(word) > 4:
+ return word[:-2]
+ if word.endswith("s") and not word.endswith("ss") and len(word) > 3:
+ return word[:-1]
+ return word
+
+
+def extract_house_terms(
+ repo_root: Path,
+ *,
+ max_terms: int = 200,
+ known_symbols: frozenset[str] | set[str] | None = None,
+) -> list[HouseTerm]:
+ """The repository's own words for its own subsystems, ranked and gated.
+
+ The same harvest :func:`extract_terms` performs, plus three things it does
+ not need and a rendered term list cannot do without.
+
+ **Release notes are skipped.** Their headings are version numbers, and a
+ changelog is usually the largest document in the repository.
+
+ **Two frequency signals, kept apart on purpose.** ``doc_frequency`` counts
+ the documents that name the term — "is this something we write about",
+ and the ranking key. ``code_frequency`` counts the source files whose own
+ prose uses it — "was it built", and the gate. Adding them together lets a
+ common English word appearing in 180 docstrings outrank the name of an
+ actual subsystem; ranking on one and gating on the other does not. The
+ gate is the same bind-or-drop discipline the planner already applies: a
+ marketed term with nothing behind it does not survive.
+
+ **A term must look like a name.** No pronouns, no repository name, and a
+ single word must be shaped like one or be spelled in a directory.
+
+ ``known_symbols`` are the names the codebase defines; a term is marked
+ ``is_indexed_symbol`` when it matches one. Pass nothing and every term
+ reports ``False``, which is the safe direction — an unbackticked term
+ renders as plain prose, a wrongly-backticked one is silently demoted.
+ """
+ symbols = {s.lower() for s in known_symbols} if known_symbols else set()
+ excluded = _repo_own_names(repo_root)
+
+ candidates = _harvest(
+ repo_root,
+ limit=max_terms,
+ expand=_expand_heading,
+ accept=lambda t: _is_name_like(t, excluded=excluded),
+ skip_release_notes=True,
+ )
+ if not candidates:
+ _warn_empty(repo_root)
+ return []
+
+ patterns = {c.term.lower(): _phrase_pattern(c.term) for c in candidates}
+ code_files: dict[str, list[str]] = {c.term.lower(): [] for c in candidates}
+ code_definitions: dict[str, tuple[str, str]] = {}
+ prose_corpus, dir_names = _scan_tree(repo_root)
+ for rel, prose in prose_corpus:
+ for key, pattern in patterns.items():
+ if not pattern.search(prose):
+ continue
+ code_files[key].append(rel)
+ if key not in code_definitions:
+ sentence = _definition_from_prose(prose, pattern)
+ if sentence:
+ code_definitions[key] = (sentence, rel)
+
+ terms: list[HouseTerm] = []
+ dropped_unbuilt = 0
+ dropped_common_word = 0
+ for candidate in candidates:
+ key = candidate.term.lower()
+ hits = code_files[key]
+ if not hits:
+ dropped_unbuilt += 1
+ continue
+ if not _survives_single_word_test(candidate.term, dir_names):
+ dropped_common_word += 1
+ continue
+ definition = candidate.definition
+ definition_source = candidate.definition_source
+ if definition is None and key in code_definitions:
+ definition, definition_source = code_definitions[key]
+ doc_paths = tuple(candidate.source_paths or ())
+ terms.append(
+ HouseTerm(
+ term=candidate.term,
+ definition=definition,
+ definition_source=definition_source,
+ source_paths=doc_paths + tuple(hits[:_MAX_CODE_PATHS]),
+ doc_frequency=len(doc_paths),
+ code_frequency=len(hits),
+ is_indexed_symbol=key in symbols,
+ )
+ )
+
+ # How many documents name it, then multi-word terms ahead of single-word
+ # ones, then how much of the code uses it.
+ #
+ # The middle key is the one doing the work. Document frequency is the
+ # right primary signal and it discriminates poorly in practice: most
+ # repositories have two or three documents worth mining, so nearly every
+ # term ties at one and the sort falls through. Word count breaks that tie
+ # in the right direction, because a subsystem is almost always named with
+ # two words ("blast radius", "dead code", "change risk") and an ordinary
+ # English word is almost always one.
+ terms.sort(
+ key=lambda t: (
+ -t.doc_frequency,
+ len(_term_words(t.term)) == 1,
+ -t.code_frequency,
+ t.term.lower(),
+ )
+ )
+ logger.info(
+ "vocabulary: %d house terms from %d candidates "
+ "(%d named but not built, %d ordinary words), %d source files read",
+ len(terms),
+ len(candidates),
+ dropped_unbuilt,
+ dropped_common_word,
+ len(prose_corpus),
+ )
+ if not terms:
+ # Distinct from having nothing to read: the documents were read and
+ # every term in them failed the gate. Usually that means the source
+ # prose was not found — a language this cannot read, or a layout where
+ # the code sits outside the walked tree.
+ logger.warning(
+ "vocabulary: no house terms found under %s — %d candidates were "
+ "read from the documents and none survived; %d source files "
+ "contributed prose",
+ repo_root,
+ len(candidates),
+ len(prose_corpus),
+ )
return terms
diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.py
new file mode 100644
index 000000000..176b4660e
--- /dev/null
+++ b/tests/unit/generation/test_vocabulary_house_terms.py
@@ -0,0 +1,482 @@
+"""The vocabulary miner's two contracts.
+
+``extract_terms`` feeds the concept planner, whose binding is measured to be
+correct. Its output is therefore pinned byte-for-byte against a fixture
+repository: any change to the harvest that moves a single term is a change to
+which pages the planner emits, and must be seen rather than discovered later.
+
+``extract_house_terms`` is the richer view — same candidates, plus the sentence
+that defines each one and where it was read from. Nothing consumes it yet.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+import pytest
+
+from repowise.core.generation.concept_tree.vocabulary import (
+ extract_house_terms,
+ extract_terms,
+)
+
+#: Named rather than left to the root logger: a suite-wide run may have raised
+#: the level on an ancestor, and the assertion is about this module reporting
+#: its own failures, not about global logging configuration.
+_LOGGER = "repowise.core.generation.concept_tree.vocabulary"
+
+# ---------------------------------------------------------------------------
+# The fixture repository
+# ---------------------------------------------------------------------------
+
+_README = """\
+# Ledger
+
+A ledger for small shops.
+
+## Blast radius
+
+Blast radius is the set of files a change can reach through the import graph.
+
+## Change risk
+
+Change risk scores a diff against the history of the files it touches.
+
+### Your ledger stops guessing
+
+Marketing copy, not a subsystem.
+
+## Getting Started
+
+Boilerplate heading, says nothing about this repository.
+
+## Local paths are masked
+
+A sentence, not a name.
+
+## Headline facts updated 2026-06-22
+
+A dated note.
+
+**Dead code** — code no import path reaches.
+
+**Co-change** is how often two files land in the same commit.
+
+## 2 · Reconciliation: matching two ledgers
+
+Reconciliation pairs a bank line against a book line.
+
+## Hotspots
+
+## Response
+"""
+
+_ARCHITECTURE = """\
+# Architecture
+
+## Blast radius
+
+Repeated on purpose: the term is named by two documents.
+
+## Ingestion pipeline
+
+The ingestion pipeline walks the working tree and parses each file.
+"""
+
+_DOCS_GUIDE = """\
+# Query guide
+
+## Bug magnet
+
+A file that has absorbed an unusual share of the repository's bug fixes.
+"""
+
+_DOCS_CHANGELOG = """\
+# Changelog
+
+## v2.1.0
+
+Added the audit trail.
+
+## v2.0.1
+
+Fixed the ledger rollover.
+
+## v2.0.0
+
+Renamed the reconciliation engine.
+
+## v1.9.4
+
+Sharpened the diff parser.
+
+## v1.9.3
+
+Rebuilt the index writer.
+
+## v1.9.2
+
+Counted the follow-ups.
+
+## v1.9.1
+
+Polished the report.
+
+## v1.9.0
+
+Introduced the audit trail.
+
+## Audit trail
+
+The audit trail records every posting.
+"""
+
+
+# Source prose. A term has to be spelled by the code as well as by the docs,
+# so the fixture is a repository rather than a documents folder. Half of it is
+# TypeScript on purpose: a miner that reads only Python scores a TypeScript
+# repository zero and then rejects its whole vocabulary.
+
+_SRC_ANALYSIS = '''\
+"""Blast radius for the ledger.
+
+Blast radius is every file a change can reach by following imports.
+"""
+
+
+def walk():
+ """Change risk is the history of the files a diff touches."""
+'''
+
+_SRC_HISTORY = '''\
+"""Co-change table.
+
+Co-change counts how often two files land in the same commit.
+"""
+'''
+
+_SRC_HOTSPOTS = '''\
+"""Churn ranking for hotspots."""
+'''
+
+_SRC_RESPONSE = '''\
+"""Response helpers.
+
+Response objects are built here. Response shaping happens later.
+"""
+'''
+
+_SRC_RECONCILE = '''\
+"""Reconciliation of bank lines against book lines."""
+'''
+
+_WEB_GRAPH = """\
+/**
+ * Bug magnet detection for the graph view.
+ *
+ * A bug magnet is a file that has absorbed far more fixes than its neighbours.
+ */
+export function bugMagnets() {}
+"""
+
+_WEB_DEAD = """\
+/**
+ * Dead code overlay.
+ */
+export function deadCode() {}
+"""
+
+
+def repo(tmp_path: Path) -> Path:
+ # The repository is a named directory, not tmp_path itself: its own name
+ # is one of the things the miner has to exclude.
+ root = tmp_path / "ledger"
+ root.mkdir()
+ (root / "README.md").write_text(_README, encoding="utf-8")
+ (root / "ARCHITECTURE.md").write_text(_ARCHITECTURE, encoding="utf-8")
+ docs = root / "docs"
+ docs.mkdir()
+ (docs / "CHANGELOG.md").write_text(_DOCS_CHANGELOG, encoding="utf-8")
+ (docs / "guide.md").write_text(_DOCS_GUIDE, encoding="utf-8")
+
+ src = root / "src"
+ src.mkdir()
+ (src / "analysis.py").write_text(_SRC_ANALYSIS, encoding="utf-8")
+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")
+ (src / "http_helpers.py").write_text(_SRC_RESPONSE, encoding="utf-8")
+ reconciliation = src / "reconciliation"
+ reconciliation.mkdir()
+ (reconciliation / "engine.py").write_text(_SRC_RECONCILE, encoding="utf-8")
+ hotspots = src / "hotspots"
+ hotspots.mkdir()
+ (hotspots / "rank.py").write_text(_SRC_HOTSPOTS, encoding="utf-8")
+
+ web = root / "web"
+ web.mkdir()
+ (web / "graph.ts").write_text(_WEB_GRAPH, encoding="utf-8")
+ (web / "dead.tsx").write_text(_WEB_DEAD, encoding="utf-8")
+
+ # Vendored code is somebody else's vocabulary and must not be counted.
+ vendored = root / "node_modules" / "other"
+ vendored.mkdir(parents=True)
+ (vendored / "index.js").write_text(
+ "/** Ingestion pipeline for a different project. */", encoding="utf-8"
+ )
+ return root
+
+
+# ---------------------------------------------------------------------------
+# The pin
+# ---------------------------------------------------------------------------
+
+#: Exactly what ``extract_terms`` returns on the fixture above, captured from
+#: the implementation that the concept planner was measured against. This list
+#: is not a wish — it is a record. Update it only alongside a deliberate,
+#: reviewed change to which concept pages the planner produces, and never to
+#: make a failing test pass.
+EXPECTED_TERMS = [
+ "Ledger", # the repository's own name
+ "Blast radius",
+ "Change risk",
+ "Your ledger stops guessing", # a marketing heading
+ "Hotspots",
+ "Response", # an ordinary English word
+ "Dead code",
+ "Co-change",
+ "Architecture",
+ "Ingestion pipeline", # named by a document, built nowhere
+ "Audit trail", # from the changelog
+ "Query guide",
+ "Bug magnet",
+]
+
+#: Five of the thirteen above are things a reader should never be shown, and
+#: every one of them stays. This list is the planner's input, and the planner
+#: discards whatever fails to bind to a real directory, so none of them costs
+#: it anything. Rejecting them is the ranked view's job — the tests below
+#: assert each is gone from *that*.
+
+
+def test_extract_terms_output_is_byte_identical(repo: Path) -> None:
+ """The planner's input does not move."""
+ assert extract_terms(repo) == EXPECTED_TERMS
+
+
+def test_extract_terms_honours_max_terms(repo: Path) -> None:
+ assert extract_terms(repo, max_terms=3) == EXPECTED_TERMS[:3]
+
+
+def test_extract_terms_on_a_repository_with_no_docs(tmp_path: Path) -> None:
+ """A repository with nothing to read is a supported outcome, not a failure."""
+ assert extract_terms(tmp_path) == []
+
+
+# ---------------------------------------------------------------------------
+# The enriched view
+# ---------------------------------------------------------------------------
+
+
+def by_term(repo: Path, **kwargs) -> dict[str, object]:
+ return {t.term: t for t in extract_house_terms(repo, **kwargs)}
+
+
+def test_the_ranked_view_keeps_only_what_the_code_also_spells(repo: Path) -> None:
+ """The gate, end to end.
+
+ Every term here is named by a document. What separates the survivors is
+ whether the source prose uses the term too.
+ """
+ terms = [t.term for t in extract_house_terms(repo)]
+ assert terms == [
+ # Two documents name it; nothing else here is named twice.
+ "Blast radius",
+ # Then multi-word terms, then single-word ones. Within each group,
+ # how much of the code uses the term.
+ "Bug magnet",
+ "Change risk",
+ "Co-change",
+ "Dead code",
+ "Hotspots",
+ "Reconciliation",
+ ]
+
+
+def test_a_term_the_documents_name_but_the_code_never_uses_is_dropped(
+ repo: Path,
+) -> None:
+ """A marketed feature with nothing behind it does not become vocabulary."""
+ assert "Ingestion pipeline" not in by_term(repo)
+
+
+def test_a_claim_is_not_a_name(repo: Path) -> None:
+ """A heading with a pronoun is a sentence about the reader."""
+ assert "Your ledger stops guessing" not in by_term(repo)
+
+
+def test_the_repository_does_not_define_its_own_name(repo: Path) -> None:
+ """It leads every frequency count in every repository and teaches nothing."""
+ assert "Ledger" not in by_term(repo)
+
+
+def test_release_notes_are_not_mined(repo: Path) -> None:
+ """The changelog names "Audit trail" under a plain heading, and the code
+ spells it, so only skipping the document keeps it out."""
+ assert "Audit trail" not in by_term(repo)
+
+
+def test_an_ordinary_word_does_not_become_a_term(repo: Path) -> None:
+ """ "Response" is named by a heading and used by the code, and is still not
+ a subsystem: no acronym shape, no internal capital, no directory."""
+ assert "Response" not in by_term(repo)
+
+
+def test_a_single_word_the_codebase_spells_in_a_directory_survives(
+ repo: Path,
+) -> None:
+ """The same test :func:`bind_terms` applies. "Hotspots" has src/hotspots/."""
+ assert "Hotspots" in by_term(repo)
+
+
+def test_a_numbered_heading_with_a_gloss_yields_its_name(repo: Path) -> None:
+ """ "2 · Reconciliation: matching two ledgers" names Reconciliation.
+
+ Without stripping the number the whole heading is rejected for carrying a
+ digit, and the section contributes nothing.
+ """
+ assert "Reconciliation" in by_term(repo)
+
+
+def test_ranking_is_by_how_many_documents_name_the_term(repo: Path) -> None:
+ terms = extract_house_terms(repo)
+ assert terms[0].term == "Blast radius"
+ assert terms[0].doc_frequency == 2
+ assert all(t.doc_frequency == 1 for t in terms[1:])
+
+
+def test_code_frequency_is_counted_but_never_added_to_document_frequency(
+ repo: Path,
+) -> None:
+ """Kept apart on purpose.
+
+ Summing them lets a word appearing in hundreds of docstrings outrank the
+ name of a real subsystem. "Response" would win on a sum and is not a term
+ at all.
+ """
+ blast = by_term(repo)["Blast radius"]
+ assert blast.doc_frequency == 2
+ assert blast.code_frequency == 1
+
+
+def test_typescript_prose_counts_as_the_code_spelling_the_term(repo: Path) -> None:
+ """A JSDoc block is a maintainer writing about a unit of code.
+
+ Reading only Python scores a TypeScript repository zero on every term and
+ the gate then rejects its entire vocabulary.
+ """
+ assert by_term(repo)["Bug magnet"].code_frequency == 1
+ assert by_term(repo)["Dead code"].code_frequency == 1
+
+
+def test_vendored_code_is_not_the_repository_talking_about_itself(
+ repo: Path,
+) -> None:
+ """node_modules names "Ingestion pipeline"; it must not gate it in."""
+ assert "Ingestion pipeline" not in by_term(repo)
+
+
+def test_a_heading_yields_its_following_sentence_as_the_definition(
+ repo: Path,
+) -> None:
+ blast = by_term(repo)["Blast radius"]
+ assert blast.definition == (
+ "Blast radius is the set of files a change can reach through the import graph."
+ )
+ assert blast.definition_source == "README.md"
+
+
+def test_a_bolded_lead_in_yields_its_definition(repo: Path) -> None:
+ dead = by_term(repo)["Dead code"]
+ assert dead.definition == "code no import path reaches."
+ assert dead.definition_source == "README.md"
+
+
+def test_a_documented_definition_wins_over_a_code_one(repo: Path) -> None:
+ """The docs are where a term is explained for a reader; a docstring is
+ where it is explained for whoever is editing that file."""
+ magnet = by_term(repo)["Bug magnet"]
+ assert magnet.definition == (
+ "A file that has absorbed an unusual share of the repository's bug fixes."
+ )
+ assert magnet.definition_source == "docs/guide.md"
+
+
+def test_code_prose_supplies_a_definition_the_documents_did_not(repo: Path) -> None:
+ """ "Co-change" is a bolded term with no dash, so the documents name it and
+ never explain it. The code does."""
+ co = by_term(repo)["Co-change"]
+ assert co.definition == ("Co-change counts how often two files land in the same commit.")
+ assert co.definition_source == "src/history.py"
+
+
+def test_a_term_records_every_document_that_names_it(repo: Path) -> None:
+ blast = by_term(repo)["Blast radius"]
+ assert blast.source_paths[:2] == ("README.md", "ARCHITECTURE.md")
+ assert "src/analysis.py" in blast.source_paths
+
+
+def test_source_paths_are_repository_relative_and_posix(repo: Path) -> None:
+ assert by_term(repo)["Bug magnet"].source_paths[0] == "docs/guide.md"
+
+
+def test_a_term_with_no_sentence_anywhere_reports_none(repo: Path) -> None:
+ """No definition beats a confident wrong one."""
+ hotspots = by_term(repo)["Hotspots"]
+ assert hotspots.code_frequency == 1, "the code does mention it"
+ assert hotspots.definition is None, "but never in a sentence that defines it"
+ assert hotspots.definition_source is None
+
+
+def test_a_term_the_codebase_defines_is_marked_as_a_symbol(repo: Path) -> None:
+ """Only a real symbol may be rendered in backticks.
+
+ Grounding strips the backticks off a symbol-shaped token it cannot
+ resolve, so a coined term dressed as code is demoted mid-page — silently,
+ which is the failure shape this repository has shipped before.
+ """
+ found = by_term(repo, known_symbols={"Blast radius"})
+ assert found["Blast radius"].is_indexed_symbol is True
+ assert found["Bug magnet"].is_indexed_symbol is False
+
+
+def test_without_a_symbol_set_nothing_claims_to_be_one(repo: Path) -> None:
+ assert all(not t.is_indexed_symbol for t in extract_house_terms(repo))
+
+
+def test_extraction_is_deterministic(repo: Path) -> None:
+ """It runs on the keyless path and has to give the same answer twice."""
+ assert extract_house_terms(repo) == extract_house_terms(repo)
+
+
+def test_house_terms_on_a_repository_with_no_docs(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ """Finding nothing is reported, not returned as if it were a finding."""
+ with caplog.at_level(logging.WARNING, logger=_LOGGER):
+ assert extract_house_terms(tmp_path) == []
+ assert "no house terms found" in caplog.text
+
+
+def test_documents_read_but_nothing_survived_is_a_different_report(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A repository whose code this cannot read looks identical to one with no
+ docs unless the two are logged apart."""
+ (tmp_path / "README.md").write_text(
+ "# Widget\n\n## Blast radius\n\nSomething about it here.\n",
+ encoding="utf-8",
+ )
+ with caplog.at_level(logging.WARNING, logger=_LOGGER):
+ assert extract_house_terms(tmp_path) == []
+ assert "none survived" in caplog.text