Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): mine the repository's own vocabulary, ranked and gated (#1233)
Swati Ahujaf10bc533c722
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 71f748a3f..50c4ffe98 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -22,11 +22,18 @@ enough that a model adds cost and variance without adding much.from __future__ import annotations+import loggingimport re+from collections.abc import Callable+from dataclasses import dataclassfrom pathlib import Path+from repowise.core import fs_walk+from .grouping import ConceptGroup+logger = logging.getLogger(__name__)+#: Documents worth mining, relative to the repository root. A doc that names a#: subsystem in a heading is a much stronger signal than one that mentions it#: in a sentence, so headings are what this reads.@@ -56,15 +63,69 @@ _BOLD_TERM = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*")_STOPWORDS = frozenset({- "the", "a", "an", "and", "or", "of", "for", "to", "in", "on", "with",- "how", "what", "why", "when", "is", "are", "it", "this", "that", "you",- "your", "we", "our", "using", "use", "usage", "getting", "started",- "installation", "install", "quickstart", "quick", "start", "license",- "contributing", "changelog", "table", "contents", "overview",- "introduction", "intro", "example", "examples", "faq", "notes",- "requirements", "setup", "configuration", "config", "options", "api",- "reference", "guide", "tutorial", "docs", "documentation", "features",- "roadmap", "credits", "acknowledgements", "badges", "support",+ "the",+ "a",+ "an",+ "and",+ "or",+ "of",+ "for",+ "to",+ "in",+ "on",+ "with",+ "how",+ "what",+ "why",+ "when",+ "is",+ "are",+ "it",+ "this",+ "that",+ "you",+ "your",+ "we",+ "our",+ "using",+ "use",+ "usage",+ "getting",+ "started",+ "installation",+ "install",+ "quickstart",+ "quick",+ "start",+ "license",+ "contributing",+ "changelog",+ "table",+ "contents",+ "overview",+ "introduction",+ "intro",+ "example",+ "examples",+ "faq",+ "notes",+ "requirements",+ "setup",+ "configuration",+ "config",+ "options",+ "api",+ "reference",+ "guide",+ "tutorial",+ "docs",+ "documentation",+ "features",+ "roadmap",+ "credits",+ "acknowledgements",+ "badges",+ "support",})@@ -76,10 +137,39 @@ _MAX_TERM_WORDS = 4# real headings from this repository's own docs, and neither is a subsystem._SENTENCE_VERBS = frozenset({- "is", "are", "was", "were", "has", "have", "had", "do", "does", "did",- "can", "will", "would", "should", "must", "may", "updated", "added",- "removed", "fixed", "works", "means", "needs", "gets", "goes", "runs",- "returns", "uses", "supports", "requires", "shows", "makes", "keeps",+ "is",+ "are",+ "was",+ "were",+ "has",+ "have",+ "had",+ "do",+ "does",+ "did",+ "can",+ "will",+ "would",+ "should",+ "must",+ "may",+ "updated",+ "added",+ "removed",+ "fixed",+ "works",+ "means",+ "needs",+ "gets",+ "goes",+ "runs",+ "returns",+ "uses",+ "supports",+ "requires",+ "shows",+ "makes",+ "keeps",})@@ -115,15 +205,38 @@ def _is_useful(term: str) -> bool:return len(words) == 1 or words[0][:1].isupper()-def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:- """Lift candidate subsystem names from the repository's own documentation.+@dataclass(frozen=True)+class HouseTerm:+ """A term the repository uses for itself, with what it was read from.- Returns terms in document order with duplicates removed. Never raises: a- repository with no docs returns an empty list, which is a supported and- common outcome, not a degraded one.+ ``definition`` is the repository's own sentence about the term — the line+ following the heading that names it, or the clause after a bolded lead-in.+ It is ``None`` far more often than not; a term with no sentence near it is+ still a term, and inventing one would be the whole failure this module+ exists to avoid."""- seen: set[str] = set()- terms: list[str] = []++ term: str+ definition: str | None+ definition_source: str | None+ #: Repository-relative POSIX paths of every document that names the term,+ #: in the order they were read.+ source_paths: tuple[str, ...]+ #: How many distinct documents name it. This is the primary ranking+ #: signal: "how often do we write about this".+ doc_frequency: int+ #: How many source files use it in their own prose. This is the gate:+ #: "was it built". It is deliberately not added to ``doc_frequency`` —+ #: see :func:`extract_house_terms`.+ code_frequency: int+ #: Whether the codebase has a symbol by this name. A term that is also a+ #: symbol can be rendered in backticks; a coined one cannot, because the+ #: grounding pass strips backticks off tokens it cannot resolve.+ is_indexed_symbol: bool+++def _doc_paths(repo_root: Path) -> list[Path]:+ """The documents worth mining, in a fixed order. Never raises."""files: list[Path] = []try:for name in DOC_FILES:@@ -142,24 +255,576 @@ def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:if len(files) >= _MAX_DOCS:breakexcept OSError:+ logger.warning(+ "vocabulary: could not list documents under %s; mining no terms for this repository",+ repo_root,+ )return []+ return files+++# A line that opens a new section. The definition scan stops here: running on+# into the next section hands back a sentence about a different subject, which+# reads as authoritative and is wrong.+_HEADING_LINE = re.compile(r"^\s{0,3}#{1,4}\s+\S")+# Lines that are structure rather than prose — tables, fences, lists, quotes.+_NOT_PROSE = ("|", "```", "-", "*", ">", "!")+# Shorter than this is a lead-in ("See below.", "Two parts:"), not a meaning.+_MIN_DEFINITION_CHARS = 20+_MAX_DEFINITION_CHARS = 300+# How far past a heading to look for the sentence that explains it.+_DEFINITION_SCAN_LINES = 5++_SENTENCE_END = re.compile(r"(?<=[.!?])(?:\s|$)")++# A bolded lead-in that carries its own definition: "**Blast radius** — ...".+_BOLD_DEFINITION = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*\s*[—–:-]\s*([^\n]{10,300})")+++def _first_sentence(line: str) -> str:+ match = _SENTENCE_END.search(line)+ return (line[: match.start() + 1] if match else line).strip()+++def _definition_after_heading(text: str, offset: int) -> str | None:+ """The first prose sentence between a heading and the next one.++ ``offset`` is the *end* of the heading text, not its start. ``_HEADING``+ opens with ``\\s{0,3}``, which in multiline mode lets a match begin on the+ blank line above; anchoring on the end sidesteps that entirely.+ """+ lines = text[offset:].split("\n")[1 : _DEFINITION_SCAN_LINES + 1]+ for line in lines:+ stripped = line.strip()+ if not stripped:+ continue+ if _HEADING_LINE.match(line):+ return None+ if stripped.startswith(_NOT_PROSE):+ continue+ sentence = _first_sentence(stripped)+ if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:+ return sentence+ return None+ return None+++# ---------------------------------------------------------------------------+# Which documents to read+# ---------------------------------------------------------------------------- for path in files:+#: Files whose headings are version numbers and shipped-feature blurbs rather+#: than subsystem names. A changelog is usually the largest document in a+#: repository and the least useful one to mine: on this repository it consumed+#: 142 of the first 200 term slots before the tool guide was ever opened.+_RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst)$", re.I)+_VERSION_HEADING = re.compile(r"^v?\d+\.\d+")+#: A document has to be overwhelmingly version headings before it is dropped. A+#: guide that cites a few release numbers is not release notes, and dropping a+#: large guide by accident loses more vocabulary than a changelog ever adds.+_RELEASE_NOTE_HEADING_RATIO = 0.6+_MIN_HEADINGS_TO_JUDGE = 8+++def _is_release_notes(path: Path, text: str) -> bool:+ if _RELEASE_NOTE_NAMES.search(path.name):+ return True+ headings = _HEADING.findall(text)+ if len(headings) < _MIN_HEADINGS_TO_JUDGE:+ return False+ versiony = sum(1 for h in headings if _VERSION_HEADING.match(h.strip()))+ return versiony / len(headings) > _RELEASE_NOTE_HEADING_RATIO+++# ---------------------------------------------------------------------------+# Which headings name something+# ---------------------------------------------------------------------------++#: A pronoun makes a heading a sentence about the reader or about us, never a+#: name: "Your agent stops guessing", "See all of it", "Who it's for". This is+#: grammar rather than a list of words we happened to dislike, so it carries+#: to a repository whose marketing copy reads nothing like ours.+_PRONOUN = re.compile(+ r"\b(you|your|yours|i|me|my|we|us|our|ours|it|its|they|them|their|theirs|he|she|him|her|his|hers)\b",+ re.I,+)++#: Numbering a heading carries ("2 · Distill: command-output compression",+#: "3. Ingestion") is document structure, not part of the name.+_ENUMERATOR = re.compile(r"^\s*\d+[.)]?\s*[·•\-–—]?\s+")+#: A heading of the form "Name: what it does" names the thing before the colon+#: and explains it after. Both halves are worth offering as candidates; the+#: lead is usually the term.+_COLON_SPLIT = re.compile(r"^([^:]{2,40}):\s+(\S.*)$")+++def _is_name_like(term: str, *, excluded: frozenset[str]) -> bool:+ """Whether a heading names a thing, as opposed to claiming something.++ Applied on top of :func:`_is_useful`, which already rejects the obvious+ sentence shapes. This adds the two tests that matter for a term list that+ gets rendered: no pronouns, and not the repository's own name — which+ leads every frequency count in every repository and teaches nobody+ anything.+ """+ if term.lower() in excluded:+ return False+ return not _PRONOUN.search(term)+++def _expand_heading(raw: str) -> list[str]:+ """The spellings of a heading worth considering as a name.++ A numbered heading yields its text without the number; a "Name: gloss"+ heading yields the name. Both are offered alongside the original, and+ :func:`_is_useful` decides which survive — a heading carrying a digit is+ rejected outright, so without stripping the enumerator a numbered section+ contributes nothing at all.+ """+ spellings = [raw]+ stripped = _ENUMERATOR.sub("", raw).strip()+ if stripped and stripped != raw:+ spellings.append(stripped)+ for candidate in list(spellings):+ colon = _COLON_SPLIT.match(candidate)+ if colon:+ lead = colon.group(1).strip()+ if lead and lead not in spellings:+ spellings.append(lead)+ return spellings+++def _repo_own_names(repo_root: Path) -> frozenset[str]:+ """What this repository calls itself, read from the repository."""+ names = {repo_root.name.lower()}+ for name in list(names):+ names.add(name.replace("-", " "))+ names.add(name.replace("_", " "))+ names.add(name.replace("-", ""))+ return frozenset(n for n in names if n)+++@dataclass+class _Candidate:+ """A term under construction, accumulating across documents."""++ term: str+ definition: str | None = None+ definition_source: str | None = None+ source_paths: list[str] | None = None+++def _harvest(+ repo_root: Path,+ *,+ limit: int | None = None,+ expand: Callable[[str], list[str]] | None = None,+ accept: Callable[[str], bool] | None = None,+ skip_release_notes: bool = False,+) -> list[_Candidate]:+ """Every candidate term in document order, deduplicated, first spelling wins.++ ``limit`` stops the walk early, so a caller wanting the first N terms does+ not pay to read documents it will discard.++ The three hooks are how the ranked view asks for more than the planner+ does without moving the planner's input. Called with none of them this is+ the harvest ``extract_terms`` has always performed, term for term.++ ``expand`` turns one raw heading into several candidate spellings.+ ``accept`` is an extra test applied after :func:`_is_useful`.+ ``skip_release_notes`` drops documents whose headings are version numbers.+ """+ candidates: dict[str, _Candidate] = {}+ order: list[str] = []+ unreadable = 0+ skipped: list[str] = []++ for path in _doc_paths(repo_root):try:text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]except OSError:+ unreadable += 1+ logger.warning("vocabulary: could not read %s; skipping it", path)+ continue+ if skip_release_notes and _is_release_notes(path, text):+ skipped.append(path.name)+ continue+ try:+ rel = path.relative_to(repo_root).as_posix()+ except ValueError: # pragma: no cover — path came from repo_root+ rel = path.name++ # Definitions found for terms seen in *this* document, keyed the same+ # way as candidates so a repeat mention can fill a gap left earlier.+ definitions = {+ _normalise(m.group(1)).lower(): m.group(2).strip()+ for m in _BOLD_DEFINITION.finditer(text)+ }++ # Headings first, then bolded lead-ins — the order the planner's+ # binding was measured against. Which of the two a term came from+ # decides where its definition may be looked for: the sentence under a+ # heading belongs to that heading, but the lines under a bolded term+ # mid-paragraph belong to the paragraph, not to the term.+ matches = [(True, m) for m in _HEADING.finditer(text)]+ matches += [(False, m) for m in _BOLD_TERM.finditer(text)]+ for is_heading, match in matches:+ # Expansion runs on the raw heading, before normalisation:+ # normalising strips the punctuation — the colon, the bullet —+ # that says where the name ends and the gloss begins.+ raw = match.group(1)+ spellings = expand(raw) if expand is not None else [raw]+ for spelling in spellings:+ term = _normalise(spelling)+ if not _is_useful(term):+ continue+ if accept is not None and not accept(term):+ continue+ key = term.lower()+ candidate = candidates.get(key)+ if candidate is None:+ if limit is not None and len(order) >= limit:+ return [candidates[k] for k in order]+ candidate = _Candidate(term=term, source_paths=[])+ candidates[key] = candidate+ order.append(key)+ assert candidate.source_paths is not None+ if rel not in candidate.source_paths:+ candidate.source_paths.append(rel)+ if candidate.definition is None:+ definition = definitions.get(key)+ if not definition and is_heading:+ definition = _definition_after_heading(text, match.end())+ if definition:+ candidate.definition = definition+ candidate.definition_source = rel++ if unreadable:+ logger.warning(+ "vocabulary: %d of the repository's documents could not be read",+ unreadable,+ )+ if skipped:+ logger.info(+ "vocabulary: skipped %d release-note document(s): %s",+ len(skipped),+ ", ".join(sorted(skipped)),+ )+ return [candidates[k] for k in order]+++# ---------------------------------------------------------------------------+# What the code says about itself+# ---------------------------------------------------------------------------++#: Directories that hold somebody else's code on top of what the shared+#: walker already prunes. Their prose is about their subsystems, not ours.+_EXTRA_PRUNED_DIRS = fs_walk.PRUNED_DIRS_DERIVED | frozenset(+ {"vendor", "third_party", "thirdparty", "site-packages"}+)++#: Python docstrings, and the block comments JSDoc and TSDoc are written in.+#: Both are prose a maintainer wrote about a unit of code, which is what makes+#: them the right place to ask whether a term names something that was built.+#: Reading only Python here would score every TypeScript-first repository zero+#: and reject its entire vocabulary.+_SOURCE_PROSE = {+ ".py": re.compile(r'(?:^|\n)\s*(?:[rubfRUBF]{0,2})"""(.*?)"""', re.S),+ ".js": re.compile(r"/\*\*(.*?)\*/", re.S),+}+for _ext in (".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts"):+ _SOURCE_PROSE[_ext] = _SOURCE_PROSE[".js"]++_MAX_SOURCE_FILES = 6_000+_MAX_SOURCE_BYTES = 60_000+_MAX_PROSE_BLOCKS = 8+#: Leading ``*`` on every line of a JSDoc block is layout, not text.+_JSDOC_GUTTER = re.compile(r"^\s*\*[ \t]?", re.M)+++def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:+ """One pruned pass over the repository, answering both tree questions.++ Returns ``(prose, directory_names)`` — ``(path, prose)`` for every source+ file that documents itself, and every name the repository has given a+ directory. Both come from the same walk because both need the whole tree+ and the walk is the expensive part.++ Never raises: an unreadable file is one fewer signal, not a failed+ generation.+ """+ prose: list[tuple[str, str]] = []+ dir_names: set[str] = set()+ unreadable = 0+ truncated = False++ for dirpath, dirnames, filenames in fs_walk.walk_repo(repo_root, prune_dirs=_EXTRA_PRUNED_DIRS):+ for name in dirnames:+ dir_names.add(_singular(name.lower()))+ if truncated:continue- for match in list(_HEADING.finditer(text)) + list(_BOLD_TERM.finditer(text)):- term = _normalise(match.group(1))- if not _is_useful(term):+ for name in sorted(filenames):+ pattern = _SOURCE_PROSE.get(Path(name).suffix)+ if pattern is None:+ continue+ if len(prose) >= _MAX_SOURCE_FILES:+ # A floor rather than a count. Said out loud, because a+ # silently truncated corpus reads downstream as "the code+ # does not use this term" — which is the gate's reject.+ logger.info(+ "vocabulary: stopped reading source prose at %d files; "+ "term code frequencies are a floor, not a count",+ _MAX_SOURCE_FILES,+ )+ truncated = True+ break+ path = dirpath / name+ try:+ text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_SOURCE_BYTES]+ except OSError:+ unreadable += 1continue- key = term.lower()- if key in seen:+ blocks = pattern.findall(text)[:_MAX_PROSE_BLOCKS]+ if not blocks:continue- seen.add(key)- terms.append(term)- if len(terms) >= max_terms:- return terms+ joined = "\n".join(_JSDOC_GUTTER.sub("", b) for b in blocks)+ try:+ rel = path.relative_to(repo_root).as_posix()+ except ValueError: # pragma: no cover — path came from repo_root+ rel = path.name+ prose.append((rel, joined))++ if unreadable:+ logger.warning("vocabulary: %d source files could not be read for prose", unreadable)+ return prose, frozenset(dir_names)+++#: The gaps between a term's words. Prose writes "co-change", an identifier+#: writes "co_change" and a heading writes "Co change"; they are one term.+_WORD_GAP = re.compile(r"[\s_\-]+")+++def _term_words(term: str) -> list[str]:+ return [w for w in _WORD_GAP.split(term) if w]+++def _phrase_pattern(term: str) -> re.Pattern[str]:+ """Match a term however the code spells the gaps between its words.++ ``bug magnet`` has to find ``bug magnet``, ``bug-magnet`` and+ ``bug_magnet``, because the docs write it one way and the code the other.+ """+ words = [re.escape(w) for w in _term_words(term)]+ return re.compile(r"\b" + r"[\s_\-]+".join(words) + r"\b", re.I)+++def _definition_from_prose(prose: str, pattern: re.Pattern[str]) -> str | None:+ """A sentence in code prose that opens by naming the term.++ A sentence leading with the term is usually the best definition in the+ repository — better than a heading's follow-on line, which is often a+ lead-in rather than a meaning. A sentence that merely mentions it is not a+ definition and is not taken.+ """+ for raw in re.split(r"(?<=[.!?])\s+", prose):+ sentence = " ".join(raw.split())+ if not (_MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS):+ continue+ if pattern.match(sentence):+ return sentence+ return None+++def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:+ """Lift candidate subsystem names from the repository's own documentation.++ Returns terms in document order with duplicates removed. Never raises: a+ repository with no docs returns an empty list, which is a supported and+ common outcome, not a degraded one.+ """+ return [c.term for c in _harvest(repo_root, limit=max_terms)]+++#: How many source paths to keep on a term. A term used by 300 files does not+#: need 300 citations; the count is kept in full on ``code_frequency``.+_MAX_CODE_PATHS = 40+++def _warn_empty(repo_root: Path) -> None:+ """Nothing was read at all — as opposed to read and rejected."""+ logger.warning(+ "vocabulary: no house terms found under %s — no readable %s and no "+ "top-level docs directory",+ repo_root,+ "/".join(DOC_FILES),+ )+++def _survives_single_word_test(term: str, dir_names: frozenset[str]) -> bool:+ """Whether a one-word term is a name rather than an ordinary word.++ A single common English word is never a house term however often it+ appears — "Code", "Response", "Query" and "Bare" all rank near the top on+ raw frequency and none of them names a subsystem. A single word earns its+ place when its *shape* marks it as a name (an acronym like MCP or FTS, an+ internal capital like PageRank), or when the codebase has named a whole+ directory after it.++ The directory test is deliberately a match against the **whole** directory+ name rather than against the words inside one. Matching word-by-word looks+ stricter than it is: a repository of a few thousand files spells almost+ every common English word somewhere inside some path, so the test passes+ everything and stops nothing. "dead_code" must not be what lets "Code"+ through.+ """+ words = _term_words(term)+ if len(words) != 1:+ return True+ word = words[0]+ acronym = word.isupper() and 2 <= len(word) <= 5+ internal_capital = any(c.isupper() for c in word[1:])+ return acronym or internal_capital or _singular(word.lower()) in dir_names+++def _singular(word: str) -> str:+ if word.endswith("ies") and len(word) > 4:+ return word[:-3] + "y"+ if word.endswith("es") and len(word) > 4:+ return word[:-2]+ if word.endswith("s") and not word.endswith("ss") and len(word) > 3:+ return word[:-1]+ return word+++def extract_house_terms(+ repo_root: Path,+ *,+ max_terms: int = 200,+ known_symbols: frozenset[str] | set[str] | None = None,+) -> list[HouseTerm]:+ """The repository's own words for its own subsystems, ranked and gated.++ The same harvest :func:`extract_terms` performs, plus three things it does+ not need and a rendered term list cannot do without.++ **Release notes are skipped.** Their headings are version numbers, and a+ changelog is usually the largest document in the repository.++ **Two frequency signals, kept apart on purpose.** ``doc_frequency`` counts+ the documents that name the term — "is this something we write about",+ and the ranking key. ``code_frequency`` counts the source files whose own+ prose uses it — "was it built", and the gate. Adding them together lets a+ common English word appearing in 180 docstrings outrank the name of an+ actual subsystem; ranking on one and gating on the other does not. The+ gate is the same bind-or-drop discipline the planner already applies: a+ marketed term with nothing behind it does not survive.++ **A term must look like a name.** No pronouns, no repository name, and a+ single word must be shaped like one or be spelled in a directory.++ ``known_symbols`` are the names the codebase defines; a term is marked+ ``is_indexed_symbol`` when it matches one. Pass nothing and every term+ reports ``False``, which is the safe direction — an unbackticked term+ renders as plain prose, a wrongly-backticked one is silently demoted.+ """+ symbols = {s.lower() for s in known_symbols} if known_symbols else set()+ excluded = _repo_own_names(repo_root)++ candidates = _harvest(+ repo_root,+ limit=max_terms,+ expand=_expand_heading,+ accept=lambda t: _is_name_like(t, excluded=excluded),+ skip_release_notes=True,+ )+ if not candidates:+ _warn_empty(repo_root)+ return []++ patterns = {c.term.lower(): _phrase_pattern(c.term) for c in candidates}+ code_files: dict[str, list[str]] = {c.term.lower(): [] for c in candidates}+ code_definitions: dict[str, tuple[str, str]] = {}+ prose_corpus, dir_names = _scan_tree(repo_root)+ for rel, prose in prose_corpus:+ for key, pattern in patterns.items():+ if not pattern.search(prose):+ continue+ code_files[key].append(rel)+ if key not in code_definitions:+ sentence = _definition_from_prose(prose, pattern)+ if sentence:+ code_definitions[key] = (sentence, rel)++ terms: list[HouseTerm] = []+ dropped_unbuilt = 0+ dropped_common_word = 0+ for candidate in candidates:+ key = candidate.term.lower()+ hits = code_files[key]+ if not hits:+ dropped_unbuilt += 1+ continue+ if not _survives_single_word_test(candidate.term, dir_names):+ dropped_common_word += 1+ continue+ definition = candidate.definition+ definition_source = candidate.definition_source+ if definition is None and key in code_definitions:+ definition, definition_source = code_definitions[key]+ doc_paths = tuple(candidate.source_paths or ())+ terms.append(+ HouseTerm(+ term=candidate.term,+ definition=definition,+ definition_source=definition_source,+ source_paths=doc_paths + tuple(hits[:_MAX_CODE_PATHS]),+ doc_frequency=len(doc_paths),+ code_frequency=len(hits),+ is_indexed_symbol=key in symbols,+ )+ )++ # How many documents name it, then multi-word terms ahead of single-word+ # ones, then how much of the code uses it.+ #+ # The middle key is the one doing the work. Document frequency is the+ # right primary signal and it discriminates poorly in practice: most+ # repositories have two or three documents worth mining, so nearly every+ # term ties at one and the sort falls through. Word count breaks that tie+ # in the right direction, because a subsystem is almost always named with+ # two words ("blast radius", "dead code", "change risk") and an ordinary+ # English word is almost always one.+ terms.sort(+ key=lambda t: (+ -t.doc_frequency,+ len(_term_words(t.term)) == 1,+ -t.code_frequency,+ t.term.lower(),+ )+ )+ logger.info(+ "vocabulary: %d house terms from %d candidates "+ "(%d named but not built, %d ordinary words), %d source files read",+ len(terms),+ len(candidates),+ dropped_unbuilt,+ dropped_common_word,+ len(prose_corpus),+ )+ if not terms:+ # Distinct from having nothing to read: the documents were read and+ # every term in them failed the gate. Usually that means the source+ # prose was not found — a language this cannot read, or a layout where+ # the code sits outside the walked tree.+ logger.warning(+ "vocabulary: no house terms found under %s — %d candidates were "+ "read from the documents and none survived; %d source files "+ "contributed prose",+ repo_root,+ len(candidates),+ len(prose_corpus),+ )return termsdiff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pynew file mode 100644index 000000000..176b4660e--- /dev/null+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -0,0 +1,482 @@+"""The vocabulary miner's two contracts.++``extract_terms`` feeds the concept planner, whose binding is measured to be+correct. Its output is therefore pinned byte-for-byte against a fixture+repository: any change to the harvest that moves a single term is a change to+which pages the planner emits, and must be seen rather than discovered later.++``extract_house_terms`` is the richer view — same candidates, plus the sentence+that defines each one and where it was read from. Nothing consumes it yet.+"""++from __future__ import annotations++import logging+from pathlib import Path++import pytest++from repowise.core.generation.concept_tree.vocabulary import (+ extract_house_terms,+ extract_terms,+)++#: Named rather than left to the root logger: a suite-wide run may have raised+#: the level on an ancestor, and the assertion is about this module reporting+#: its own failures, not about global logging configuration.+_LOGGER = "repowise.core.generation.concept_tree.vocabulary"++# ---------------------------------------------------------------------------+# The fixture repository+# ---------------------------------------------------------------------------++_README = """\+# Ledger++A ledger for small shops.++## Blast radius++Blast radius is the set of files a change can reach through the import graph.++## Change risk++Change risk scores a diff against the history of the files it touches.++### Your ledger stops guessing++Marketing copy, not a subsystem.++## Getting Started++Boilerplate heading, says nothing about this repository.++## Local paths are masked++A sentence, not a name.++## Headline facts updated 2026-06-22++A dated note.++**Dead code** — code no import path reaches.++**Co-change** is how often two files land in the same commit.++## 2 · Reconciliation: matching two ledgers++Reconciliation pairs a bank line against a book line.++## Hotspots++## Response+"""++_ARCHITECTURE = """\+# Architecture++## Blast radius++Repeated on purpose: the term is named by two documents.++## Ingestion pipeline++The ingestion pipeline walks the working tree and parses each file.+"""++_DOCS_GUIDE = """\+# Query guide++## Bug magnet++A file that has absorbed an unusual share of the repository's bug fixes.+"""++_DOCS_CHANGELOG = """\+# Changelog++## v2.1.0++Added the audit trail.++## v2.0.1++Fixed the ledger rollover.++## v2.0.0++Renamed the reconciliation engine.++## v1.9.4++Sharpened the diff parser.++## v1.9.3++Rebuilt the index writer.++## v1.9.2++Counted the follow-ups.++## v1.9.1++Polished the report.++## v1.9.0++Introduced the audit trail.++## Audit trail++The audit trail records every posting.+"""+++# Source prose. A term has to be spelled by the code as well as by the docs,+# so the fixture is a repository rather than a documents folder. Half of it is+# TypeScript on purpose: a miner that reads only Python scores a TypeScript+# repository zero and then rejects its whole vocabulary.++_SRC_ANALYSIS = '''\+"""Blast radius for the ledger.++Blast radius is every file a change can reach by following imports.+"""+++def walk():+ """Change risk is the history of the files a diff touches."""+'''++_SRC_HISTORY = '''\+"""Co-change table.++Co-change counts how often two files land in the same commit.+"""+'''++_SRC_HOTSPOTS = '''\+"""Churn ranking for hotspots."""+'''++_SRC_RESPONSE = '''\+"""Response helpers.++Response objects are built here. Response shaping happens later.+"""+'''++_SRC_RECONCILE = '''\+"""Reconciliation of bank lines against book lines."""+'''++_WEB_GRAPH = """\+/**+ * Bug magnet detection for the graph view.+ *+ * A bug magnet is a file that has absorbed far more fixes than its neighbours.+ */+export function bugMagnets() {}+"""++_WEB_DEAD = """\+/**+ * Dead code overlay.+ */+export function deadCode() {}+"""+++def repo(tmp_path: Path) -> Path:+ # The repository is a named directory, not tmp_path itself: its own name+ # is one of the things the miner has to exclude.+ root = tmp_path / "ledger"+ root.mkdir()+ (root / "README.md").write_text(_README, encoding="utf-8")+ (root / "ARCHITECTURE.md").write_text(_ARCHITECTURE, encoding="utf-8")+ docs = root / "docs"+ docs.mkdir()+ (docs / "CHANGELOG.md").write_text(_DOCS_CHANGELOG, encoding="utf-8")+ (docs / "guide.md").write_text(_DOCS_GUIDE, encoding="utf-8")++ src = root / "src"+ src.mkdir()+ (src / "analysis.py").write_text(_SRC_ANALYSIS, encoding="utf-8")+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")+ (src / "http_helpers.py").write_text(_SRC_RESPONSE, encoding="utf-8")+ reconciliation = src / "reconciliation"+ reconciliation.mkdir()+ (reconciliation / "engine.py").write_text(_SRC_RECONCILE, encoding="utf-8")+ hotspots = src / "hotspots"+ hotspots.mkdir()+ (hotspots / "rank.py").write_text(_SRC_HOTSPOTS, encoding="utf-8")++ web = root / "web"+ web.mkdir()+ (web / "graph.ts").write_text(_WEB_GRAPH, encoding="utf-8")+ (web / "dead.tsx").write_text(_WEB_DEAD, encoding="utf-8")++ # Vendored code is somebody else's vocabulary and must not be counted.+ vendored = root / "node_modules" / "other"+ vendored.mkdir(parents=True)+ (vendored / "index.js").write_text(+ "/** Ingestion pipeline for a different project. */", encoding="utf-8"+ )+ return root+++# ---------------------------------------------------------------------------+# The pin+# ---------------------------------------------------------------------------++#: Exactly what ``extract_terms`` returns on the fixture above, captured from+#: the implementation that the concept planner was measured against. This list+#: is not a wish — it is a record. Update it only alongside a deliberate,+#: reviewed change to which concept pages the planner produces, and never to+#: make a failing test pass.+EXPECTED_TERMS = [+ "Ledger", # the repository's own name+ "Blast radius",+ "Change risk",+ "Your ledger stops guessing", # a marketing heading+ "Hotspots",+ "Response", # an ordinary English word+ "Dead code",+ "Co-change",+ "Architecture",+ "Ingestion pipeline", # named by a document, built nowhere+ "Audit trail", # from the changelog+ "Query guide",+ "Bug magnet",+]++#: Five of the thirteen above are things a reader should never be shown, and+#: every one of them stays. This list is the planner's input, and the planner+#: discards whatever fails to bind to a real directory, so none of them costs+#: it anything. Rejecting them is the ranked view's job — the tests below+#: assert each is gone from *that*.+++def test_extract_terms_output_is_byte_identical(repo: Path) -> None:+ """The planner's input does not move."""+ assert extract_terms(repo) == EXPECTED_TERMS+++def test_extract_terms_honours_max_terms(repo: Path) -> None:+ assert extract_terms(repo, max_terms=3) == EXPECTED_TERMS[:3]+++def test_extract_terms_on_a_repository_with_no_docs(tmp_path: Path) -> None:+ """A repository with nothing to read is a supported outcome, not a failure."""+ assert extract_terms(tmp_path) == []+++# ---------------------------------------------------------------------------+# The enriched view+# ---------------------------------------------------------------------------+++def by_term(repo: Path, **kwargs) -> dict[str, object]:+ return {t.term: t for t in extract_house_terms(repo, **kwargs)}+++def test_the_ranked_view_keeps_only_what_the_code_also_spells(repo: Path) -> None:+ """The gate, end to end.++ Every term here is named by a document. What separates the survivors is+ whether the source prose uses the term too.+ """+ terms = [t.term for t in extract_house_terms(repo)]+ assert terms == [+ # Two documents name it; nothing else here is named twice.+ "Blast radius",+ # Then multi-word terms, then single-word ones. Within each group,+ # how much of the code uses the term.+ "Bug magnet",+ "Change risk",+ "Co-change",+ "Dead code",+ "Hotspots",+ "Reconciliation",+ ]+++def test_a_term_the_documents_name_but_the_code_never_uses_is_dropped(+ repo: Path,+) -> None:+ """A marketed feature with nothing behind it does not become vocabulary."""+ assert "Ingestion pipeline" not in by_term(repo)+++def test_a_claim_is_not_a_name(repo: Path) -> None:+ """A heading with a pronoun is a sentence about the reader."""+ assert "Your ledger stops guessing" not in by_term(repo)+++def test_the_repository_does_not_define_its_own_name(repo: Path) -> None:+ """It leads every frequency count in every repository and teaches nothing."""+ assert "Ledger" not in by_term(repo)+++def test_release_notes_are_not_mined(repo: Path) -> None:+ """The changelog names "Audit trail" under a plain heading, and the code+ spells it, so only skipping the document keeps it out."""+ assert "Audit trail" not in by_term(repo)+++def test_an_ordinary_word_does_not_become_a_term(repo: Path) -> None:+ """ "Response" is named by a heading and used by the code, and is still not+ a subsystem: no acronym shape, no internal capital, no directory."""+ assert "Response" not in by_term(repo)+++def test_a_single_word_the_codebase_spells_in_a_directory_survives(+ repo: Path,+) -> None:+ """The same test :func:`bind_terms` applies. "Hotspots" has src/hotspots/."""+ assert "Hotspots" in by_term(repo)+++def test_a_numbered_heading_with_a_gloss_yields_its_name(repo: Path) -> None:+ """ "2 · Reconciliation: matching two ledgers" names Reconciliation.++ Without stripping the number the whole heading is rejected for carrying a+ digit, and the section contributes nothing.+ """+ assert "Reconciliation" in by_term(repo)+++def test_ranking_is_by_how_many_documents_name_the_term(repo: Path) -> None:+ terms = extract_house_terms(repo)+ assert terms[0].term == "Blast radius"+ assert terms[0].doc_frequency == 2+ assert all(t.doc_frequency == 1 for t in terms[1:])+++def test_code_frequency_is_counted_but_never_added_to_document_frequency(+ repo: Path,+) -> None:+ """Kept apart on purpose.++ Summing them lets a word appearing in hundreds of docstrings outrank the+ name of a real subsystem. "Response" would win on a sum and is not a term+ at all.+ """+ blast = by_term(repo)["Blast radius"]+ assert blast.doc_frequency == 2+ assert blast.code_frequency == 1+++def test_typescript_prose_counts_as_the_code_spelling_the_term(repo: Path) -> None:+ """A JSDoc block is a maintainer writing about a unit of code.++ Reading only Python scores a TypeScript repository zero on every term and+ the gate then rejects its entire vocabulary.+ """+ assert by_term(repo)["Bug magnet"].code_frequency == 1+ assert by_term(repo)["Dead code"].code_frequency == 1+++def test_vendored_code_is_not_the_repository_talking_about_itself(+ repo: Path,+) -> None:+ """node_modules names "Ingestion pipeline"; it must not gate it in."""+ assert "Ingestion pipeline" not in by_term(repo)+++def test_a_heading_yields_its_following_sentence_as_the_definition(+ repo: Path,+) -> None:+ blast = by_term(repo)["Blast radius"]+ assert blast.definition == (+ "Blast radius is the set of files a change can reach through the import graph."+ )+ assert blast.definition_source == "README.md"+++def test_a_bolded_lead_in_yields_its_definition(repo: Path) -> None:+ dead = by_term(repo)["Dead code"]+ assert dead.definition == "code no import path reaches."+ assert dead.definition_source == "README.md"+++def test_a_documented_definition_wins_over_a_code_one(repo: Path) -> None:+ """The docs are where a term is explained for a reader; a docstring is+ where it is explained for whoever is editing that file."""+ magnet = by_term(repo)["Bug magnet"]+ assert magnet.definition == (+ "A file that has absorbed an unusual share of the repository's bug fixes."+ )+ assert magnet.definition_source == "docs/guide.md"+++def test_code_prose_supplies_a_definition_the_documents_did_not(repo: Path) -> None:+ """ "Co-change" is a bolded term with no dash, so the documents name it and+ never explain it. The code does."""+ co = by_term(repo)["Co-change"]+ assert co.definition == ("Co-change counts how often two files land in the same commit.")+ assert co.definition_source == "src/history.py"+++def test_a_term_records_every_document_that_names_it(repo: Path) -> None:+ blast = by_term(repo)["Blast radius"]+ assert blast.source_paths[:2] == ("README.md", "ARCHITECTURE.md")+ assert "src/analysis.py" in blast.source_paths+++def test_source_paths_are_repository_relative_and_posix(repo: Path) -> None:+ assert by_term(repo)["Bug magnet"].source_paths[0] == "docs/guide.md"+++def test_a_term_with_no_sentence_anywhere_reports_none(repo: Path) -> None:+ """No definition beats a confident wrong one."""+ hotspots = by_term(repo)["Hotspots"]+ assert hotspots.code_frequency == 1, "the code does mention it"+ assert hotspots.definition is None, "but never in a sentence that defines it"+ assert hotspots.definition_source is None+++def test_a_term_the_codebase_defines_is_marked_as_a_symbol(repo: Path) -> None:+ """Only a real symbol may be rendered in backticks.++ Grounding strips the backticks off a symbol-shaped token it cannot+ resolve, so a coined term dressed as code is demoted mid-page — silently,+ which is the failure shape this repository has shipped before.+ """+ found = by_term(repo, known_symbols={"Blast radius"})+ assert found["Blast radius"].is_indexed_symbol is True+ assert found["Bug magnet"].is_indexed_symbol is False+++def test_without_a_symbol_set_nothing_claims_to_be_one(repo: Path) -> None:+ assert all(not t.is_indexed_symbol for t in extract_house_terms(repo))+++def test_extraction_is_deterministic(repo: Path) -> None:+ """It runs on the keyless path and has to give the same answer twice."""+ assert extract_house_terms(repo) == extract_house_terms(repo)+++def test_house_terms_on_a_repository_with_no_docs(+ tmp_path: Path, caplog: pytest.LogCaptureFixture+) -> None:+ """Finding nothing is reported, not returned as if it were a finding."""+ with caplog.at_level(logging.WARNING, logger=_LOGGER):+ assert extract_house_terms(tmp_path) == []+ assert "no house terms found" in caplog.text+++def test_documents_read_but_nothing_survived_is_a_different_report(+ tmp_path: Path, caplog: pytest.LogCaptureFixture+) -> None:+ """A repository whose code this cannot read looks identical to one with no+ docs unless the two are logged apart."""+ (tmp_path / "README.md").write_text(+ "# Widget\n\n## Blast radius\n\nSomething about it here.\n",+ encoding="utf-8",+ )+ with caplog.at_level(logging.WARNING, logger=_LOGGER):+ assert extract_house_terms(tmp_path) == []+ assert "none survived" in caplog.text