Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): give onboarding the repository's own vocabulary (#1241)

Raghav Chamadiya79859832aee4
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
index 541e1b830..fe0786530 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -859,11 +859,23 @@ def extract_house_terms(
return []
patterns = {c.term.lower(): _phrase_pattern(c.term) for c in candidates}
+ # Every match of a term's pattern contains that term's first word, so a
+ # substring test on the lowercased prose rejects most (file, term) pairs
+ # before the regex engine is started. It is a necessary condition and
+ # nothing else, so the terms that survive are exactly the ones that
+ # survived before — this is the same scan, done on a fraction of the
+ # pairs. It is worth the two lines: the corpus is every source file in the
+ # repository and the pattern set is up to two hundred, so the product is
+ # where the whole cost of mining sits.
+ first_words = {c.term.lower(): _term_words(c.term)[0].lower() for c in candidates}
code_files: dict[str, list[str]] = {c.term.lower(): [] for c in candidates}
code_definitions: dict[str, tuple[str, str]] = {}
prose_corpus, dir_names = _scan_tree(repo_root)
for rel, prose in prose_corpus:
+ folded = prose.lower()
for key, pattern in patterns.items():
+ if first_words[key] not in folded:
+ continue
if not pattern.search(prose):
continue
code_files[key].append(rel)
diff --git a/packages/core/src/repowise/core/generation/onboarding/signals.py b/packages/core/src/repowise/core/generation/onboarding/signals.py
index c7b85faed..7939f39a4 100644
--- a/packages/core/src/repowise/core/generation/onboarding/signals.py
+++ b/packages/core/src/repowise/core/generation/onboarding/signals.py
@@ -15,6 +15,8 @@ from typing import Any
from repowise.core.ingestion.models import ParsedFile, RepoStructure
+from ..concept_tree.vocabulary import HouseTerm
+
@dataclass(frozen=True)
class OnboardingSignals:
@@ -47,3 +49,9 @@ class OnboardingSignals:
tour_stops: tuple[dict, ...] = ()
# Layers ordered top→bottom by dependency direction (the grouping spine).
layer_order: tuple[str, ...] = ()
+ # The repository's own words for its own subsystems, ranked by how many of
+ # its documents name them and gated on the code using them. Empty when the
+ # run had no repository path to read, when the repository documents
+ # nothing, or when nothing it documents was built — all three are logged
+ # where the mining happens, because an empty tuple here cannot say which.
+ house_terms: tuple[HouseTerm, ...] = ()
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index a541bafa6..7495582a8 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -34,6 +34,7 @@ from ..models import (
compute_page_id,
compute_source_hash,
)
+from ..report import reset_house_terms
from ..styles import ONBOARDING_PAGE_TYPE, resolve_style
from .helpers import _extract_summary, _now_iso, collapse_empty_duplicate_headings
from .pertype import PerTypeGenerationMixin
@@ -250,6 +251,10 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
# generates more than once. Reset so the report's denominator counts
# this run's responses rather than every response since start-up.
reset_artifact_check_counts()
+ # Same reason, and it matters more here: a run that never reaches the
+ # onboarding level would otherwise report the previous run's
+ # vocabulary as its own.
+ reset_house_terms()
return await run_generate_all(
self,
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index 9165033f9..58ab024fb 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -9,6 +9,7 @@ object. Behaviour mirrors the original inline ``generate_all`` exactly.
from __future__ import annotations
import contextlib
+from pathlib import Path
from typing import TYPE_CHECKING, Any
import structlog
@@ -21,6 +22,7 @@ from ..models import compute_page_id
from .helpers import _is_infra_file
if TYPE_CHECKING:
+ from ..concept_tree.vocabulary import HouseTerm
from .orchestrate import _GenerationRun
log = structlog.get_logger(__name__)
@@ -333,6 +335,53 @@ def build_level7_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
]
+def _mine_house_terms(run: _GenerationRun) -> tuple[HouseTerm, ...]:
+ """The repository's own vocabulary, read once for the whole onboarding level.
+
+ Mined here rather than inside a subkind because reading it walks the
+ repository: once per run is a cost, once per slot is the same cost eight
+ times over for the same answer.
+
+ ``repo_path`` is optional on every generation entry point, and a run
+ without one has nothing to read. That is reported rather than absorbed —
+ an empty vocabulary from a repository that was never opened looks exactly
+ like an empty vocabulary from a repository that writes about nothing, and
+ the two want opposite responses.
+ """
+ from ..concept_tree.vocabulary import extract_house_terms
+ from ..report import record_house_terms
+
+ if not run.repo_path:
+ log.warning(
+ "onboarding.house_terms_skipped",
+ repo_name=run.repo_name,
+ reason="no_repo_path",
+ )
+ record_house_terms(None)
+ return ()
+
+ # The names the codebase defines. A term matching one may be rendered in
+ # backticks; a coined term may not, because the grounding pass strips
+ # backticks off any token it cannot resolve to a symbol.
+ known_symbols = {sym.name for pf in run.parsed_files for sym in pf.symbols if sym.name}
+ terms = tuple(extract_house_terms(Path(run.repo_path), known_symbols=known_symbols))
+ record_house_terms(terms)
+ if not terms:
+ log.warning(
+ "onboarding.house_terms_empty",
+ repo_name=run.repo_name,
+ known_symbols=len(known_symbols),
+ )
+ else:
+ log.info(
+ "onboarding.house_terms_mined",
+ repo_name=run.repo_name,
+ terms=len(terms),
+ top=[t.term for t in terms[:8]],
+ )
+ return terms
+
+
def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
"""Level 8 (curated onboarding collection)."""
gen = run.gen
@@ -348,6 +397,20 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
if run.on_subphase is not None:
with contextlib.suppress(Exception):
run.on_subphase("onboarding", len(specs))
+ # Which pages this run will actually write, decided before anything is
+ # assembled for them. ``_emit`` carries a side effect (a resumed run
+ # records the ids it is protecting from the stale sweep), so it is called
+ # exactly once per spec here and not again below. Assembling the signals
+ # costs a walk of the repository, and a scoped run that asked for one file
+ # page should not pay it to emit nothing.
+ emitted: list[tuple[str, Any]] = []
+ for spec in specs:
+ page_id = compute_page_id("onboarding", _onboarding.target_path(spec.slot))
+ if run._emit(page_id):
+ emitted.append((page_id, spec))
+ if not emitted:
+ return coros
+
kg_layers: tuple[dict, ...] = ()
kg_tour_steps: tuple[dict, ...] = ()
if run.kg_ctx and run.kg_ctx.available:
@@ -373,10 +436,8 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
kg_tour_steps=kg_tour_steps,
tour_stops=tuple(run.tour_stops),
layer_order=tuple(run.layer_order),
+ house_terms=_mine_house_terms(run),
)
- for spec in specs:
- page_id = compute_page_id("onboarding", _onboarding.target_path(spec.slot))
- if not run._emit(page_id):
- continue
+ for page_id, spec in emitted:
coros.append((page_id, gen.generate_onboarding_page(spec, signals)))
return coros
diff --git a/packages/core/src/repowise/core/generation/report.py b/packages/core/src/repowise/core/generation/report.py
index 466f25991..62c18a967 100644
--- a/packages/core/src/repowise/core/generation/report.py
+++ b/packages/core/src/repowise/core/generation/report.py
@@ -7,9 +7,14 @@ from __future__ import annotations
import re
from collections import Counter
+from collections.abc import Sequence
from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
from .models import GeneratedPage
+
+if TYPE_CHECKING:
+ from .concept_tree.vocabulary import HouseTerm
from .page_overlap import OverlapReport, measure_orientation_overlap
from .page_tree import LayerGroupingReport, measure_layer_grouping
from .prose import prose_word_count
@@ -159,6 +164,68 @@ def _count_question_blocks(pages: list[GeneratedPage]) -> dict[str, int]:
}
+# How many of a run's terms the report names. Enough to tell at a glance
+# whether the miner read the repository or read its marketing copy; not so many
+# that the checks table stops fitting on a screen.
+_HOUSE_TERMS_SHOWN = 8
+
+
+@dataclass(frozen=True)
+class HouseTermReport:
+ """The repository's own vocabulary, as mined for this run's orientation.
+
+ ``mined`` is the denominator and has to be read first. A run given no
+ repository path has nothing to read, and its zero says nothing about the
+ repository — which is the whole distinction the miner exists to keep:
+ "we found nothing to read" must never render as "this repository has no
+ vocabulary".
+ """
+
+ mined: bool = False
+ count: int = 0
+ #: The highest-ranked terms, in the miner's order.
+ top: tuple[str, ...] = ()
+
+ def summary_line(self) -> str:
+ if not self.mined:
+ return "not measured (no repository path)"
+ if not self.count:
+ return "0 terms — nothing the documents name is used by the code"
+ return f"{self.count} terms — {', '.join(self.top)}"
+
+
+_house_terms = HouseTermReport()
+
+
+def record_house_terms(terms: Sequence[HouseTerm] | None) -> None:
+ """Record what the vocabulary miner returned for this run.
+
+ ``None`` means the miner never ran — there was no repository path to read.
+ An empty sequence means it ran and found nothing, which is a different
+ fact and reads differently in the report.
+ """
+ global _house_terms
+ if terms is None:
+ _house_terms = HouseTermReport()
+ return
+ _house_terms = HouseTermReport(
+ mined=True,
+ count=len(terms),
+ top=tuple(t.term for t in terms[:_HOUSE_TERMS_SHOWN]),
+ )
+
+
+def house_terms_mined() -> HouseTermReport:
+ """Snapshot of the vocabulary this process last mined."""
+ return _house_terms
+
+
+def reset_house_terms() -> None:
+ """Clear the record. Used between runs and between tests."""
+ global _house_terms
+ _house_terms = HouseTermReport()
+
+
@dataclass
class GenerationReport:
"""Summary produced after ``generate_all`` completes."""
@@ -212,6 +279,12 @@ class GenerationReport:
# run wrote no module pages" and "the append stopped happening" stay
# separate facts, and neither one raises on its own.
concept_indexes: dict[str, int] = field(default_factory=dict)
+ # The repository's own words for its own subsystems, mined once per run and
+ # handed to the onboarding builders. Reported rather than inferred from the
+ # pages: no page renders a term yet, so a miner that stopped returning any
+ # would otherwise be invisible from outside. Read ``mined`` before the
+ # count.
+ house_terms: HouseTermReport = field(default_factory=HouseTermReport)
@classmethod
def from_pages(
@@ -249,6 +322,7 @@ class GenerationReport:
question_blocks=_count_question_blocks(pages),
opening_frames=measure_opening_frames(pages),
concept_indexes=_count_concept_indexes(pages),
+ house_terms=house_terms_mined(),
overview_prose_words=next(
(
prose_word_count(p.content or "")
@@ -424,6 +498,12 @@ def render_generation_checks(report: GenerationReport, console: object) -> None:
concept_text = f"[yellow]{concept_text}[/yellow]"
table.add_row("Module concept index", concept_text)
+ vocabulary = report.house_terms
+ vocabulary_text = vocabulary.summary_line()
+ if not vocabulary.mined or not vocabulary.count:
+ vocabulary_text = f"[yellow]{vocabulary_text}[/yellow]"
+ table.add_row("House vocabulary", vocabulary_text)
+
console.print(table) # type: ignore[union-attr]
for pair in overlap.flagged:
diff --git a/tests/unit/generation/test_onboarding_house_terms.py b/tests/unit/generation/test_onboarding_house_terms.py
new file mode 100644
index 000000000..f49e68b19
--- /dev/null
+++ b/tests/unit/generation/test_onboarding_house_terms.py
@@ -0,0 +1,290 @@
+"""The repository's own vocabulary reaches the onboarding builders, or says so.
+
+The miner has existed for three merges with no consumer. What this covers is
+the wiring: a run reads the repository once, hands every subkind the same
+ranked terms, and reports what it found.
+
+The negative cases carry the weight. A run given no repository path and a
+repository that documents nothing both produce an empty tuple, and an empty
+tuple on its own cannot say which happened — so each one is asserted to log,
+and the report is asserted to keep "nothing was read" and "nothing was found"
+apart.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+from structlog.testing import capture_logs
+
+from repowise.core.generation.page_generator.levels import build_level8_coros
+from repowise.core.generation.report import (
+ GenerationReport,
+ house_terms_mined,
+ record_house_terms,
+ reset_house_terms,
+)
+from repowise.core.ingestion.models import FileInfo, ParsedFile, RepoStructure, Symbol
+
+# ---------------------------------------------------------------------------
+# A repository on disk, because the miner reads one
+# ---------------------------------------------------------------------------
+
+_README = """\
+# Ledger
+
+## Blast radius
+
+Blast radius is the set of files a change can reach through the import graph.
+
+## Change risk
+
+Change risk scores a diff against the history of the files it touches.
+"""
+
+_MODULE = '''\
+"""Compute the blast radius of a change, and its change risk.
+
+Blast radius is measured over the resolved import graph.
+"""
+
+
+class BlastRadius:
+ """One change's blast radius."""
+'''
+
+
+def _repo(tmp_path: Path, *, documented: bool = True) -> Path:
+ root = tmp_path / "ledger"
+ (root / "src").mkdir(parents=True)
+ (root / "src" / "blast_radius.py").write_text(_MODULE, encoding="utf-8")
+ if documented:
+ (root / "README.md").write_text(_README, encoding="utf-8")
+ return root
+
+
+def _parsed_file(path: str, *, symbols: tuple[str, ...] = ()) -> ParsedFile:
+ info = FileInfo(
+ path=path,
+ abs_path=f"/repo/{path}",
+ language="python",
+ size_bytes=256,
+ git_hash="abc",
+ last_modified=datetime(2026, 1, 1, tzinfo=UTC),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+ return ParsedFile(
+ file_info=info,
+ symbols=[
+ Symbol(
+ id=f"{path}::{name}",
+ name=name,
+ qualified_name=f"{path}::{name}",
+ kind="class",
+ signature=f"class {name}:",
+ start_line=1,
+ end_line=8,
+ docstring=f"Docstring for {name}.",
+ decorators=[],
+ visibility="public",
+ is_async=False,
+ complexity_estimate=1,
+ language="python",
+ parent_name=None,
+ )
+ for name in symbols
+ ],
+ imports=[],
+ exports=list(symbols),
+ docstring=None,
+ parse_errors=[],
+ content_hash="abc",
+ )
+
+
+class _Gen:
+ """Stands in for the page generator, keeping the signals it was handed.
+
+ Returns a sentinel rather than a coroutine: the level builder only
+ collects what it is given, and a coroutine nothing awaits is a warning
+ with no upside here.
+ """
+
+ def __init__(self) -> None:
+ self.seen: list[Any] = []
+
+ def generate_onboarding_page(self, spec: Any, signals: Any) -> object:
+ self.seen.append(signals)
+ return object()
+
+
+def _run(*, repo_path: Path | None) -> SimpleNamespace:
+ """A generation run carrying only what level 8 reads."""
+ parsed = [_parsed_file("src/blast_radius.py", symbols=("BlastRadius",))]
+ return SimpleNamespace(
+ gen=_Gen(),
+ config=SimpleNamespace(enable_onboarding=True, source_evidence_files={}),
+ repo_path=repo_path,
+ repo_name="ledger",
+ repo_structure=RepoStructure(
+ is_monorepo=False,
+ packages=[],
+ root_language_distribution={"python": 1.0},
+ total_files=1,
+ total_loc=10,
+ entry_points=[],
+ ),
+ parsed_files=parsed,
+ source_map={},
+ graph_builder=SimpleNamespace(
+ community_info=lambda: {},
+ execution_flows=lambda: SimpleNamespace(flows=[]),
+ ),
+ pagerank={},
+ betweenness={},
+ community={},
+ sccs=(),
+ git_meta_map=None,
+ dead_code_by_file={},
+ decisions_all=(),
+ external_systems=(),
+ completed_page_summaries={},
+ tour_stops=(),
+ layer_order=(),
+ kg_ctx=None,
+ on_subphase=None,
+ _emit=lambda page_id: True,
+ )
+
+
+def _signals_from_level8(run: SimpleNamespace) -> Any:
+ coros = build_level8_coros(run)
+ assert coros, "level 8 produced no pages, so no signals were built"
+ assert run.gen.seen, "no subkind was handed the signals"
+ return run.gen.seen[0]
+
+
+# ---------------------------------------------------------------------------
+# The field
+# ---------------------------------------------------------------------------
+
+
+def test_documented_repository_populates_the_field(tmp_path: Path) -> None:
+ reset_house_terms()
+ signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+
+ terms = {t.term for t in signals.house_terms}
+ assert "Blast radius" in terms
+ assert "Change risk" in terms
+
+
+def test_every_subkind_reads_the_same_mined_terms(tmp_path: Path) -> None:
+ """One walk of the repository per run, not one per slot."""
+ reset_house_terms()
+ run = _run(repo_path=_repo(tmp_path))
+ build_level8_coros(run)
+
+ assert len(run.gen.seen) > 1, "expected more than one onboarding slot"
+ first = run.gen.seen[0].house_terms
+ assert all(s.house_terms is first for s in run.gen.seen)
+
+
+def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(
+ tmp_path: Path,
+) -> None:
+ """Mining walks the whole repository, so a run writing none must not pay.
+
+ A scoped run asking for one file page, and a resumed run whose onboarding
+ pages already exist, both reach this level and emit nothing from it.
+ """
+ reset_house_terms()
+ run = _run(repo_path=_repo(tmp_path))
+ run._emit = lambda page_id: False
+
+ assert build_level8_coros(run) == []
+ assert run.gen.seen == []
+ assert house_terms_mined().mined is False
+
+
+def test_a_term_the_codebase_defines_is_marked_as_a_symbol(tmp_path: Path) -> None:
+ """``is_indexed_symbol`` decides whether a term may be backticked.
+
+ Without the run's symbol names it is ``False`` for every term, which reads
+ downstream as "this repository defines none of its own vocabulary".
+ """
+ reset_house_terms()
+ signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+
+ by_term = {t.term: t for t in signals.house_terms}
+ assert by_term["Blast radius"].is_indexed_symbol is False
+ assert any(t.is_indexed_symbol for t in signals.house_terms) is False
+
+
+def test_undocumented_repository_yields_nothing_and_says_so(tmp_path: Path) -> None:
+ reset_house_terms()
+ run = _run(repo_path=_repo(tmp_path, documented=False))
+ with capture_logs() as logs:
+ signals = _signals_from_level8(run)
+
+ assert signals.house_terms == ()
+ assert any(entry["event"] == "onboarding.house_terms_empty" for entry in logs)
+ assert house_terms_mined().mined is True
+
+
+def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:
+ reset_house_terms()
+ run = _run(repo_path=None)
+ with capture_logs() as logs:
+ signals = _signals_from_level8(run)
+
+ assert signals.house_terms == ()
+ skipped = [e for e in logs if e["event"] == "onboarding.house_terms_skipped"]
+ assert skipped and skipped[0]["reason"] == "no_repo_path"
+ # Not the same fact as a repository that documents nothing.
+ assert house_terms_mined().mined is False
+
+
+# ---------------------------------------------------------------------------
+# The report
+# ---------------------------------------------------------------------------
+
+
+def test_the_count_and_the_top_terms_reach_the_report(tmp_path: Path) -> None:
+ reset_house_terms()
+ signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+
+ report = GenerationReport.from_pages([])
+ assert report.house_terms.mined is True
+ assert report.house_terms.count == len(signals.house_terms)
+ assert "Blast radius" in report.house_terms.top
+ assert "Blast radius" in report.house_terms.summary_line()
+
+
+def test_the_report_keeps_nothing_read_apart_from_nothing_found() -> None:
+ reset_house_terms()
+ not_mined = GenerationReport.from_pages([]).house_terms
+ assert not_mined.mined is False
+ assert not_mined.count == 0
+ assert "not measured" in not_mined.summary_line()
+
+ record_house_terms([])
+ empty = GenerationReport.from_pages([]).house_terms
+ assert empty.mined is True
+ assert empty.count == 0
+ assert "not measured" not in empty.summary_line()
+
+
+def test_a_new_run_does_not_report_the_previous_run_s_vocabulary(tmp_path: Path) -> None:
+ """The reset lives on ``generate_all``; this pins what it is there for."""
+ reset_house_terms()
+ _signals_from_level8(_run(repo_path=_repo(tmp_path)))
+ assert house_terms_mined().count > 0
+
+ reset_house_terms()
+ assert house_terms_mined() == GenerationReport().house_terms