Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): give onboarding the repository's own vocabulary (#1241)
Raghav Chamadiya79859832aee4
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 541e1b830..fe0786530 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -859,11 +859,23 @@ def extract_house_terms(return []patterns = {c.term.lower(): _phrase_pattern(c.term) for c in candidates}+ # Every match of a term's pattern contains that term's first word, so a+ # substring test on the lowercased prose rejects most (file, term) pairs+ # before the regex engine is started. It is a necessary condition and+ # nothing else, so the terms that survive are exactly the ones that+ # survived before — this is the same scan, done on a fraction of the+ # pairs. It is worth the two lines: the corpus is every source file in the+ # repository and the pattern set is up to two hundred, so the product is+ # where the whole cost of mining sits.+ first_words = {c.term.lower(): _term_words(c.term)[0].lower() for c in candidates}code_files: dict[str, list[str]] = {c.term.lower(): [] for c in candidates}code_definitions: dict[str, tuple[str, str]] = {}prose_corpus, dir_names = _scan_tree(repo_root)for rel, prose in prose_corpus:+ folded = prose.lower()for key, pattern in patterns.items():+ if first_words[key] not in folded:+ continueif not pattern.search(prose):continuecode_files[key].append(rel)diff --git a/packages/core/src/repowise/core/generation/onboarding/signals.py b/packages/core/src/repowise/core/generation/onboarding/signals.pyindex c7b85faed..7939f39a4 100644--- a/packages/core/src/repowise/core/generation/onboarding/signals.py+++ b/packages/core/src/repowise/core/generation/onboarding/signals.py@@ -15,6 +15,8 @@ from typing import Anyfrom repowise.core.ingestion.models import ParsedFile, RepoStructure+from ..concept_tree.vocabulary import HouseTerm+@dataclass(frozen=True)class OnboardingSignals:@@ -47,3 +49,9 @@ class OnboardingSignals:tour_stops: tuple[dict, ...] = ()# Layers ordered top→bottom by dependency direction (the grouping spine).layer_order: tuple[str, ...] = ()+ # The repository's own words for its own subsystems, ranked by how many of+ # its documents name them and gated on the code using them. Empty when the+ # run had no repository path to read, when the repository documents+ # nothing, or when nothing it documents was built — all three are logged+ # where the mining happens, because an empty tuple here cannot say which.+ house_terms: tuple[HouseTerm, ...] = ()diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.pyindex a541bafa6..7495582a8 100644--- a/packages/core/src/repowise/core/generation/page_generator/core.py+++ b/packages/core/src/repowise/core/generation/page_generator/core.py@@ -34,6 +34,7 @@ from ..models import (compute_page_id,compute_source_hash,)+from ..report import reset_house_termsfrom ..styles import ONBOARDING_PAGE_TYPE, resolve_stylefrom .helpers import _extract_summary, _now_iso, collapse_empty_duplicate_headingsfrom .pertype import PerTypeGenerationMixin@@ -250,6 +251,10 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):# generates more than once. Reset so the report's denominator counts# this run's responses rather than every response since start-up.reset_artifact_check_counts()+ # Same reason, and it matters more here: a run that never reaches the+ # onboarding level would otherwise report the previous run's+ # vocabulary as its own.+ reset_house_terms()return await run_generate_all(self,diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.pyindex 9165033f9..58ab024fb 100644--- a/packages/core/src/repowise/core/generation/page_generator/levels.py+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py@@ -9,6 +9,7 @@ object. Behaviour mirrors the original inline ``generate_all`` exactly.from __future__ import annotationsimport contextlib+from pathlib import Pathfrom typing import TYPE_CHECKING, Anyimport structlog@@ -21,6 +22,7 @@ from ..models import compute_page_idfrom .helpers import _is_infra_fileif TYPE_CHECKING:+ from ..concept_tree.vocabulary import HouseTermfrom .orchestrate import _GenerationRunlog = structlog.get_logger(__name__)@@ -333,6 +335,53 @@ def build_level7_coros(run: _GenerationRun) -> list[tuple[str, Any]]:]+def _mine_house_terms(run: _GenerationRun) -> tuple[HouseTerm, ...]:+ """The repository's own vocabulary, read once for the whole onboarding level.++ Mined here rather than inside a subkind because reading it walks the+ repository: once per run is a cost, once per slot is the same cost eight+ times over for the same answer.++ ``repo_path`` is optional on every generation entry point, and a run+ without one has nothing to read. That is reported rather than absorbed —+ an empty vocabulary from a repository that was never opened looks exactly+ like an empty vocabulary from a repository that writes about nothing, and+ the two want opposite responses.+ """+ from ..concept_tree.vocabulary import extract_house_terms+ from ..report import record_house_terms++ if not run.repo_path:+ log.warning(+ "onboarding.house_terms_skipped",+ repo_name=run.repo_name,+ reason="no_repo_path",+ )+ record_house_terms(None)+ return ()++ # The names the codebase defines. A term matching one may be rendered in+ # backticks; a coined term may not, because the grounding pass strips+ # backticks off any token it cannot resolve to a symbol.+ known_symbols = {sym.name for pf in run.parsed_files for sym in pf.symbols if sym.name}+ terms = tuple(extract_house_terms(Path(run.repo_path), known_symbols=known_symbols))+ record_house_terms(terms)+ if not terms:+ log.warning(+ "onboarding.house_terms_empty",+ repo_name=run.repo_name,+ known_symbols=len(known_symbols),+ )+ else:+ log.info(+ "onboarding.house_terms_mined",+ repo_name=run.repo_name,+ terms=len(terms),+ top=[t.term for t in terms[:8]],+ )+ return terms++def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:"""Level 8 (curated onboarding collection)."""gen = run.gen@@ -348,6 +397,20 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:if run.on_subphase is not None:with contextlib.suppress(Exception):run.on_subphase("onboarding", len(specs))+ # Which pages this run will actually write, decided before anything is+ # assembled for them. ``_emit`` carries a side effect (a resumed run+ # records the ids it is protecting from the stale sweep), so it is called+ # exactly once per spec here and not again below. Assembling the signals+ # costs a walk of the repository, and a scoped run that asked for one file+ # page should not pay it to emit nothing.+ emitted: list[tuple[str, Any]] = []+ for spec in specs:+ page_id = compute_page_id("onboarding", _onboarding.target_path(spec.slot))+ if run._emit(page_id):+ emitted.append((page_id, spec))+ if not emitted:+ return coros+kg_layers: tuple[dict, ...] = ()kg_tour_steps: tuple[dict, ...] = ()if run.kg_ctx and run.kg_ctx.available:@@ -373,10 +436,8 @@ def build_level8_coros(run: _GenerationRun) -> list[tuple[str, Any]]:kg_tour_steps=kg_tour_steps,tour_stops=tuple(run.tour_stops),layer_order=tuple(run.layer_order),+ house_terms=_mine_house_terms(run),)- for spec in specs:- page_id = compute_page_id("onboarding", _onboarding.target_path(spec.slot))- if not run._emit(page_id):- continue+ for page_id, spec in emitted:coros.append((page_id, gen.generate_onboarding_page(spec, signals)))return corosdiff --git a/packages/core/src/repowise/core/generation/report.py b/packages/core/src/repowise/core/generation/report.pyindex 466f25991..62c18a967 100644--- a/packages/core/src/repowise/core/generation/report.py+++ b/packages/core/src/repowise/core/generation/report.py@@ -7,9 +7,14 @@ from __future__ import annotationsimport refrom collections import Counter+from collections.abc import Sequencefrom dataclasses import dataclass, field+from typing import TYPE_CHECKINGfrom .models import GeneratedPage++if TYPE_CHECKING:+ from .concept_tree.vocabulary import HouseTermfrom .page_overlap import OverlapReport, measure_orientation_overlapfrom .page_tree import LayerGroupingReport, measure_layer_groupingfrom .prose import prose_word_count@@ -159,6 +164,68 @@ def _count_question_blocks(pages: list[GeneratedPage]) -> dict[str, int]:}+# How many of a run's terms the report names. Enough to tell at a glance+# whether the miner read the repository or read its marketing copy; not so many+# that the checks table stops fitting on a screen.+_HOUSE_TERMS_SHOWN = 8+++@dataclass(frozen=True)+class HouseTermReport:+ """The repository's own vocabulary, as mined for this run's orientation.++ ``mined`` is the denominator and has to be read first. A run given no+ repository path has nothing to read, and its zero says nothing about the+ repository — which is the whole distinction the miner exists to keep:+ "we found nothing to read" must never render as "this repository has no+ vocabulary".+ """++ mined: bool = False+ count: int = 0+ #: The highest-ranked terms, in the miner's order.+ top: tuple[str, ...] = ()++ def summary_line(self) -> str:+ if not self.mined:+ return "not measured (no repository path)"+ if not self.count:+ return "0 terms — nothing the documents name is used by the code"+ return f"{self.count} terms — {', '.join(self.top)}"+++_house_terms = HouseTermReport()+++def record_house_terms(terms: Sequence[HouseTerm] | None) -> None:+ """Record what the vocabulary miner returned for this run.++ ``None`` means the miner never ran — there was no repository path to read.+ An empty sequence means it ran and found nothing, which is a different+ fact and reads differently in the report.+ """+ global _house_terms+ if terms is None:+ _house_terms = HouseTermReport()+ return+ _house_terms = HouseTermReport(+ mined=True,+ count=len(terms),+ top=tuple(t.term for t in terms[:_HOUSE_TERMS_SHOWN]),+ )+++def house_terms_mined() -> HouseTermReport:+ """Snapshot of the vocabulary this process last mined."""+ return _house_terms+++def reset_house_terms() -> None:+ """Clear the record. Used between runs and between tests."""+ global _house_terms+ _house_terms = HouseTermReport()++@dataclassclass GenerationReport:"""Summary produced after ``generate_all`` completes."""@@ -212,6 +279,12 @@ class GenerationReport:# run wrote no module pages" and "the append stopped happening" stay# separate facts, and neither one raises on its own.concept_indexes: dict[str, int] = field(default_factory=dict)+ # The repository's own words for its own subsystems, mined once per run and+ # handed to the onboarding builders. Reported rather than inferred from the+ # pages: no page renders a term yet, so a miner that stopped returning any+ # would otherwise be invisible from outside. Read ``mined`` before the+ # count.+ house_terms: HouseTermReport = field(default_factory=HouseTermReport)@classmethoddef from_pages(@@ -249,6 +322,7 @@ class GenerationReport:question_blocks=_count_question_blocks(pages),opening_frames=measure_opening_frames(pages),concept_indexes=_count_concept_indexes(pages),+ house_terms=house_terms_mined(),overview_prose_words=next((prose_word_count(p.content or "")@@ -424,6 +498,12 @@ def render_generation_checks(report: GenerationReport, console: object) -> None:concept_text = f"[yellow]{concept_text}[/yellow]"table.add_row("Module concept index", concept_text)+ vocabulary = report.house_terms+ vocabulary_text = vocabulary.summary_line()+ if not vocabulary.mined or not vocabulary.count:+ vocabulary_text = f"[yellow]{vocabulary_text}[/yellow]"+ table.add_row("House vocabulary", vocabulary_text)+console.print(table) # type: ignore[union-attr]for pair in overlap.flagged:diff --git a/tests/unit/generation/test_onboarding_house_terms.py b/tests/unit/generation/test_onboarding_house_terms.pynew file mode 100644index 000000000..f49e68b19--- /dev/null+++ b/tests/unit/generation/test_onboarding_house_terms.py@@ -0,0 +1,290 @@+"""The repository's own vocabulary reaches the onboarding builders, or says so.++The miner has existed for three merges with no consumer. What this covers is+the wiring: a run reads the repository once, hands every subkind the same+ranked terms, and reports what it found.++The negative cases carry the weight. A run given no repository path and a+repository that documents nothing both produce an empty tuple, and an empty+tuple on its own cannot say which happened — so each one is asserted to log,+and the report is asserted to keep "nothing was read" and "nothing was found"+apart.+"""++from __future__ import annotations++from datetime import UTC, datetime+from pathlib import Path+from types import SimpleNamespace+from typing import Any++from structlog.testing import capture_logs++from repowise.core.generation.page_generator.levels import build_level8_coros+from repowise.core.generation.report import (+ GenerationReport,+ house_terms_mined,+ record_house_terms,+ reset_house_terms,+)+from repowise.core.ingestion.models import FileInfo, ParsedFile, RepoStructure, Symbol++# ---------------------------------------------------------------------------+# A repository on disk, because the miner reads one+# ---------------------------------------------------------------------------++_README = """\+# Ledger++## Blast radius++Blast radius is the set of files a change can reach through the import graph.++## Change risk++Change risk scores a diff against the history of the files it touches.+"""++_MODULE = '''\+"""Compute the blast radius of a change, and its change risk.++Blast radius is measured over the resolved import graph.+"""+++class BlastRadius:+ """One change's blast radius."""+'''+++def _repo(tmp_path: Path, *, documented: bool = True) -> Path:+ root = tmp_path / "ledger"+ (root / "src").mkdir(parents=True)+ (root / "src" / "blast_radius.py").write_text(_MODULE, encoding="utf-8")+ if documented:+ (root / "README.md").write_text(_README, encoding="utf-8")+ return root+++def _parsed_file(path: str, *, symbols: tuple[str, ...] = ()) -> ParsedFile:+ info = FileInfo(+ path=path,+ abs_path=f"/repo/{path}",+ language="python",+ size_bytes=256,+ git_hash="abc",+ last_modified=datetime(2026, 1, 1, tzinfo=UTC),+ is_test=False,+ is_config=False,+ is_api_contract=False,+ is_entry_point=False,+ )+ return ParsedFile(+ file_info=info,+ symbols=[+ Symbol(+ id=f"{path}::{name}",+ name=name,+ qualified_name=f"{path}::{name}",+ kind="class",+ signature=f"class {name}:",+ start_line=1,+ end_line=8,+ docstring=f"Docstring for {name}.",+ decorators=[],+ visibility="public",+ is_async=False,+ complexity_estimate=1,+ language="python",+ parent_name=None,+ )+ for name in symbols+ ],+ imports=[],+ exports=list(symbols),+ docstring=None,+ parse_errors=[],+ content_hash="abc",+ )+++class _Gen:+ """Stands in for the page generator, keeping the signals it was handed.++ Returns a sentinel rather than a coroutine: the level builder only+ collects what it is given, and a coroutine nothing awaits is a warning+ with no upside here.+ """++ def __init__(self) -> None:+ self.seen: list[Any] = []++ def generate_onboarding_page(self, spec: Any, signals: Any) -> object:+ self.seen.append(signals)+ return object()+++def _run(*, repo_path: Path | None) -> SimpleNamespace:+ """A generation run carrying only what level 8 reads."""+ parsed = [_parsed_file("src/blast_radius.py", symbols=("BlastRadius",))]+ return SimpleNamespace(+ gen=_Gen(),+ config=SimpleNamespace(enable_onboarding=True, source_evidence_files={}),+ repo_path=repo_path,+ repo_name="ledger",+ repo_structure=RepoStructure(+ is_monorepo=False,+ packages=[],+ root_language_distribution={"python": 1.0},+ total_files=1,+ total_loc=10,+ entry_points=[],+ ),+ parsed_files=parsed,+ source_map={},+ graph_builder=SimpleNamespace(+ community_info=lambda: {},+ execution_flows=lambda: SimpleNamespace(flows=[]),+ ),+ pagerank={},+ betweenness={},+ community={},+ sccs=(),+ git_meta_map=None,+ dead_code_by_file={},+ decisions_all=(),+ external_systems=(),+ completed_page_summaries={},+ tour_stops=(),+ layer_order=(),+ kg_ctx=None,+ on_subphase=None,+ _emit=lambda page_id: True,+ )+++def _signals_from_level8(run: SimpleNamespace) -> Any:+ coros = build_level8_coros(run)+ assert coros, "level 8 produced no pages, so no signals were built"+ assert run.gen.seen, "no subkind was handed the signals"+ return run.gen.seen[0]+++# ---------------------------------------------------------------------------+# The field+# ---------------------------------------------------------------------------+++def test_documented_repository_populates_the_field(tmp_path: Path) -> None:+ reset_house_terms()+ signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))++ terms = {t.term for t in signals.house_terms}+ assert "Blast radius" in terms+ assert "Change risk" in terms+++def test_every_subkind_reads_the_same_mined_terms(tmp_path: Path) -> None:+ """One walk of the repository per run, not one per slot."""+ reset_house_terms()+ run = _run(repo_path=_repo(tmp_path))+ build_level8_coros(run)++ assert len(run.gen.seen) > 1, "expected more than one onboarding slot"+ first = run.gen.seen[0].house_terms+ assert all(s.house_terms is first for s in run.gen.seen)+++def test_a_run_emitting_no_onboarding_page_does_not_read_the_repository(+ tmp_path: Path,+) -> None:+ """Mining walks the whole repository, so a run writing none must not pay.++ A scoped run asking for one file page, and a resumed run whose onboarding+ pages already exist, both reach this level and emit nothing from it.+ """+ reset_house_terms()+ run = _run(repo_path=_repo(tmp_path))+ run._emit = lambda page_id: False++ assert build_level8_coros(run) == []+ assert run.gen.seen == []+ assert house_terms_mined().mined is False+++def test_a_term_the_codebase_defines_is_marked_as_a_symbol(tmp_path: Path) -> None:+ """``is_indexed_symbol`` decides whether a term may be backticked.++ Without the run's symbol names it is ``False`` for every term, which reads+ downstream as "this repository defines none of its own vocabulary".+ """+ reset_house_terms()+ signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))++ by_term = {t.term: t for t in signals.house_terms}+ assert by_term["Blast radius"].is_indexed_symbol is False+ assert any(t.is_indexed_symbol for t in signals.house_terms) is False+++def test_undocumented_repository_yields_nothing_and_says_so(tmp_path: Path) -> None:+ reset_house_terms()+ run = _run(repo_path=_repo(tmp_path, documented=False))+ with capture_logs() as logs:+ signals = _signals_from_level8(run)++ assert signals.house_terms == ()+ assert any(entry["event"] == "onboarding.house_terms_empty" for entry in logs)+ assert house_terms_mined().mined is True+++def test_a_run_without_a_repository_path_yields_nothing_and_says_so() -> None:+ reset_house_terms()+ run = _run(repo_path=None)+ with capture_logs() as logs:+ signals = _signals_from_level8(run)++ assert signals.house_terms == ()+ skipped = [e for e in logs if e["event"] == "onboarding.house_terms_skipped"]+ assert skipped and skipped[0]["reason"] == "no_repo_path"+ # Not the same fact as a repository that documents nothing.+ assert house_terms_mined().mined is False+++# ---------------------------------------------------------------------------+# The report+# ---------------------------------------------------------------------------+++def test_the_count_and_the_top_terms_reach_the_report(tmp_path: Path) -> None:+ reset_house_terms()+ signals = _signals_from_level8(_run(repo_path=_repo(tmp_path)))++ report = GenerationReport.from_pages([])+ assert report.house_terms.mined is True+ assert report.house_terms.count == len(signals.house_terms)+ assert "Blast radius" in report.house_terms.top+ assert "Blast radius" in report.house_terms.summary_line()+++def test_the_report_keeps_nothing_read_apart_from_nothing_found() -> None:+ reset_house_terms()+ not_mined = GenerationReport.from_pages([]).house_terms+ assert not_mined.mined is False+ assert not_mined.count == 0+ assert "not measured" in not_mined.summary_line()++ record_house_terms([])+ empty = GenerationReport.from_pages([]).house_terms+ assert empty.mined is True+ assert empty.count == 0+ assert "not measured" not in empty.summary_line()+++def test_a_new_run_does_not_report_the_previous_run_s_vocabulary(tmp_path: Path) -> None:+ """The reset lives on ``generate_all``; this pins what it is there for."""+ reset_house_terms()+ _signals_from_level8(_run(repo_path=_repo(tmp_path)))+ assert house_terms_mined().count > 0++ reset_house_terms()+ assert house_terms_mined() == GenerationReport().house_terms