Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): rank key concepts by what the repository writes about (#1242)
Raghav Chamadiya36b40cdb7db5
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.pyindex 02d8bbaec..6eb2dae62 100644--- a/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py@@ -21,13 +21,18 @@ from __future__ import annotationsimport mathimport os+import refrom dataclasses import dataclass, fieldfrom typing import Any+import structlog+from ..registry import SubkindSpec, registerfrom ..signals import OnboardingSignalsfrom ..slots import SLOT_KEY_CONCEPTS, SLOT_TITLES+log = structlog.get_logger(__name__)+_GATE_MIN_CONCEPTS = 4_TOP_CONCEPTS = 6_MAX_DECISION_RECORDS = 6@@ -224,6 +229,101 @@ def _relations_among(return relations+# The words of a name, however it is spelled. The documents write "blast+# radius", the code writes ``BlastRadius`` or ``blast_radius``, and a term is+# only useful here if those three meet.+_NAME_WORDS = re.compile(r"[A-Z]+(?![a-z])|[A-Z][a-z0-9]*|[a-z0-9]+")+++def _name_key(text: str) -> tuple[str, ...]:+ return tuple(word.lower() for word in _NAME_WORDS.findall(text))+++def _document_frequency(signals: OnboardingSignals) -> dict[tuple[str, ...], int]:+ """How many of the repository's documents name each mined term.++ Keyed on the term's words rather than its spelling, so a heading of+ "Blast radius" answers for a class called ``BlastRadius``. Empty whenever+ nothing was mined, which leaves ranking exactly as it was.+ """+ frequency: dict[tuple[str, ...], int] = {}+ for term in signals.house_terms:+ key = _name_key(term.term)+ if key:+ frequency[key] = max(frequency.get(key, 0), term.doc_frequency)+ return frequency+++def _prose_hits(name: str, frequency: dict[tuple[str, ...], int]) -> int:+ """How many documents name the idea this symbol is named after.++ A term rarely *is* a class name. Codebases name the type after the idea+ and then say what it does with it — the documents here say "dead code"+ and the code says ``DeadCodeAnalyzer``, ``DeadCodeReport``,+ ``DeadCodeFinding``. Matching the term exactly found two classes in this+ repository; matching it as a run of words inside the name found 175, and+ the difference is entirely names of that shape.++ One asymmetry, because the two cases are not equally good evidence. A+ multi-word term may sit anywhere in the name: ``CrossRepoBlastRadius`` is+ about blast radius wherever the words fall. A single-word term has to+ lead the name, or be the whole of it. Without that, "risk" claims+ ``OwnershipRiskDetector`` and "stats" claims ``CFGPassStats`` — names+ that contain the word while being about something else, and there are+ many more of them than there are real matches.+ """+ if not frequency:+ # The common case: nothing was mined, so nothing can match and the+ # name never needs splitting. Checked here rather than at the call+ # site because this runs once per candidate symbol, and a large+ # repository offers tens of thousands.+ return 0+ words = _name_key(name)+ best = 0+ for size in range(len(words), 0, -1):+ for start in range(len(words) - size + 1):+ if size == 1 and start != 0:+ continue+ best = max(best, frequency.get(words[start : start + size], 0))+ return best+++#: Phrases in a symbol's own docstring that say it exists to support+#: development rather than to do the work. Deliberately about purpose rather+#: than about implementation: "for unit tests" says what a thing is for in any+#: repository, while "in-memory" describes plenty of production caches and+#: would demote them.+_SCAFFOLD_PHRASES = (+ "for unit tests",+ "for testing",+ "for tests",+ "in tests",+ "test double",+ "test fixture",+ "test harness",+ "test helper",+ "not for production",+ "not intended for production",+ "development use",+ "development only",+)+++def _is_scaffolding(docstring: str) -> bool:+ """Whether a symbol's own docstring marks it as scaffolding.++ Used to demote, never to exclude. A test helper that the rest of the+ system genuinely leans on can still reach the page — it just stops+ outranking the types the reader came for. The signal is the symbol's own+ words about itself, which is the cheapest honest evidence available:+ ``InMemoryVectorStore`` describes itself as "primarily tailored for unit+ tests" and was ranked as a load-bearing concept of the system regardless,+ because nothing in the ranking read what it said.+ """+ folded = docstring.lower()+ return any(phrase in folded for phrase in _SCAFFOLD_PHRASES)++def _is_trivial(name: str, kind: str) -> bool:"""Drop dunders, constructors, tiny names, and trivial accessors."""if name.startswith("__") and name.endswith("__"):@@ -354,8 +454,11 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:candidates: list[ConceptSymbol] = []id_by_name: dict[str, str] = {}seen_names: set[str] = set()- scored: list[tuple[int, float, int, int, ConceptSymbol]] = []+ scored: list[tuple[int, int, int, float, int, int, ConceptSymbol]] = []any_symbol_signal = False+ document_frequency = _document_frequency(signals)+ written_about = 0+ scaffolding = 0for raw in _iter_raw_symbols(signals, gs):path = raw["file_path"]if path in test_files:@@ -381,8 +484,16 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:cluster=cluster_of(path),cross_file_callers=xcallers,)+ prose_hits = _prose_hits(name, document_frequency)+ if prose_hits:+ written_about += 1+ is_scaffolding = _is_scaffolding(concept.docstring)+ if is_scaffolding:+ scaffolding += 1scored.append((+ 0 if is_scaffolding else 1,+ prose_hits,xcallers,spr,1 if raw["is_exported"] else 0,@@ -395,15 +506,30 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:if not scored:return None+ # Two signals lead, and both are about what the repository says rather than+ # what its graph does.+ #+ # Scaffolding sorts last because a symbol that describes itself as existing+ # for tests is not the mental model, however many callers it has — and one+ # such symbol was shipped as a core concept of this system for exactly that+ # reason. It is a demotion and not a filter: a genuinely central helper can+ # still fill a slot the concepts left empty.+ #+ # Then how many of the repository's own documents name the symbol. A class+ # the team writes about outranks a class the graph merely calls, which the+ # remaining four keys cannot express: they measure how much code leans on a+ # symbol, and never whether a human found it worth explaining.if any_symbol_signal:- # Importance: cross-file callers, then symbol PageRank, then export- # marker, then presence of a docstring (a deliberate public surface).- scored.sort(key=lambda t: (t[0], t[1], t[2], t[3]), reverse=True)+ # Then cross-file callers, symbol PageRank, export marker, and the+ # presence of a docstring (a deliberate public surface).+ scored.sort(key=lambda t: t[:6], reverse=True)else:# No resolved symbol edges (thin / rehydrated graph): fall back to the- # file's PageRank so a page still generates on small repos.+ # file's PageRank so a page still generates on small repos. The two+ # prose signals still lead — a thin graph is the case where they carry+ # the most, because everything they sit above is close to noise.scored.sort(- key=lambda t: (signals.pagerank.get(t[4].file_path, 0.0), t[3]),+ key=lambda t: (t[0], t[1], signals.pagerank.get(t[6].file_path, 0.0), t[5]),reverse=True,)@@ -412,7 +538,7 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:# callers and no export marker are not something the rest of the system# depends on, so they pad the gate but never a full page.filler: list[ConceptSymbol] = []- for _, _, exported_flag, _, concept in scored:+ for *_, exported_flag, _has_doc, concept in scored:if concept.name in seen_names:continueseen_names.add(concept.name)@@ -427,6 +553,23 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:if len(concept_symbols) < _GATE_MIN_CONCEPTS:return None+ # Said out loud because both new signals fail quietly. A repository whose+ # vocabulary was never mined ranks exactly as it did before and looks+ # identical from outside, and a scaffolding list that stops matching+ # anything reads the same as a repository with no scaffolding in it.+ log.info(+ "onboarding.key_concepts_ranked",+ repo_name=signals.repo_name,+ candidates=len(scored),+ house_terms=len(signals.house_terms),+ symbols_written_about=written_about,+ symbols_demoted_as_scaffolding=scaffolding,+ chosen_written_about=sum(+ 1 for c in concept_symbols if _prose_hits(c.name, document_frequency)+ ),+ chosen=[c.name for c in concept_symbols],+ )+relations = _relations_among(signals.graph_builder, concept_symbols, id_by_name)community_labels = sorted(set(labels_by_cid.values()))[:_MAX_COMMUNITY_LABELS]diff --git a/tests/unit/generation/test_onboarding_key_concepts.py b/tests/unit/generation/test_onboarding_key_concepts.pyindex c5057c7b9..b3d08b5de 100644--- a/tests/unit/generation/test_onboarding_key_concepts.py+++ b/tests/unit/generation/test_onboarding_key_concepts.py@@ -14,6 +14,7 @@ from types import SimpleNamespaceimport networkx as nxfrom repowise.core.generation import onboarding+from repowise.core.generation.concept_tree.vocabulary import HouseTermfrom repowise.core.generation.models import compute_source_hashfrom repowise.core.generation.onboarding.grounding import check_grounding, collect_knownfrom repowise.core.generation.onboarding.signals import OnboardingSignals@@ -24,6 +25,7 @@ from repowise.core.generation.onboarding.slots import (from repowise.core.generation.onboarding.subkinds.key_concepts import (ConceptSymbol,KeyConceptsContext,+ _prose_hits,)from repowise.core.ingestion.models import FileInfo, ParsedFile, RepoStructure, Symbol@@ -117,8 +119,21 @@ def _graph_builder(files: list[ParsedFile], edges: list[tuple[str, str, str]]):)+def _term(term: str, *, docs: int = 1) -> HouseTerm:+ """One mined term, with the fields ranking reads set truthfully."""+ return HouseTerm(+ term=term,+ definition=None,+ definition_source=None,+ source_paths=tuple(f"docs/{i}.md" for i in range(docs)),+ doc_frequency=docs,+ code_frequency=1,+ is_indexed_symbol=True,+ )++def _signals(- files, graph_builder, *, kg_layers=(), layer_order=(), community=None+ files, graph_builder, *, kg_layers=(), layer_order=(), community=None, house_terms=()) -> OnboardingSignals:paths = [f.file_info.path for f in files]return OnboardingSignals(@@ -140,6 +155,7 @@ def _signals(sccs=(),kg_layers=kg_layers,layer_order=layer_order,+ house_terms=tuple(house_terms),)@@ -315,6 +331,157 @@ def test_key_concepts_uses_full_graph_when_parsed_files_empty() -> None:# ---------------------------------------------------------------------------+# ---------------------------------------------------------------------------+# Ranking on what the repository says, not only on what its graph does+# ---------------------------------------------------------------------------+++def _repo_for_ranking(*, scaffold_doc: str = "The vector store.") -> list[ParsedFile]:+ """Five concepts, one per directory so no spread cap can reorder them.++ ``VectorStore`` is the graph's favourite: every caller reaches it. Nothing+ written down mentions it. ``BlastRadius`` has one caller and is what the+ documents are about.+ """+ files = [+ _file(+ "store/db.py",+ [_sym("store/db.py", "VectorStore", "class", exported=True, doc=scaffold_doc)],+ ),+ _file(+ "analysis/blast.py",+ [+ _sym(+ "analysis/blast.py",+ "BlastRadius",+ "class",+ exported=True,+ doc="What a change can reach.",+ )+ ],+ ),+ _file(+ "analysis/risk.py",+ [_sym("analysis/risk.py", "ChangeRisk", "class", exported=True, doc="Scores a diff.")],+ ),+ _file("io/reader.py", [_sym("io/reader.py", "Reader", "class", exported=True)]),+ _file("io/writer.py", [_sym("io/writer.py", "Writer", "class", exported=True)]),+ ]+ edges: list[tuple[str, str, str]] = []+ for i in range(9):+ caller = f"caller/c{i}.py"+ files.append(_file(caller, [_sym(caller, f"use{i}", "function")]))+ edges.append((f"{caller}::use{i}", "store/db.py::VectorStore", "calls"))+ if i < 5:+ edges.append((f"{caller}::use{i}", "io/reader.py::Reader", "calls"))+ if i < 4:+ edges.append((f"{caller}::use{i}", "io/writer.py::Writer", "calls"))+ if i < 2:+ edges.append((f"{caller}::use{i}", "analysis/risk.py::ChangeRisk", "calls"))+ if i < 1:+ edges.append((f"{caller}::use{i}", "analysis/blast.py::BlastRadius", "calls"))+ return files, edges+++def _rank(*, house_terms=(), scaffold_doc: str = "The vector store.") -> list[str]:+ files, edges = _repo_for_ranking(scaffold_doc=scaffold_doc)+ signals = _signals(files, _graph_builder(files, edges), house_terms=house_terms)+ ctx = onboarding.get_spec(SLOT_KEY_CONCEPTS).build_context(signals)+ assert ctx is not None+ return [c.name for c in ctx.concept_symbols]+++def test_a_symbol_the_documents_name_outranks_one_the_graph_merely_calls() -> None:+ """A class the team writes about beats a class with nine callers.++ The four original keys all measure how much code leans on a symbol. None+ of them can see whether a human thought it worth explaining.+ """+ graph_only = _rank()+ assert graph_only.index("VectorStore") < graph_only.index("BlastRadius")++ with_prose = _rank(house_terms=[_term("Blast radius", docs=3), _term("Change risk", docs=2)])+ assert with_prose.index("BlastRadius") < with_prose.index("VectorStore")+ # Two documents beat one, so the ranking reads the count and not just the+ # fact of a match.+ assert with_prose.index("BlastRadius") < with_prose.index("ChangeRisk")+++def test_a_mined_term_matches_however_the_code_spells_it() -> None:+ """The documents write "blast radius"; the code writes ``BlastRadius``."""+ assert _rank(house_terms=[_term("blast radius")]).index("BlastRadius") == 0+ assert _rank(house_terms=[_term("Blast Radius")]).index("BlastRadius") == 0+++def test_a_term_matches_the_type_named_after_it() -> None:+ """A term is rarely a class name; it is the idea the class is named after.++ Matching "blast radius" exactly finds nothing in a codebase whose class+ is ``BlastRadiusReport``, which is how codebases are usually named.+ """+ assert _prose_hits("BlastRadiusReport", {("blast", "radius"): 2}) == 2+ assert _prose_hits("CrossRepoBlastRadius", {("blast", "radius"): 2}) == 2+ assert _prose_hits("blast_radius_of", {("blast", "radius"): 2}) == 2+ assert _prose_hits("RadiusBlast", {("blast", "radius"): 2}) == 0+++def test_a_single_word_term_has_to_lead_the_name() -> None:+ """Otherwise the short terms claim every name that contains the word.++ "risk" would take ``OwnershipRiskDetector`` and "stats" would take+ ``CFGPassStats`` — names that contain the word while being about+ something else, and there are far more of those than real matches.+ """+ assert _prose_hits("Risk", {("risk",): 3}) == 3+ assert _prose_hits("RiskDirective", {("risk",): 3}) == 3+ assert _prose_hits("OwnershipRiskDetector", {("risk",): 3}) == 0+++def test_ranking_is_unchanged_when_no_vocabulary_was_mined() -> None:+ """Most repositories will mine nothing. They must rank as they always did."""+ assert _rank(house_terms=()) == _rank(house_terms=[_term("Nothing In This Repository")])+++def test_a_symbol_that_says_it_is_for_tests_is_demoted() -> None:+ """Demoted, not excluded — the page can still fall back to it."""+ ordinary = _rank()+ assert ordinary[0] == "VectorStore"++ demoted = _rank(scaffold_doc="An in-memory store, primarily tailored for unit tests.")+ assert demoted[-1] == "VectorStore"+ assert "VectorStore" in demoted+++def test_scaffolding_outranks_nothing_even_when_the_documents_name_it() -> None:+ """The demotion leads, so a written-about test double still sinks."""+ demoted = _rank(+ house_terms=[_term("Vector store", docs=5)],+ scaffold_doc="A store used in tests and small-scale development.",+ )+ assert demoted[-1] == "VectorStore"+++def test_every_concept_still_carries_its_path_for_the_page_to_cite() -> None:+ """The path citations are what ``get_answer`` quotes off this page.++ Asserted as a count against the concepts, not as a presence check: a+ prompt that lost the path on one concept of six still contains the word.+ """+ files, edges = _repo_for_ranking()+ signals = _signals(+ files, _graph_builder(files, edges), house_terms=[_term("Blast radius", docs=3)]+ )+ ctx = onboarding.get_spec(SLOT_KEY_CONCEPTS).build_context(signals)+ assert ctx is not None++ rendered = _render_key_concepts(ctx)+ assert len(ctx.concept_symbols) >= 4+ for concept in ctx.concept_symbols:+ assert f"`{concept.file_path}`" in rendered+ assert rendered.count("`") >= 2 * len(ctx.concept_symbols)+ assert "**Where it lives:**" in rendered++def _render_key_concepts(ctx) -> str:from pathlib import Path