Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): rank key concepts by what the repository writes about (#1242)

Raghav Chamadiya36b40cdb7db5
diff --git a/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py b/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py
index 02d8bbaec..6eb2dae62 100644
--- a/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py
+++ b/packages/core/src/repowise/core/generation/onboarding/subkinds/key_concepts.py
@@ -21,13 +21,18 @@ from __future__ import annotations
import math
import os
+import re
from dataclasses import dataclass, field
from typing import Any
+import structlog
+
from ..registry import SubkindSpec, register
from ..signals import OnboardingSignals
from ..slots import SLOT_KEY_CONCEPTS, SLOT_TITLES
+log = structlog.get_logger(__name__)
+
_GATE_MIN_CONCEPTS = 4
_TOP_CONCEPTS = 6
_MAX_DECISION_RECORDS = 6
@@ -224,6 +229,101 @@ def _relations_among(
return relations
+# The words of a name, however it is spelled. The documents write "blast
+# radius", the code writes ``BlastRadius`` or ``blast_radius``, and a term is
+# only useful here if those three meet.
+_NAME_WORDS = re.compile(r"[A-Z]+(?![a-z])|[A-Z][a-z0-9]*|[a-z0-9]+")
+
+
+def _name_key(text: str) -> tuple[str, ...]:
+ return tuple(word.lower() for word in _NAME_WORDS.findall(text))
+
+
+def _document_frequency(signals: OnboardingSignals) -> dict[tuple[str, ...], int]:
+ """How many of the repository's documents name each mined term.
+
+ Keyed on the term's words rather than its spelling, so a heading of
+ "Blast radius" answers for a class called ``BlastRadius``. Empty whenever
+ nothing was mined, which leaves ranking exactly as it was.
+ """
+ frequency: dict[tuple[str, ...], int] = {}
+ for term in signals.house_terms:
+ key = _name_key(term.term)
+ if key:
+ frequency[key] = max(frequency.get(key, 0), term.doc_frequency)
+ return frequency
+
+
+def _prose_hits(name: str, frequency: dict[tuple[str, ...], int]) -> int:
+ """How many documents name the idea this symbol is named after.
+
+ A term rarely *is* a class name. Codebases name the type after the idea
+ and then say what it does with it — the documents here say "dead code"
+ and the code says ``DeadCodeAnalyzer``, ``DeadCodeReport``,
+ ``DeadCodeFinding``. Matching the term exactly found two classes in this
+ repository; matching it as a run of words inside the name found 175, and
+ the difference is entirely names of that shape.
+
+ One asymmetry, because the two cases are not equally good evidence. A
+ multi-word term may sit anywhere in the name: ``CrossRepoBlastRadius`` is
+ about blast radius wherever the words fall. A single-word term has to
+ lead the name, or be the whole of it. Without that, "risk" claims
+ ``OwnershipRiskDetector`` and "stats" claims ``CFGPassStats`` — names
+ that contain the word while being about something else, and there are
+ many more of them than there are real matches.
+ """
+ if not frequency:
+ # The common case: nothing was mined, so nothing can match and the
+ # name never needs splitting. Checked here rather than at the call
+ # site because this runs once per candidate symbol, and a large
+ # repository offers tens of thousands.
+ return 0
+ words = _name_key(name)
+ best = 0
+ for size in range(len(words), 0, -1):
+ for start in range(len(words) - size + 1):
+ if size == 1 and start != 0:
+ continue
+ best = max(best, frequency.get(words[start : start + size], 0))
+ return best
+
+
+#: Phrases in a symbol's own docstring that say it exists to support
+#: development rather than to do the work. Deliberately about purpose rather
+#: than about implementation: "for unit tests" says what a thing is for in any
+#: repository, while "in-memory" describes plenty of production caches and
+#: would demote them.
+_SCAFFOLD_PHRASES = (
+ "for unit tests",
+ "for testing",
+ "for tests",
+ "in tests",
+ "test double",
+ "test fixture",
+ "test harness",
+ "test helper",
+ "not for production",
+ "not intended for production",
+ "development use",
+ "development only",
+)
+
+
+def _is_scaffolding(docstring: str) -> bool:
+ """Whether a symbol's own docstring marks it as scaffolding.
+
+ Used to demote, never to exclude. A test helper that the rest of the
+ system genuinely leans on can still reach the page — it just stops
+ outranking the types the reader came for. The signal is the symbol's own
+ words about itself, which is the cheapest honest evidence available:
+ ``InMemoryVectorStore`` describes itself as "primarily tailored for unit
+ tests" and was ranked as a load-bearing concept of the system regardless,
+ because nothing in the ranking read what it said.
+ """
+ folded = docstring.lower()
+ return any(phrase in folded for phrase in _SCAFFOLD_PHRASES)
+
+
def _is_trivial(name: str, kind: str) -> bool:
"""Drop dunders, constructors, tiny names, and trivial accessors."""
if name.startswith("__") and name.endswith("__"):
@@ -354,8 +454,11 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:
candidates: list[ConceptSymbol] = []
id_by_name: dict[str, str] = {}
seen_names: set[str] = set()
- scored: list[tuple[int, float, int, int, ConceptSymbol]] = []
+ scored: list[tuple[int, int, int, float, int, int, ConceptSymbol]] = []
any_symbol_signal = False
+ document_frequency = _document_frequency(signals)
+ written_about = 0
+ scaffolding = 0
for raw in _iter_raw_symbols(signals, gs):
path = raw["file_path"]
if path in test_files:
@@ -381,8 +484,16 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:
cluster=cluster_of(path),
cross_file_callers=xcallers,
)
+ prose_hits = _prose_hits(name, document_frequency)
+ if prose_hits:
+ written_about += 1
+ is_scaffolding = _is_scaffolding(concept.docstring)
+ if is_scaffolding:
+ scaffolding += 1
scored.append(
(
+ 0 if is_scaffolding else 1,
+ prose_hits,
xcallers,
spr,
1 if raw["is_exported"] else 0,
@@ -395,15 +506,30 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:
if not scored:
return None
+ # Two signals lead, and both are about what the repository says rather than
+ # what its graph does.
+ #
+ # Scaffolding sorts last because a symbol that describes itself as existing
+ # for tests is not the mental model, however many callers it has — and one
+ # such symbol was shipped as a core concept of this system for exactly that
+ # reason. It is a demotion and not a filter: a genuinely central helper can
+ # still fill a slot the concepts left empty.
+ #
+ # Then how many of the repository's own documents name the symbol. A class
+ # the team writes about outranks a class the graph merely calls, which the
+ # remaining four keys cannot express: they measure how much code leans on a
+ # symbol, and never whether a human found it worth explaining.
if any_symbol_signal:
- # Importance: cross-file callers, then symbol PageRank, then export
- # marker, then presence of a docstring (a deliberate public surface).
- scored.sort(key=lambda t: (t[0], t[1], t[2], t[3]), reverse=True)
+ # Then cross-file callers, symbol PageRank, export marker, and the
+ # presence of a docstring (a deliberate public surface).
+ scored.sort(key=lambda t: t[:6], reverse=True)
else:
# No resolved symbol edges (thin / rehydrated graph): fall back to the
- # file's PageRank so a page still generates on small repos.
+ # file's PageRank so a page still generates on small repos. The two
+ # prose signals still lead — a thin graph is the case where they carry
+ # the most, because everything they sit above is close to noise.
scored.sort(
- key=lambda t: (signals.pagerank.get(t[4].file_path, 0.0), t[3]),
+ key=lambda t: (t[0], t[1], signals.pagerank.get(t[6].file_path, 0.0), t[5]),
reverse=True,
)
@@ -412,7 +538,7 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:
# callers and no export marker are not something the rest of the system
# depends on, so they pad the gate but never a full page.
filler: list[ConceptSymbol] = []
- for _, _, exported_flag, _, concept in scored:
+ for *_, exported_flag, _has_doc, concept in scored:
if concept.name in seen_names:
continue
seen_names.add(concept.name)
@@ -427,6 +553,23 @@ def _build(signals: OnboardingSignals) -> KeyConceptsContext | None:
if len(concept_symbols) < _GATE_MIN_CONCEPTS:
return None
+ # Said out loud because both new signals fail quietly. A repository whose
+ # vocabulary was never mined ranks exactly as it did before and looks
+ # identical from outside, and a scaffolding list that stops matching
+ # anything reads the same as a repository with no scaffolding in it.
+ log.info(
+ "onboarding.key_concepts_ranked",
+ repo_name=signals.repo_name,
+ candidates=len(scored),
+ house_terms=len(signals.house_terms),
+ symbols_written_about=written_about,
+ symbols_demoted_as_scaffolding=scaffolding,
+ chosen_written_about=sum(
+ 1 for c in concept_symbols if _prose_hits(c.name, document_frequency)
+ ),
+ chosen=[c.name for c in concept_symbols],
+ )
+
relations = _relations_among(signals.graph_builder, concept_symbols, id_by_name)
community_labels = sorted(set(labels_by_cid.values()))[:_MAX_COMMUNITY_LABELS]
diff --git a/tests/unit/generation/test_onboarding_key_concepts.py b/tests/unit/generation/test_onboarding_key_concepts.py
index c5057c7b9..b3d08b5de 100644
--- a/tests/unit/generation/test_onboarding_key_concepts.py
+++ b/tests/unit/generation/test_onboarding_key_concepts.py
@@ -14,6 +14,7 @@ from types import SimpleNamespace
import networkx as nx
from repowise.core.generation import onboarding
+from repowise.core.generation.concept_tree.vocabulary import HouseTerm
from repowise.core.generation.models import compute_source_hash
from repowise.core.generation.onboarding.grounding import check_grounding, collect_known
from repowise.core.generation.onboarding.signals import OnboardingSignals
@@ -24,6 +25,7 @@ from repowise.core.generation.onboarding.slots import (
from repowise.core.generation.onboarding.subkinds.key_concepts import (
ConceptSymbol,
KeyConceptsContext,
+ _prose_hits,
)
from repowise.core.ingestion.models import FileInfo, ParsedFile, RepoStructure, Symbol
@@ -117,8 +119,21 @@ def _graph_builder(files: list[ParsedFile], edges: list[tuple[str, str, str]]):
)
+def _term(term: str, *, docs: int = 1) -> HouseTerm:
+ """One mined term, with the fields ranking reads set truthfully."""
+ return HouseTerm(
+ term=term,
+ definition=None,
+ definition_source=None,
+ source_paths=tuple(f"docs/{i}.md" for i in range(docs)),
+ doc_frequency=docs,
+ code_frequency=1,
+ is_indexed_symbol=True,
+ )
+
+
def _signals(
- files, graph_builder, *, kg_layers=(), layer_order=(), community=None
+ files, graph_builder, *, kg_layers=(), layer_order=(), community=None, house_terms=()
) -> OnboardingSignals:
paths = [f.file_info.path for f in files]
return OnboardingSignals(
@@ -140,6 +155,7 @@ def _signals(
sccs=(),
kg_layers=kg_layers,
layer_order=layer_order,
+ house_terms=tuple(house_terms),
)
@@ -315,6 +331,157 @@ def test_key_concepts_uses_full_graph_when_parsed_files_empty() -> None:
# ---------------------------------------------------------------------------
+# ---------------------------------------------------------------------------
+# Ranking on what the repository says, not only on what its graph does
+# ---------------------------------------------------------------------------
+
+
+def _repo_for_ranking(*, scaffold_doc: str = "The vector store.") -> list[ParsedFile]:
+ """Five concepts, one per directory so no spread cap can reorder them.
+
+ ``VectorStore`` is the graph's favourite: every caller reaches it. Nothing
+ written down mentions it. ``BlastRadius`` has one caller and is what the
+ documents are about.
+ """
+ files = [
+ _file(
+ "store/db.py",
+ [_sym("store/db.py", "VectorStore", "class", exported=True, doc=scaffold_doc)],
+ ),
+ _file(
+ "analysis/blast.py",
+ [
+ _sym(
+ "analysis/blast.py",
+ "BlastRadius",
+ "class",
+ exported=True,
+ doc="What a change can reach.",
+ )
+ ],
+ ),
+ _file(
+ "analysis/risk.py",
+ [_sym("analysis/risk.py", "ChangeRisk", "class", exported=True, doc="Scores a diff.")],
+ ),
+ _file("io/reader.py", [_sym("io/reader.py", "Reader", "class", exported=True)]),
+ _file("io/writer.py", [_sym("io/writer.py", "Writer", "class", exported=True)]),
+ ]
+ edges: list[tuple[str, str, str]] = []
+ for i in range(9):
+ caller = f"caller/c{i}.py"
+ files.append(_file(caller, [_sym(caller, f"use{i}", "function")]))
+ edges.append((f"{caller}::use{i}", "store/db.py::VectorStore", "calls"))
+ if i < 5:
+ edges.append((f"{caller}::use{i}", "io/reader.py::Reader", "calls"))
+ if i < 4:
+ edges.append((f"{caller}::use{i}", "io/writer.py::Writer", "calls"))
+ if i < 2:
+ edges.append((f"{caller}::use{i}", "analysis/risk.py::ChangeRisk", "calls"))
+ if i < 1:
+ edges.append((f"{caller}::use{i}", "analysis/blast.py::BlastRadius", "calls"))
+ return files, edges
+
+
+def _rank(*, house_terms=(), scaffold_doc: str = "The vector store.") -> list[str]:
+ files, edges = _repo_for_ranking(scaffold_doc=scaffold_doc)
+ signals = _signals(files, _graph_builder(files, edges), house_terms=house_terms)
+ ctx = onboarding.get_spec(SLOT_KEY_CONCEPTS).build_context(signals)
+ assert ctx is not None
+ return [c.name for c in ctx.concept_symbols]
+
+
+def test_a_symbol_the_documents_name_outranks_one_the_graph_merely_calls() -> None:
+ """A class the team writes about beats a class with nine callers.
+
+ The four original keys all measure how much code leans on a symbol. None
+ of them can see whether a human thought it worth explaining.
+ """
+ graph_only = _rank()
+ assert graph_only.index("VectorStore") < graph_only.index("BlastRadius")
+
+ with_prose = _rank(house_terms=[_term("Blast radius", docs=3), _term("Change risk", docs=2)])
+ assert with_prose.index("BlastRadius") < with_prose.index("VectorStore")
+ # Two documents beat one, so the ranking reads the count and not just the
+ # fact of a match.
+ assert with_prose.index("BlastRadius") < with_prose.index("ChangeRisk")
+
+
+def test_a_mined_term_matches_however_the_code_spells_it() -> None:
+ """The documents write "blast radius"; the code writes ``BlastRadius``."""
+ assert _rank(house_terms=[_term("blast radius")]).index("BlastRadius") == 0
+ assert _rank(house_terms=[_term("Blast Radius")]).index("BlastRadius") == 0
+
+
+def test_a_term_matches_the_type_named_after_it() -> None:
+ """A term is rarely a class name; it is the idea the class is named after.
+
+ Matching "blast radius" exactly finds nothing in a codebase whose class
+ is ``BlastRadiusReport``, which is how codebases are usually named.
+ """
+ assert _prose_hits("BlastRadiusReport", {("blast", "radius"): 2}) == 2
+ assert _prose_hits("CrossRepoBlastRadius", {("blast", "radius"): 2}) == 2
+ assert _prose_hits("blast_radius_of", {("blast", "radius"): 2}) == 2
+ assert _prose_hits("RadiusBlast", {("blast", "radius"): 2}) == 0
+
+
+def test_a_single_word_term_has_to_lead_the_name() -> None:
+ """Otherwise the short terms claim every name that contains the word.
+
+ "risk" would take ``OwnershipRiskDetector`` and "stats" would take
+ ``CFGPassStats`` — names that contain the word while being about
+ something else, and there are far more of those than real matches.
+ """
+ assert _prose_hits("Risk", {("risk",): 3}) == 3
+ assert _prose_hits("RiskDirective", {("risk",): 3}) == 3
+ assert _prose_hits("OwnershipRiskDetector", {("risk",): 3}) == 0
+
+
+def test_ranking_is_unchanged_when_no_vocabulary_was_mined() -> None:
+ """Most repositories will mine nothing. They must rank as they always did."""
+ assert _rank(house_terms=()) == _rank(house_terms=[_term("Nothing In This Repository")])
+
+
+def test_a_symbol_that_says_it_is_for_tests_is_demoted() -> None:
+ """Demoted, not excluded — the page can still fall back to it."""
+ ordinary = _rank()
+ assert ordinary[0] == "VectorStore"
+
+ demoted = _rank(scaffold_doc="An in-memory store, primarily tailored for unit tests.")
+ assert demoted[-1] == "VectorStore"
+ assert "VectorStore" in demoted
+
+
+def test_scaffolding_outranks_nothing_even_when_the_documents_name_it() -> None:
+ """The demotion leads, so a written-about test double still sinks."""
+ demoted = _rank(
+ house_terms=[_term("Vector store", docs=5)],
+ scaffold_doc="A store used in tests and small-scale development.",
+ )
+ assert demoted[-1] == "VectorStore"
+
+
+def test_every_concept_still_carries_its_path_for_the_page_to_cite() -> None:
+ """The path citations are what ``get_answer`` quotes off this page.
+
+ Asserted as a count against the concepts, not as a presence check: a
+ prompt that lost the path on one concept of six still contains the word.
+ """
+ files, edges = _repo_for_ranking()
+ signals = _signals(
+ files, _graph_builder(files, edges), house_terms=[_term("Blast radius", docs=3)]
+ )
+ ctx = onboarding.get_spec(SLOT_KEY_CONCEPTS).build_context(signals)
+ assert ctx is not None
+
+ rendered = _render_key_concepts(ctx)
+ assert len(ctx.concept_symbols) >= 4
+ for concept in ctx.concept_symbols:
+ assert f"`{concept.file_path}`" in rendered
+ assert rendered.count("`") >= 2 * len(ctx.concept_symbols)
+ assert "**Where it lives:**" in rendered
+
+
def _render_key_concepts(ctx) -> str:
from pathlib import Path