Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): measure vocabulary overlap across orientation pages (#1160)
Swati Ahuja7e5b05926a77
diff --git a/packages/core/src/repowise/core/generation/page_overlap.py b/packages/core/src/repowise/core/generation/page_overlap.pynew file mode 100644index 000000000..d91e518f4--- /dev/null+++ b/packages/core/src/repowise/core/generation/page_overlap.py@@ -0,0 +1,234 @@+"""Vocabulary overlap between orientation pages.++Orientation pages (the repository overview, the architecture diagram, the+layer pages and the onboarding collection) are what a reader meets before any+reference content. Nothing in generation checks whether two of them say the+same thing, so near-duplicates accumulate silently: they cost the reader time+and they give retrieval several adjacent-but-different pages to confuse for+each other.++This module measures that overlap. Each page is reduced to its vocabulary —+the set of distinct word-ish tokens it uses — and every pair is scored by+Jaccard similarity (shared tokens divided by total distinct tokens). It is+deliberately a vocabulary measure and not an embedding one: it needs no model,+it is stable across runs, and "these two pages draw on the same words" is+exactly the property that makes two orientation pages redundant to read.++Pairs are scored in two classes, because the two behave very differently:++* **Different page types** — an overview and an architecture diagram have no+ reason to share much vocabulary. Real overlap here is a genuine duplicate.+* **Same page type** — eleven layer pages legitimately share a house style and+ a domain vocabulary, so they sit much higher as a matter of course. Holding+ them to the cross-type threshold would flag almost every pair and bury the+ signal.++Both thresholds are configurable so they can be tuned without a code change.++The report distinguishes "no pairs were comparable" from "pairs were compared+and none overlapped". Those are opposite facts — the first means the check+did not run on anything and its silence proves nothing — and collapsing them+into a single zero would let a broken orientation set read as a clean bill of+health.+"""++from __future__ import annotations++import logging+import re+from dataclasses import dataclass, field+from itertools import combinations+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from collections.abc import Iterable, Sequence++ from .models import GeneratedPage++logger = logging.getLogger(__name__)++# The pages a reader meets before any reference content.+ORIENTATION_PAGE_TYPES: frozenset[str] = frozenset(+ {"onboarding", "layer_page", "repo_overview", "architecture_diagram"}+)++# Measured across the orientation set of a full index: pairs of *different*+# page types sit at or below 0.19, except one known duplicate at 0.24. 0.22+# separates them with margin on both sides.+DEFAULT_CROSS_TYPE_THRESHOLD = 0.22++# Pages of the *same* type share a template and a domain vocabulary, so they+# run far higher — the layer pages have a median near 0.38 with nothing wrong.+# 0.45 catches the genuinely interchangeable ones without flagging the rest.+DEFAULT_SAME_TYPE_THRESHOLD = 0.45++_TOKEN_RE = re.compile(r"[a-z0-9_]+")+++def _vocabulary(content: str) -> frozenset[str]:+ """The set of distinct tokens in a page, lowercased."""+ return frozenset(_TOKEN_RE.findall(content.lower()))+++def jaccard(left: Iterable[str], right: Iterable[str]) -> float:+ """Shared tokens over total distinct tokens. 0.0 when both are empty."""+ a, b = frozenset(left), frozenset(right)+ union = a | b+ if not union:+ return 0.0+ return len(a & b) / len(union)+++@dataclass(frozen=True)+class OverlapPair:+ """Two orientation pages and how much vocabulary they share."""++ page_id_a: str+ page_id_b: str+ title_a: str+ title_b: str+ page_type_a: str+ page_type_b: str+ similarity: float+ threshold: float++ @property+ def same_type(self) -> bool:+ return self.page_type_a == self.page_type_b++ def describe(self) -> str:+ return (+ f"{self.title_a!r} and {self.title_b!r} share "+ f"{self.similarity:.0%} of their vocabulary "+ f"(threshold {self.threshold:.0%})"+ )+++@dataclass(frozen=True)+class OverlapReport:+ """Result of comparing the orientation pages in one generation run."""++ pages_compared: int = 0+ pairs_compared: int = 0+ pages_skipped_empty: int = 0+ flagged: tuple[OverlapPair, ...] = field(default_factory=tuple)+ cross_type_threshold: float = DEFAULT_CROSS_TYPE_THRESHOLD+ same_type_threshold: float = DEFAULT_SAME_TYPE_THRESHOLD++ @property+ def comparable(self) -> bool:+ """Whether the check actually compared anything.++ ``False`` means the run produced fewer than two orientation pages with+ content, so ``flagged`` being empty says nothing about duplication.+ Callers must not read an empty ``flagged`` as a pass without checking+ this first.+ """+ return self.pairs_compared > 0++ @property+ def flagged_count(self) -> int:+ return len(self.flagged)++ @property+ def highest_similarity(self) -> float | None:+ """The worst pair's score, or ``None`` if nothing was comparable."""+ if not self.pairs_compared:+ return None+ return max((p.similarity for p in self.flagged), default=0.0)++ def summary_line(self) -> str:+ """One line fit for a report table, honest about the not-run case."""+ if not self.comparable:+ return f"not computed ({self.pages_compared} orientation pages)"+ if not self.flagged:+ return f"no overlapping pairs ({self.pairs_compared} compared)"+ return f"{self.flagged_count} of {self.pairs_compared} pairs overlap"+++def measure_orientation_overlap(+ pages: Sequence[GeneratedPage],+ *,+ cross_type_threshold: float = DEFAULT_CROSS_TYPE_THRESHOLD,+ same_type_threshold: float = DEFAULT_SAME_TYPE_THRESHOLD,+) -> OverlapReport:+ """Score vocabulary overlap across the orientation pages in ``pages``.++ Non-orientation pages are ignored. Pages whose content has no tokens are+ counted in ``pages_skipped_empty`` rather than scored, because an empty+ page trivially shares nothing and would otherwise look like a clean result.++ Raises:+ ValueError: if either threshold is outside 0.0-1.0. A nonsensical+ threshold would silently flag everything or nothing.+ """+ for name, value in (+ ("cross_type_threshold", cross_type_threshold),+ ("same_type_threshold", same_type_threshold),+ ):+ if not 0.0 <= value <= 1.0:+ raise ValueError(f"{name} must be between 0.0 and 1.0, got {value!r}")++ scored: list[tuple[GeneratedPage, frozenset[str]]] = []+ skipped = 0+ for page in pages:+ if page.page_type not in ORIENTATION_PAGE_TYPES:+ continue+ vocab = _vocabulary(page.content or "")+ if not vocab:+ skipped += 1+ logger.warning(+ "Orientation page %r has no comparable content; "+ "excluding it from the overlap check",+ page.page_id,+ )+ continue+ scored.append((page, vocab))++ flagged: list[OverlapPair] = []+ pairs_compared = 0+ for (page_a, vocab_a), (page_b, vocab_b) in combinations(scored, 2):+ pairs_compared += 1+ threshold = (+ same_type_threshold if page_a.page_type == page_b.page_type else cross_type_threshold+ )+ similarity = jaccard(vocab_a, vocab_b)+ if similarity >= threshold:+ flagged.append(+ OverlapPair(+ page_id_a=page_a.page_id,+ page_id_b=page_b.page_id,+ title_a=page_a.title,+ title_b=page_b.title,+ page_type_a=page_a.page_type,+ page_type_b=page_b.page_type,+ similarity=similarity,+ threshold=threshold,+ )+ )++ flagged.sort(key=lambda p: (-p.similarity, p.page_id_a, p.page_id_b))++ report = OverlapReport(+ pages_compared=len(scored),+ pairs_compared=pairs_compared,+ pages_skipped_empty=skipped,+ flagged=tuple(flagged),+ cross_type_threshold=cross_type_threshold,+ same_type_threshold=same_type_threshold,+ )++ if not report.comparable:+ # Loud on purpose: a run that compared nothing must not be mistaken+ # for a run that found nothing.+ logger.warning(+ "Orientation overlap check compared no pairs (%d pages scored, "+ "%d skipped as empty); an empty result here is not a pass",+ report.pages_compared,+ skipped,+ )+ else:+ for pair in flagged:+ logger.warning("Orientation pages overlap: %s", pair.describe())++ return reportdiff --git a/packages/core/src/repowise/core/generation/report.py b/packages/core/src/repowise/core/generation/report.pyindex d5d59895a..4231a7822 100644--- a/packages/core/src/repowise/core/generation/report.py+++ b/packages/core/src/repowise/core/generation/report.py@@ -9,6 +9,7 @@ from collections import Counterfrom dataclasses import dataclass, fieldfrom .models import GeneratedPage+from .page_overlap import OverlapReport, measure_orientation_overlap@dataclass@@ -25,6 +26,11 @@ class GenerationReport:elapsed_seconds: float = 0.0hallucination_warning_count: int = 0self_repaired_page_count: int = 0+ # Vocabulary overlap across the orientation pages this run produced.+ # Warn-only: a flagged pair is reported, never fatal. Read+ # ``comparable`` before reading ``flagged`` — an empty ``flagged`` on a+ # run that compared nothing is not a clean result.+ orientation_overlap: OverlapReport = field(default_factory=OverlapReport)@classmethoddef from_pages(@@ -50,6 +56,7 @@ class GenerationReport:elapsed_seconds=elapsed,hallucination_warning_count=hal_count,self_repaired_page_count=repair_count,+ orientation_overlap=measure_orientation_overlap(pages),)@property@@ -98,4 +105,15 @@ def render_report(report: GenerationReport, console: object) -> None:if report.self_repaired_page_count:table.add_row("Self-repaired pages", str(report.self_repaired_page_count))+ # Always shown, including when nothing was comparable. Hiding the row on+ # a zero would make "the check did not run" look like "the check passed".+ overlap = report.orientation_overlap+ overlap_text = overlap.summary_line()+ if overlap.flagged:+ overlap_text = f"[yellow]{overlap_text}[/yellow]"+ table.add_row("Orientation overlap", overlap_text)+console.print(table) # type: ignore[union-attr]++ for pair in overlap.flagged:+ console.print(f" [yellow]overlap[/yellow] {pair.describe()}") # type: ignore[union-attr]diff --git a/tests/unit/generation/test_page_overlap.py b/tests/unit/generation/test_page_overlap.pynew file mode 100644index 000000000..174ce2cde--- /dev/null+++ b/tests/unit/generation/test_page_overlap.py@@ -0,0 +1,326 @@+"""Vocabulary overlap between orientation pages."""++from __future__ import annotations++import logging++import pytest++from repowise.core.generation.models import GeneratedPage+from repowise.core.generation.page_overlap import (+ ORIENTATION_PAGE_TYPES,+ OverlapReport,+ jaccard,+ measure_orientation_overlap,+)+from repowise.core.generation.report import GenerationReport++# Warnings are asserted against this logger by name rather than the root+# logger. The CLI sets ``repowise.core`` to ERROR unless run verbose, and that+# level survives for the whole process — raising only the root level would+# capture nothing once any CLI code has run in the same session.+_LOGGER = "repowise.core.generation.page_overlap"++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------++# The overview and the architecture diagram are the known duplicate pair: both+# describe the same repository at the same altitude, in the same words.+_OVERVIEW = """+Repowise is a repository documentation engine. It ingests a codebase and its+git metadata, parses the source into structured entities, resolves the+cross-references and dependencies between them, and generates wiki pages and+graph data that an API serves to a web UI. The ingestion layer walks the tree,+the analysis layer scores health and risk, and the generation layer synthesises+the prose. Persistence keeps the parsed entities and the generated pages.+"""++_ARCHITECTURE_DIAGRAM = """+This diagram shows the repowise documentation engine. A codebase and its git+metadata are ingested, the source is parsed into structured entities, the+cross-references and dependencies between them are resolved, and wiki pages+and graph data are generated for an API that serves a web UI. Ingestion walks+the tree, analysis scores health and risk, generation synthesises the prose,+and persistence keeps the parsed entities and the generated pages.+"""++# Genuinely different: a first-run walkthrough shares almost no vocabulary with+# an architectural description.+_GETTING_STARTED = """+Install the command line tool with pip. Point it at a checkout and run the+index command; the first pass takes a few minutes on a large tree. When it+finishes, open the local server in a browser to read what it wrote. If you+would rather stay in your editor, install the extension and use the sidebar.+Set your API key in the environment first or the run will stop and tell you.+"""+++def _page(+ page_id: str,+ page_type: str,+ title: str,+ content: str,+) -> GeneratedPage:+ return GeneratedPage(+ page_id=page_id,+ page_type=page_type,+ title=title,+ content=content,+ source_hash="",+ model_name="mock",+ provider_name="mock",+ input_tokens=0,+ output_tokens=0,+ cached_tokens=0,+ generation_level=6,+ target_path="",+ created_at="2026-07-29T00:00:00Z",+ updated_at="2026-07-29T00:00:00Z",+ )+++def overview() -> GeneratedPage:+ return _page("repo_overview:.", "repo_overview", "Repository Overview", _OVERVIEW)+++def architecture() -> GeneratedPage:+ return _page(+ "architecture_diagram:.",+ "architecture_diagram",+ "Architecture Diagram",+ _ARCHITECTURE_DIAGRAM,+ )+++def getting_started() -> GeneratedPage:+ return _page(+ "onboarding:getting-started",+ "onboarding",+ "Getting Started",+ _GETTING_STARTED,+ )+++# ---------------------------------------------------------------------------+# The metric+# ---------------------------------------------------------------------------+++def test_flags_the_overview_and_architecture_duplicate(overview, architecture):+ """Two pages describing the same thing in the same words are flagged."""+ report = measure_orientation_overlap([overview, architecture])++ assert report.comparable+ assert report.pairs_compared == 1+ assert report.flagged_count == 1++ pair = report.flagged[0]+ assert {pair.page_type_a, pair.page_type_b} == {+ "repo_overview",+ "architecture_diagram",+ }+ assert pair.similarity >= report.cross_type_threshold+++def test_does_not_flag_genuinely_different_pages(overview, getting_started):+ """A walkthrough and an overview are compared, and pass."""+ report = measure_orientation_overlap([overview, getting_started])++ assert report.comparable+ assert report.pairs_compared == 1+ assert report.flagged_count == 0+ assert report.highest_similarity == 0.0+++def test_pair_records_the_threshold_it_was_judged_against(overview, architecture):+ """A cross-type pair is judged by the cross-type threshold."""+ report = measure_orientation_overlap([overview, architecture])++ pair = report.flagged[0]+ assert pair.same_type is False+ assert pair.threshold == report.cross_type_threshold+++def test_same_type_pages_are_held_to_their_own_threshold():+ """Pages of one type share a house style, so they get a higher bar.++ One body of text against another, identical in both runs — flagged when+ the two arrive as different page types, not flagged when they arrive as+ two pages of the same type, because the same-type bar sits higher.+ """+ thresholds = {"cross_type_threshold": 0.22, "same_type_threshold": 0.45}++ # Two texts overlapping by exactly 1/3 — above the cross-type bar, below+ # the same-type one, so the routing is what decides the outcome.+ left = " ".join(f"word{i}" for i in range(30))+ right = " ".join(f"word{i}" for i in range(15, 45))++ as_different_types = measure_orientation_overlap(+ [+ _page("repo_overview:.", "repo_overview", "Overview", left),+ _page("architecture_diagram:.", "architecture_diagram", "Diagram", right),+ ],+ **thresholds,+ )+ as_same_type = measure_orientation_overlap(+ [+ _page("layer_page:one", "layer_page", "Layer: One", left),+ _page("layer_page:two", "layer_page", "Layer: Two", right),+ ],+ **thresholds,+ )++ # Both runs compared the same text, so the score is identical; only the+ # bar it is measured against differs.+ assert as_same_type.pairs_compared == as_different_types.pairs_compared == 1+ assert as_different_types.flagged_count == 1+ assert as_different_types.flagged[0].threshold == 0.22+ assert as_same_type.flagged_count == 0+++def test_non_orientation_pages_are_ignored(overview):+ """Reference pages are not part of the orientation set."""+ file_page = _page("file_page:a.py", "file_page", "a.py", _OVERVIEW)+ other_file = _page("file_page:b.py", "file_page", "b.py", _ARCHITECTURE_DIAGRAM)++ report = measure_orientation_overlap([overview, file_page, other_file])++ assert report.pages_compared == 1+ assert report.pairs_compared == 0+ assert not report.comparable+++# ---------------------------------------------------------------------------+# The not-run case must not read as a pass+# ---------------------------------------------------------------------------+++def test_empty_orientation_set_is_not_a_pass():+ """Zero pairs because there was nothing to compare is not zero overlap."""+ report = measure_orientation_overlap([])++ assert report.flagged_count == 0+ assert report.pairs_compared == 0+ assert report.comparable is False+ assert report.highest_similarity is None+ assert "not computed" in report.summary_line()+++def test_no_overlap_is_distinguishable_from_nothing_compared(overview, getting_started):+ """The two zeroes report differently."""+ nothing_compared = measure_orientation_overlap([])+ nothing_overlapped = measure_orientation_overlap([overview, getting_started])++ assert nothing_compared.flagged_count == nothing_overlapped.flagged_count == 0+ assert nothing_compared.comparable is False+ assert nothing_overlapped.comparable is True+ assert nothing_compared.summary_line() != nothing_overlapped.summary_line()+++def test_empty_orientation_set_warns(caplog):+ """A check that compared nothing says so out loud."""+ with caplog.at_level(logging.WARNING, logger=_LOGGER):+ measure_orientation_overlap([])++ assert any("compared no pairs" in r.message for r in caplog.records)+++def test_flagged_pair_warns(caplog, overview, architecture):+ with caplog.at_level(logging.WARNING, logger=_LOGGER):+ measure_orientation_overlap([overview, architecture])++ assert any("overlap" in r.message.lower() for r in caplog.records)+++def test_warnings_survive_a_quietened_parent_logger(caplog, overview, architecture):+ """The CLI silences ``repowise.core`` unless run verbose.++ That is a process-wide level on an ancestor logger, so a test that only+ raises the root level captures nothing once any CLI code has run. Pin the+ behaviour: asking this module's own logger for warnings must work no+ matter what an ancestor was set to.+ """+ parent = logging.getLogger("repowise.core")+ previous = parent.level+ parent.setLevel(logging.ERROR)+ try:+ with caplog.at_level(logging.WARNING, logger=_LOGGER):+ measure_orientation_overlap([overview, architecture])++ assert any("overlap" in r.message.lower() for r in caplog.records)+ finally:+ # Restore, or this test leaks the very state it is guarding against.+ parent.setLevel(previous)+++def test_page_without_content_is_skipped_not_scored(overview):+ """An empty page shares nothing with everything; that is not a result."""+ blank = _page("onboarding:blank", "onboarding", "Blank", "")++ report = measure_orientation_overlap([overview, blank])++ assert report.pages_skipped_empty == 1+ assert report.pages_compared == 1+ assert report.pairs_compared == 0+ assert not report.comparable+++# ---------------------------------------------------------------------------+# Thresholds+# ---------------------------------------------------------------------------+++def test_threshold_is_configurable(overview, architecture):+ strict = measure_orientation_overlap([overview, architecture], cross_type_threshold=0.01)+ lenient = measure_orientation_overlap([overview, architecture], cross_type_threshold=0.99)++ assert strict.flagged_count == 1+ assert lenient.flagged_count == 0+ assert lenient.comparable, "a lenient threshold still compares the pair"++[email protected]("bad", [-0.1, 1.5])+def test_nonsense_threshold_raises(bad, overview, architecture):+ with pytest.raises(ValueError, match=r"between 0\.0 and 1\.0"):+ measure_orientation_overlap([overview, architecture], cross_type_threshold=bad)++ with pytest.raises(ValueError, match=r"between 0\.0 and 1\.0"):+ measure_orientation_overlap([overview, architecture], same_type_threshold=bad)+++def test_jaccard_of_two_empty_sets_is_zero():+ assert jaccard([], []) == 0.0+++def test_orientation_types_are_the_pages_read_before_reference_content():+ assert set(ORIENTATION_PAGE_TYPES) == {+ "onboarding",+ "layer_page",+ "repo_overview",+ "architecture_diagram",+ }+++# ---------------------------------------------------------------------------+# Surfaced on the generation report+# ---------------------------------------------------------------------------+++def test_generation_report_carries_the_overlap_result(overview, architecture):+ report = GenerationReport.from_pages([overview, architecture])++ assert isinstance(report.orientation_overlap, OverlapReport)+ assert report.orientation_overlap.flagged_count == 1+ assert report.orientation_overlap.comparable+++def test_generation_report_overlap_defaults_to_not_computed():+ """A report built from no pages must not claim a clean orientation set."""+ report = GenerationReport.from_pages([])++ assert report.orientation_overlap.comparable is False+ assert report.orientation_overlap.flagged_count == 0