Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): measure vocabulary overlap across orientation pages (#1160)

Swati Ahuja7e5b05926a77
diff --git a/packages/core/src/repowise/core/generation/page_overlap.py b/packages/core/src/repowise/core/generation/page_overlap.py
new file mode 100644
index 000000000..d91e518f4
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/page_overlap.py
@@ -0,0 +1,234 @@
+"""Vocabulary overlap between orientation pages.
+
+Orientation pages (the repository overview, the architecture diagram, the
+layer pages and the onboarding collection) are what a reader meets before any
+reference content. Nothing in generation checks whether two of them say the
+same thing, so near-duplicates accumulate silently: they cost the reader time
+and they give retrieval several adjacent-but-different pages to confuse for
+each other.
+
+This module measures that overlap. Each page is reduced to its vocabulary —
+the set of distinct word-ish tokens it uses — and every pair is scored by
+Jaccard similarity (shared tokens divided by total distinct tokens). It is
+deliberately a vocabulary measure and not an embedding one: it needs no model,
+it is stable across runs, and "these two pages draw on the same words" is
+exactly the property that makes two orientation pages redundant to read.
+
+Pairs are scored in two classes, because the two behave very differently:
+
+* **Different page types** — an overview and an architecture diagram have no
+ reason to share much vocabulary. Real overlap here is a genuine duplicate.
+* **Same page type** — eleven layer pages legitimately share a house style and
+ a domain vocabulary, so they sit much higher as a matter of course. Holding
+ them to the cross-type threshold would flag almost every pair and bury the
+ signal.
+
+Both thresholds are configurable so they can be tuned without a code change.
+
+The report distinguishes "no pairs were comparable" from "pairs were compared
+and none overlapped". Those are opposite facts — the first means the check
+did not run on anything and its silence proves nothing — and collapsing them
+into a single zero would let a broken orientation set read as a clean bill of
+health.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from itertools import combinations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Sequence
+
+ from .models import GeneratedPage
+
+logger = logging.getLogger(__name__)
+
+# The pages a reader meets before any reference content.
+ORIENTATION_PAGE_TYPES: frozenset[str] = frozenset(
+ {"onboarding", "layer_page", "repo_overview", "architecture_diagram"}
+)
+
+# Measured across the orientation set of a full index: pairs of *different*
+# page types sit at or below 0.19, except one known duplicate at 0.24. 0.22
+# separates them with margin on both sides.
+DEFAULT_CROSS_TYPE_THRESHOLD = 0.22
+
+# Pages of the *same* type share a template and a domain vocabulary, so they
+# run far higher — the layer pages have a median near 0.38 with nothing wrong.
+# 0.45 catches the genuinely interchangeable ones without flagging the rest.
+DEFAULT_SAME_TYPE_THRESHOLD = 0.45
+
+_TOKEN_RE = re.compile(r"[a-z0-9_]+")
+
+
+def _vocabulary(content: str) -> frozenset[str]:
+ """The set of distinct tokens in a page, lowercased."""
+ return frozenset(_TOKEN_RE.findall(content.lower()))
+
+
+def jaccard(left: Iterable[str], right: Iterable[str]) -> float:
+ """Shared tokens over total distinct tokens. 0.0 when both are empty."""
+ a, b = frozenset(left), frozenset(right)
+ union = a | b
+ if not union:
+ return 0.0
+ return len(a & b) / len(union)
+
+
+@dataclass(frozen=True)
+class OverlapPair:
+ """Two orientation pages and how much vocabulary they share."""
+
+ page_id_a: str
+ page_id_b: str
+ title_a: str
+ title_b: str
+ page_type_a: str
+ page_type_b: str
+ similarity: float
+ threshold: float
+
+ @property
+ def same_type(self) -> bool:
+ return self.page_type_a == self.page_type_b
+
+ def describe(self) -> str:
+ return (
+ f"{self.title_a!r} and {self.title_b!r} share "
+ f"{self.similarity:.0%} of their vocabulary "
+ f"(threshold {self.threshold:.0%})"
+ )
+
+
+@dataclass(frozen=True)
+class OverlapReport:
+ """Result of comparing the orientation pages in one generation run."""
+
+ pages_compared: int = 0
+ pairs_compared: int = 0
+ pages_skipped_empty: int = 0
+ flagged: tuple[OverlapPair, ...] = field(default_factory=tuple)
+ cross_type_threshold: float = DEFAULT_CROSS_TYPE_THRESHOLD
+ same_type_threshold: float = DEFAULT_SAME_TYPE_THRESHOLD
+
+ @property
+ def comparable(self) -> bool:
+ """Whether the check actually compared anything.
+
+ ``False`` means the run produced fewer than two orientation pages with
+ content, so ``flagged`` being empty says nothing about duplication.
+ Callers must not read an empty ``flagged`` as a pass without checking
+ this first.
+ """
+ return self.pairs_compared > 0
+
+ @property
+ def flagged_count(self) -> int:
+ return len(self.flagged)
+
+ @property
+ def highest_similarity(self) -> float | None:
+ """The worst pair's score, or ``None`` if nothing was comparable."""
+ if not self.pairs_compared:
+ return None
+ return max((p.similarity for p in self.flagged), default=0.0)
+
+ def summary_line(self) -> str:
+ """One line fit for a report table, honest about the not-run case."""
+ if not self.comparable:
+ return f"not computed ({self.pages_compared} orientation pages)"
+ if not self.flagged:
+ return f"no overlapping pairs ({self.pairs_compared} compared)"
+ return f"{self.flagged_count} of {self.pairs_compared} pairs overlap"
+
+
+def measure_orientation_overlap(
+ pages: Sequence[GeneratedPage],
+ *,
+ cross_type_threshold: float = DEFAULT_CROSS_TYPE_THRESHOLD,
+ same_type_threshold: float = DEFAULT_SAME_TYPE_THRESHOLD,
+) -> OverlapReport:
+ """Score vocabulary overlap across the orientation pages in ``pages``.
+
+ Non-orientation pages are ignored. Pages whose content has no tokens are
+ counted in ``pages_skipped_empty`` rather than scored, because an empty
+ page trivially shares nothing and would otherwise look like a clean result.
+
+ Raises:
+ ValueError: if either threshold is outside 0.0-1.0. A nonsensical
+ threshold would silently flag everything or nothing.
+ """
+ for name, value in (
+ ("cross_type_threshold", cross_type_threshold),
+ ("same_type_threshold", same_type_threshold),
+ ):
+ if not 0.0 <= value <= 1.0:
+ raise ValueError(f"{name} must be between 0.0 and 1.0, got {value!r}")
+
+ scored: list[tuple[GeneratedPage, frozenset[str]]] = []
+ skipped = 0
+ for page in pages:
+ if page.page_type not in ORIENTATION_PAGE_TYPES:
+ continue
+ vocab = _vocabulary(page.content or "")
+ if not vocab:
+ skipped += 1
+ logger.warning(
+ "Orientation page %r has no comparable content; "
+ "excluding it from the overlap check",
+ page.page_id,
+ )
+ continue
+ scored.append((page, vocab))
+
+ flagged: list[OverlapPair] = []
+ pairs_compared = 0
+ for (page_a, vocab_a), (page_b, vocab_b) in combinations(scored, 2):
+ pairs_compared += 1
+ threshold = (
+ same_type_threshold if page_a.page_type == page_b.page_type else cross_type_threshold
+ )
+ similarity = jaccard(vocab_a, vocab_b)
+ if similarity >= threshold:
+ flagged.append(
+ OverlapPair(
+ page_id_a=page_a.page_id,
+ page_id_b=page_b.page_id,
+ title_a=page_a.title,
+ title_b=page_b.title,
+ page_type_a=page_a.page_type,
+ page_type_b=page_b.page_type,
+ similarity=similarity,
+ threshold=threshold,
+ )
+ )
+
+ flagged.sort(key=lambda p: (-p.similarity, p.page_id_a, p.page_id_b))
+
+ report = OverlapReport(
+ pages_compared=len(scored),
+ pairs_compared=pairs_compared,
+ pages_skipped_empty=skipped,
+ flagged=tuple(flagged),
+ cross_type_threshold=cross_type_threshold,
+ same_type_threshold=same_type_threshold,
+ )
+
+ if not report.comparable:
+ # Loud on purpose: a run that compared nothing must not be mistaken
+ # for a run that found nothing.
+ logger.warning(
+ "Orientation overlap check compared no pairs (%d pages scored, "
+ "%d skipped as empty); an empty result here is not a pass",
+ report.pages_compared,
+ skipped,
+ )
+ else:
+ for pair in flagged:
+ logger.warning("Orientation pages overlap: %s", pair.describe())
+
+ return report
diff --git a/packages/core/src/repowise/core/generation/report.py b/packages/core/src/repowise/core/generation/report.py
index d5d59895a..4231a7822 100644
--- a/packages/core/src/repowise/core/generation/report.py
+++ b/packages/core/src/repowise/core/generation/report.py
@@ -9,6 +9,7 @@ from collections import Counter
from dataclasses import dataclass, field
from .models import GeneratedPage
+from .page_overlap import OverlapReport, measure_orientation_overlap
@dataclass
@@ -25,6 +26,11 @@ class GenerationReport:
elapsed_seconds: float = 0.0
hallucination_warning_count: int = 0
self_repaired_page_count: int = 0
+ # Vocabulary overlap across the orientation pages this run produced.
+ # Warn-only: a flagged pair is reported, never fatal. Read
+ # ``comparable`` before reading ``flagged`` — an empty ``flagged`` on a
+ # run that compared nothing is not a clean result.
+ orientation_overlap: OverlapReport = field(default_factory=OverlapReport)
@classmethod
def from_pages(
@@ -50,6 +56,7 @@ class GenerationReport:
elapsed_seconds=elapsed,
hallucination_warning_count=hal_count,
self_repaired_page_count=repair_count,
+ orientation_overlap=measure_orientation_overlap(pages),
)
@property
@@ -98,4 +105,15 @@ def render_report(report: GenerationReport, console: object) -> None:
if report.self_repaired_page_count:
table.add_row("Self-repaired pages", str(report.self_repaired_page_count))
+ # Always shown, including when nothing was comparable. Hiding the row on
+ # a zero would make "the check did not run" look like "the check passed".
+ overlap = report.orientation_overlap
+ overlap_text = overlap.summary_line()
+ if overlap.flagged:
+ overlap_text = f"[yellow]{overlap_text}[/yellow]"
+ table.add_row("Orientation overlap", overlap_text)
+
console.print(table) # type: ignore[union-attr]
+
+ for pair in overlap.flagged:
+ console.print(f" [yellow]overlap[/yellow] {pair.describe()}") # type: ignore[union-attr]
diff --git a/tests/unit/generation/test_page_overlap.py b/tests/unit/generation/test_page_overlap.py
new file mode 100644
index 000000000..174ce2cde
--- /dev/null
+++ b/tests/unit/generation/test_page_overlap.py
@@ -0,0 +1,326 @@
+"""Vocabulary overlap between orientation pages."""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+from repowise.core.generation.models import GeneratedPage
+from repowise.core.generation.page_overlap import (
+ ORIENTATION_PAGE_TYPES,
+ OverlapReport,
+ jaccard,
+ measure_orientation_overlap,
+)
+from repowise.core.generation.report import GenerationReport
+
+# Warnings are asserted against this logger by name rather than the root
+# logger. The CLI sets ``repowise.core`` to ERROR unless run verbose, and that
+# level survives for the whole process — raising only the root level would
+# capture nothing once any CLI code has run in the same session.
+_LOGGER = "repowise.core.generation.page_overlap"
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+# The overview and the architecture diagram are the known duplicate pair: both
+# describe the same repository at the same altitude, in the same words.
+_OVERVIEW = """
+Repowise is a repository documentation engine. It ingests a codebase and its
+git metadata, parses the source into structured entities, resolves the
+cross-references and dependencies between them, and generates wiki pages and
+graph data that an API serves to a web UI. The ingestion layer walks the tree,
+the analysis layer scores health and risk, and the generation layer synthesises
+the prose. Persistence keeps the parsed entities and the generated pages.
+"""
+
+_ARCHITECTURE_DIAGRAM = """
+This diagram shows the repowise documentation engine. A codebase and its git
+metadata are ingested, the source is parsed into structured entities, the
+cross-references and dependencies between them are resolved, and wiki pages
+and graph data are generated for an API that serves a web UI. Ingestion walks
+the tree, analysis scores health and risk, generation synthesises the prose,
+and persistence keeps the parsed entities and the generated pages.
+"""
+
+# Genuinely different: a first-run walkthrough shares almost no vocabulary with
+# an architectural description.
+_GETTING_STARTED = """
+Install the command line tool with pip. Point it at a checkout and run the
+index command; the first pass takes a few minutes on a large tree. When it
+finishes, open the local server in a browser to read what it wrote. If you
+would rather stay in your editor, install the extension and use the sidebar.
+Set your API key in the environment first or the run will stop and tell you.
+"""
+
+
+def _page(
+ page_id: str,
+ page_type: str,
+ title: str,
+ content: str,
+) -> GeneratedPage:
+ return GeneratedPage(
+ page_id=page_id,
+ page_type=page_type,
+ title=title,
+ content=content,
+ source_hash="",
+ model_name="mock",
+ provider_name="mock",
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ generation_level=6,
+ target_path="",
+ created_at="2026-07-29T00:00:00Z",
+ updated_at="2026-07-29T00:00:00Z",
+ )
+
+
+def overview() -> GeneratedPage:
+ return _page("repo_overview:.", "repo_overview", "Repository Overview", _OVERVIEW)
+
+
+def architecture() -> GeneratedPage:
+ return _page(
+ "architecture_diagram:.",
+ "architecture_diagram",
+ "Architecture Diagram",
+ _ARCHITECTURE_DIAGRAM,
+ )
+
+
+def getting_started() -> GeneratedPage:
+ return _page(
+ "onboarding:getting-started",
+ "onboarding",
+ "Getting Started",
+ _GETTING_STARTED,
+ )
+
+
+# ---------------------------------------------------------------------------
+# The metric
+# ---------------------------------------------------------------------------
+
+
+def test_flags_the_overview_and_architecture_duplicate(overview, architecture):
+ """Two pages describing the same thing in the same words are flagged."""
+ report = measure_orientation_overlap([overview, architecture])
+
+ assert report.comparable
+ assert report.pairs_compared == 1
+ assert report.flagged_count == 1
+
+ pair = report.flagged[0]
+ assert {pair.page_type_a, pair.page_type_b} == {
+ "repo_overview",
+ "architecture_diagram",
+ }
+ assert pair.similarity >= report.cross_type_threshold
+
+
+def test_does_not_flag_genuinely_different_pages(overview, getting_started):
+ """A walkthrough and an overview are compared, and pass."""
+ report = measure_orientation_overlap([overview, getting_started])
+
+ assert report.comparable
+ assert report.pairs_compared == 1
+ assert report.flagged_count == 0
+ assert report.highest_similarity == 0.0
+
+
+def test_pair_records_the_threshold_it_was_judged_against(overview, architecture):
+ """A cross-type pair is judged by the cross-type threshold."""
+ report = measure_orientation_overlap([overview, architecture])
+
+ pair = report.flagged[0]
+ assert pair.same_type is False
+ assert pair.threshold == report.cross_type_threshold
+
+
+def test_same_type_pages_are_held_to_their_own_threshold():
+ """Pages of one type share a house style, so they get a higher bar.
+
+ One body of text against another, identical in both runs — flagged when
+ the two arrive as different page types, not flagged when they arrive as
+ two pages of the same type, because the same-type bar sits higher.
+ """
+ thresholds = {"cross_type_threshold": 0.22, "same_type_threshold": 0.45}
+
+ # Two texts overlapping by exactly 1/3 — above the cross-type bar, below
+ # the same-type one, so the routing is what decides the outcome.
+ left = " ".join(f"word{i}" for i in range(30))
+ right = " ".join(f"word{i}" for i in range(15, 45))
+
+ as_different_types = measure_orientation_overlap(
+ [
+ _page("repo_overview:.", "repo_overview", "Overview", left),
+ _page("architecture_diagram:.", "architecture_diagram", "Diagram", right),
+ ],
+ **thresholds,
+ )
+ as_same_type = measure_orientation_overlap(
+ [
+ _page("layer_page:one", "layer_page", "Layer: One", left),
+ _page("layer_page:two", "layer_page", "Layer: Two", right),
+ ],
+ **thresholds,
+ )
+
+ # Both runs compared the same text, so the score is identical; only the
+ # bar it is measured against differs.
+ assert as_same_type.pairs_compared == as_different_types.pairs_compared == 1
+ assert as_different_types.flagged_count == 1
+ assert as_different_types.flagged[0].threshold == 0.22
+ assert as_same_type.flagged_count == 0
+
+
+def test_non_orientation_pages_are_ignored(overview):
+ """Reference pages are not part of the orientation set."""
+ file_page = _page("file_page:a.py", "file_page", "a.py", _OVERVIEW)
+ other_file = _page("file_page:b.py", "file_page", "b.py", _ARCHITECTURE_DIAGRAM)
+
+ report = measure_orientation_overlap([overview, file_page, other_file])
+
+ assert report.pages_compared == 1
+ assert report.pairs_compared == 0
+ assert not report.comparable
+
+
+# ---------------------------------------------------------------------------
+# The not-run case must not read as a pass
+# ---------------------------------------------------------------------------
+
+
+def test_empty_orientation_set_is_not_a_pass():
+ """Zero pairs because there was nothing to compare is not zero overlap."""
+ report = measure_orientation_overlap([])
+
+ assert report.flagged_count == 0
+ assert report.pairs_compared == 0
+ assert report.comparable is False
+ assert report.highest_similarity is None
+ assert "not computed" in report.summary_line()
+
+
+def test_no_overlap_is_distinguishable_from_nothing_compared(overview, getting_started):
+ """The two zeroes report differently."""
+ nothing_compared = measure_orientation_overlap([])
+ nothing_overlapped = measure_orientation_overlap([overview, getting_started])
+
+ assert nothing_compared.flagged_count == nothing_overlapped.flagged_count == 0
+ assert nothing_compared.comparable is False
+ assert nothing_overlapped.comparable is True
+ assert nothing_compared.summary_line() != nothing_overlapped.summary_line()
+
+
+def test_empty_orientation_set_warns(caplog):
+ """A check that compared nothing says so out loud."""
+ with caplog.at_level(logging.WARNING, logger=_LOGGER):
+ measure_orientation_overlap([])
+
+ assert any("compared no pairs" in r.message for r in caplog.records)
+
+
+def test_flagged_pair_warns(caplog, overview, architecture):
+ with caplog.at_level(logging.WARNING, logger=_LOGGER):
+ measure_orientation_overlap([overview, architecture])
+
+ assert any("overlap" in r.message.lower() for r in caplog.records)
+
+
+def test_warnings_survive_a_quietened_parent_logger(caplog, overview, architecture):
+ """The CLI silences ``repowise.core`` unless run verbose.
+
+ That is a process-wide level on an ancestor logger, so a test that only
+ raises the root level captures nothing once any CLI code has run. Pin the
+ behaviour: asking this module's own logger for warnings must work no
+ matter what an ancestor was set to.
+ """
+ parent = logging.getLogger("repowise.core")
+ previous = parent.level
+ parent.setLevel(logging.ERROR)
+ try:
+ with caplog.at_level(logging.WARNING, logger=_LOGGER):
+ measure_orientation_overlap([overview, architecture])
+
+ assert any("overlap" in r.message.lower() for r in caplog.records)
+ finally:
+ # Restore, or this test leaks the very state it is guarding against.
+ parent.setLevel(previous)
+
+
+def test_page_without_content_is_skipped_not_scored(overview):
+ """An empty page shares nothing with everything; that is not a result."""
+ blank = _page("onboarding:blank", "onboarding", "Blank", "")
+
+ report = measure_orientation_overlap([overview, blank])
+
+ assert report.pages_skipped_empty == 1
+ assert report.pages_compared == 1
+ assert report.pairs_compared == 0
+ assert not report.comparable
+
+
+# ---------------------------------------------------------------------------
+# Thresholds
+# ---------------------------------------------------------------------------
+
+
+def test_threshold_is_configurable(overview, architecture):
+ strict = measure_orientation_overlap([overview, architecture], cross_type_threshold=0.01)
+ lenient = measure_orientation_overlap([overview, architecture], cross_type_threshold=0.99)
+
+ assert strict.flagged_count == 1
+ assert lenient.flagged_count == 0
+ assert lenient.comparable, "a lenient threshold still compares the pair"
+
+
[email protected]("bad", [-0.1, 1.5])
+def test_nonsense_threshold_raises(bad, overview, architecture):
+ with pytest.raises(ValueError, match=r"between 0\.0 and 1\.0"):
+ measure_orientation_overlap([overview, architecture], cross_type_threshold=bad)
+
+ with pytest.raises(ValueError, match=r"between 0\.0 and 1\.0"):
+ measure_orientation_overlap([overview, architecture], same_type_threshold=bad)
+
+
+def test_jaccard_of_two_empty_sets_is_zero():
+ assert jaccard([], []) == 0.0
+
+
+def test_orientation_types_are_the_pages_read_before_reference_content():
+ assert set(ORIENTATION_PAGE_TYPES) == {
+ "onboarding",
+ "layer_page",
+ "repo_overview",
+ "architecture_diagram",
+ }
+
+
+# ---------------------------------------------------------------------------
+# Surfaced on the generation report
+# ---------------------------------------------------------------------------
+
+
+def test_generation_report_carries_the_overlap_result(overview, architecture):
+ report = GenerationReport.from_pages([overview, architecture])
+
+ assert isinstance(report.orientation_overlap, OverlapReport)
+ assert report.orientation_overlap.flagged_count == 1
+ assert report.orientation_overlap.comparable
+
+
+def test_generation_report_overlap_defaults_to_not_computed():
+ """A report built from no pages must not claim a clean orientation set."""
+ report = GenerationReport.from_pages([])
+
+ assert report.orientation_overlap.comparable is False
+ assert report.orientation_overlap.flagged_count == 0