Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(health): compute duplication_pct from the union of clone ranges (#388)

Raghav Chamadiya95fcea8148a0
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/README.md b/packages/core/src/repowise/core/analysis/health/duplication/README.md
index 38733afa7..f5b2f23a2 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/README.md
+++ b/packages/core/src/repowise/core/analysis/health/duplication/README.md
@@ -48,8 +48,9 @@ report = detect_clones(parsed_files, git_meta_map)
- `DuplicationReport`:
- `pairs`: every verified, merged clone region.
- - `duplication_pct`: per-file duplicate-line percentage, capped at
- 100% to handle overlapping clones.
+ - `duplication_pct`: per-file duplicate-line percentage — the union
+ of clone-pair line ranges over file NLOC, so overlapping pairs
+ don't double-count the same physical lines.
- `pairs_by_file`: lookup map used by the `dry_violation` biomarker.
## Extension points
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
index dfaaedb21..105cecf92 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
@@ -469,28 +469,53 @@ def _aggregate(
final: list[ClonePair],
per_file_nloc: dict[str, int],
) -> tuple[dict[str, list[ClonePair]], dict[str, float]]:
- """Build the per-file pair index and duplication percentages."""
+ """Build the per-file pair index and duplication percentages.
+
+ The percentage is the *union* of the line ranges involved in clone
+ pairs, not the sum of per-pair line counts — the same physical lines
+ often appear in many pairs (repeated handler blocks), and summing
+ would overstate duplication well past 100% (#377).
+ """
pairs_by_file: dict[str, list[ClonePair]] = defaultdict(list)
- dup_lines: dict[str, int] = defaultdict(int)
+ dup_ranges: dict[str, list[tuple[int, int]]] = defaultdict(list)
for p in final:
pairs_by_file[p.file_a].append(p)
- dup_lines[p.file_a] += p.a_line_count
- if not p.is_intra_file:
+ dup_ranges[p.file_a].append((p.a_start_line, p.a_end_line))
+ if p.is_intra_file:
+ # Both regions live in the same file; the b-side is
+ # duplicated coverage too.
+ dup_ranges[p.file_a].append((p.b_start_line, p.b_end_line))
+ else:
pairs_by_file[p.file_b].append(p)
- dup_lines[p.file_b] += p.b_line_count
+ dup_ranges[p.file_b].append((p.b_start_line, p.b_end_line))
duplication_pct: dict[str, float] = {}
- for path, dup in dup_lines.items():
+ for path, ranges in dup_ranges.items():
nloc = per_file_nloc.get(path, 0)
if nloc <= 0:
continue
- # Cap at 100% — overlapping clones can otherwise double-count
- # the same physical lines.
- duplication_pct[path] = round(min(100.0, 100.0 * dup / nloc), 2)
+ # Cap at 100% — covered ranges count physical lines (blanks and
+ # comments included) while the denominator is NLOC, so dense
+ # clone coverage can still nudge past 100.
+ pct = 100.0 * _union_line_count(ranges) / nloc
+ duplication_pct[path] = round(min(100.0, pct), 2)
return dict(pairs_by_file), duplication_pct
+def _union_line_count(ranges: list[tuple[int, int]]) -> int:
+ """Total number of distinct lines covered by inclusive ``ranges``."""
+ merged_total = 0
+ cur_start, cur_end = -1, -2 # empty sentinel
+ for start, end in sorted(ranges):
+ if start > cur_end + 1:
+ merged_total += cur_end - cur_start + 1
+ cur_start, cur_end = start, end
+ elif end > cur_end:
+ cur_end = end
+ return merged_total + (cur_end - cur_start + 1)
+
+
def _nloc(source: bytes) -> int:
try:
text = source.decode("utf-8", errors="replace")
diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.py
index 2bc8ff94d..2dd22bbf6 100644
--- a/tests/unit/health/test_duplication.py
+++ b/tests/unit/health/test_duplication.py
@@ -16,6 +16,11 @@ from repowise.core.analysis.health.duplication import (
looks_minified,
tokenize_file,
)
+from repowise.core.analysis.health.duplication.detector import (
+ ClonePair,
+ _aggregate,
+ _union_line_count,
+)
from repowise.core.analysis.health.duplication.rabin_karp import (
index_by_hash,
rolling_hashes,
@@ -262,3 +267,83 @@ def test_detect_clones_reports_diagnostics_on_normal_run(tmp_path: Path):
assert report.diagnostics["files_considered"] == 1
assert report.diagnostics["degenerate_buckets"] == 0
assert report.diagnostics["timed_out"] is False
+
+
+# --- duplication_pct aggregation (union, not sum — #377) ----------------
+
+
+def _pair(
+ file_a: str,
+ a_start: int,
+ a_end: int,
+ file_b: str,
+ b_start: int,
+ b_end: int,
+) -> ClonePair:
+ return ClonePair(
+ file_a=file_a,
+ file_b=file_b,
+ a_start_line=a_start,
+ a_end_line=a_end,
+ b_start_line=b_start,
+ b_end_line=b_end,
+ token_count=50,
+ )
+
+
+ ("ranges", "expected"),
+ [
+ ([(1, 5)], 5),
+ ([(1, 5), (10, 12)], 8), # disjoint
+ ([(1, 10), (5, 15)], 15), # overlapping
+ ([(1, 20), (5, 8)], 20), # nested
+ ([(1, 5), (6, 10)], 10), # adjacent
+ ([(10, 12), (1, 5)], 8), # unsorted input
+ ([(1, 10), (1, 10), (1, 10)], 10), # identical repeats
+ ],
+)
+def test_union_line_count(ranges: list[tuple[int, int]], expected: int):
+ assert _union_line_count(ranges) == expected
+
+
+def test_aggregate_overlapping_pairs_do_not_double_count():
+ # The #377 shape: many clone pairs over the SAME physical lines.
+ # Summing per-pair counts gives 10 * 20 = 200 lines over nloc 100
+ # (200% → capped 100%); the union is just 20 lines → 20%.
+ pairs = [_pair("a.py", 1, 20, f"other{i}.py", 1, 20) for i in range(10)]
+ nloc = {"a.py": 100, **{f"other{i}.py": 100 for i in range(10)}}
+ _, pct = _aggregate(pairs, nloc)
+ assert pct["a.py"] == 20.0
+
+
+def test_aggregate_intra_file_pair_counts_both_regions():
+ # Both regions of an intra-file clone are duplicated coverage.
+ pairs = [_pair("a.py", 1, 10, "a.py", 21, 30)]
+ _, pct = _aggregate(pairs, {"a.py": 100})
+ assert pct["a.py"] == 20.0
+
+
+def test_aggregate_distinct_regions_still_add_up():
+ pairs = [
+ _pair("a.py", 1, 10, "b.py", 1, 10),
+ _pair("a.py", 31, 40, "c.py", 1, 10),
+ ]
+ _, pct = _aggregate(pairs, {"a.py": 50, "b.py": 50, "c.py": 50})
+ assert pct["a.py"] == 40.0
+ assert pct["b.py"] == 20.0
+ assert pct["c.py"] == 20.0
+
+
+def test_aggregate_caps_at_100_when_ranges_exceed_nloc():
+ # Covered ranges count physical lines while nloc excludes blanks, so
+ # the ratio can still exceed 100 — the cap stays as a safety net.
+ pairs = [_pair("a.py", 1, 60, "b.py", 1, 60)]
+ _, pct = _aggregate(pairs, {"a.py": 40, "b.py": 40})
+ assert pct["a.py"] == 100.0
+
+
+def test_aggregate_skips_files_without_nloc():
+ pairs = [_pair("a.py", 1, 10, "b.py", 1, 10)]
+ _, pct = _aggregate(pairs, {"a.py": 50})
+ assert "b.py" not in pct