Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(health): compute duplication_pct from the union of clone ranges (#388)
Raghav Chamadiya95fcea8148a0
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/README.md b/packages/core/src/repowise/core/analysis/health/duplication/README.mdindex 38733afa7..f5b2f23a2 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/README.md+++ b/packages/core/src/repowise/core/analysis/health/duplication/README.md@@ -48,8 +48,9 @@ report = detect_clones(parsed_files, git_meta_map)- `DuplicationReport`:- `pairs`: every verified, merged clone region.- - `duplication_pct`: per-file duplicate-line percentage, capped at- 100% to handle overlapping clones.+ - `duplication_pct`: per-file duplicate-line percentage — the union+ of clone-pair line ranges over file NLOC, so overlapping pairs+ don't double-count the same physical lines.- `pairs_by_file`: lookup map used by the `dry_violation` biomarker.## Extension pointsdiff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.pyindex dfaaedb21..105cecf92 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py@@ -469,28 +469,53 @@ def _aggregate(final: list[ClonePair],per_file_nloc: dict[str, int],) -> tuple[dict[str, list[ClonePair]], dict[str, float]]:- """Build the per-file pair index and duplication percentages."""+ """Build the per-file pair index and duplication percentages.++ The percentage is the *union* of the line ranges involved in clone+ pairs, not the sum of per-pair line counts — the same physical lines+ often appear in many pairs (repeated handler blocks), and summing+ would overstate duplication well past 100% (#377).+ """pairs_by_file: dict[str, list[ClonePair]] = defaultdict(list)- dup_lines: dict[str, int] = defaultdict(int)+ dup_ranges: dict[str, list[tuple[int, int]]] = defaultdict(list)for p in final:pairs_by_file[p.file_a].append(p)- dup_lines[p.file_a] += p.a_line_count- if not p.is_intra_file:+ dup_ranges[p.file_a].append((p.a_start_line, p.a_end_line))+ if p.is_intra_file:+ # Both regions live in the same file; the b-side is+ # duplicated coverage too.+ dup_ranges[p.file_a].append((p.b_start_line, p.b_end_line))+ else:pairs_by_file[p.file_b].append(p)- dup_lines[p.file_b] += p.b_line_count+ dup_ranges[p.file_b].append((p.b_start_line, p.b_end_line))duplication_pct: dict[str, float] = {}- for path, dup in dup_lines.items():+ for path, ranges in dup_ranges.items():nloc = per_file_nloc.get(path, 0)if nloc <= 0:continue- # Cap at 100% — overlapping clones can otherwise double-count- # the same physical lines.- duplication_pct[path] = round(min(100.0, 100.0 * dup / nloc), 2)+ # Cap at 100% — covered ranges count physical lines (blanks and+ # comments included) while the denominator is NLOC, so dense+ # clone coverage can still nudge past 100.+ pct = 100.0 * _union_line_count(ranges) / nloc+ duplication_pct[path] = round(min(100.0, pct), 2)return dict(pairs_by_file), duplication_pct+def _union_line_count(ranges: list[tuple[int, int]]) -> int:+ """Total number of distinct lines covered by inclusive ``ranges``."""+ merged_total = 0+ cur_start, cur_end = -1, -2 # empty sentinel+ for start, end in sorted(ranges):+ if start > cur_end + 1:+ merged_total += cur_end - cur_start + 1+ cur_start, cur_end = start, end+ elif end > cur_end:+ cur_end = end+ return merged_total + (cur_end - cur_start + 1)++def _nloc(source: bytes) -> int:try:text = source.decode("utf-8", errors="replace")diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.pyindex 2bc8ff94d..2dd22bbf6 100644--- a/tests/unit/health/test_duplication.py+++ b/tests/unit/health/test_duplication.py@@ -16,6 +16,11 @@ from repowise.core.analysis.health.duplication import (looks_minified,tokenize_file,)+from repowise.core.analysis.health.duplication.detector import (+ ClonePair,+ _aggregate,+ _union_line_count,+)from repowise.core.analysis.health.duplication.rabin_karp import (index_by_hash,rolling_hashes,@@ -262,3 +267,83 @@ def test_detect_clones_reports_diagnostics_on_normal_run(tmp_path: Path):assert report.diagnostics["files_considered"] == 1assert report.diagnostics["degenerate_buckets"] == 0assert report.diagnostics["timed_out"] is False+++# --- duplication_pct aggregation (union, not sum — #377) ----------------+++def _pair(+ file_a: str,+ a_start: int,+ a_end: int,+ file_b: str,+ b_start: int,+ b_end: int,+) -> ClonePair:+ return ClonePair(+ file_a=file_a,+ file_b=file_b,+ a_start_line=a_start,+ a_end_line=a_end,+ b_start_line=b_start,+ b_end_line=b_end,+ token_count=50,+ )+++ ("ranges", "expected"),+ [+ ([(1, 5)], 5),+ ([(1, 5), (10, 12)], 8), # disjoint+ ([(1, 10), (5, 15)], 15), # overlapping+ ([(1, 20), (5, 8)], 20), # nested+ ([(1, 5), (6, 10)], 10), # adjacent+ ([(10, 12), (1, 5)], 8), # unsorted input+ ([(1, 10), (1, 10), (1, 10)], 10), # identical repeats+ ],+)+def test_union_line_count(ranges: list[tuple[int, int]], expected: int):+ assert _union_line_count(ranges) == expected+++def test_aggregate_overlapping_pairs_do_not_double_count():+ # The #377 shape: many clone pairs over the SAME physical lines.+ # Summing per-pair counts gives 10 * 20 = 200 lines over nloc 100+ # (200% → capped 100%); the union is just 20 lines → 20%.+ pairs = [_pair("a.py", 1, 20, f"other{i}.py", 1, 20) for i in range(10)]+ nloc = {"a.py": 100, **{f"other{i}.py": 100 for i in range(10)}}+ _, pct = _aggregate(pairs, nloc)+ assert pct["a.py"] == 20.0+++def test_aggregate_intra_file_pair_counts_both_regions():+ # Both regions of an intra-file clone are duplicated coverage.+ pairs = [_pair("a.py", 1, 10, "a.py", 21, 30)]+ _, pct = _aggregate(pairs, {"a.py": 100})+ assert pct["a.py"] == 20.0+++def test_aggregate_distinct_regions_still_add_up():+ pairs = [+ _pair("a.py", 1, 10, "b.py", 1, 10),+ _pair("a.py", 31, 40, "c.py", 1, 10),+ ]+ _, pct = _aggregate(pairs, {"a.py": 50, "b.py": 50, "c.py": 50})+ assert pct["a.py"] == 40.0+ assert pct["b.py"] == 20.0+ assert pct["c.py"] == 20.0+++def test_aggregate_caps_at_100_when_ranges_exceed_nloc():+ # Covered ranges count physical lines while nloc excludes blanks, so+ # the ratio can still exceed 100 — the cap stays as a safety net.+ pairs = [_pair("a.py", 1, 60, "b.py", 1, 60)]+ _, pct = _aggregate(pairs, {"a.py": 40, "b.py": 40})+ assert pct["a.py"] == 100.0+++def test_aggregate_skips_files_without_nloc():+ pairs = [_pair("a.py", 1, 10, "b.py", 1, 10)]+ _, pct = _aggregate(pairs, {"a.py": 50})+ assert "b.py" not in pct