Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(health): grade dry_violation on code shared with other files only (#2907)

Raghav Chamadiya1b46689f8e51
diff --git a/packages/core/src/repowise/core/analysis/health/biomarkers/dry_violation.py b/packages/core/src/repowise/core/analysis/health/biomarkers/dry_violation.py
index 5b7ccade7..fbd4bd963 100644
--- a/packages/core/src/repowise/core/analysis/health/biomarkers/dry_violation.py
+++ b/packages/core/src/repowise/core/analysis/health/biomarkers/dry_violation.py
@@ -11,10 +11,19 @@ Severity grades on:
The detector reads from ``ctx.clones`` / ``ctx.duplication_pct``, which
the engine populates from ``duplication.detect_clones`` once per
analyze() call.
+
+Only clones shared with another file count. A clone inside one file is
+mostly a run of sibling members with the same shape (getters, operator
+overloads, one-call test actions), which the token-shape matcher pairs with
+each other; its "partner" is the file itself, so headlining it as duplicated
+misreads structure as copying, and co-change, the signal this marker grades
+on, means nothing for a file paired with itself. A real copy inside one file
+is lost with them; that is the price of not reporting the rest.
"""
from __future__ import annotations
+from ..duplication.detector import ClonePair, clone_ranges, union_line_count
from ..models import Severity
from .base import BiomarkerResult, FileContext
@@ -28,16 +37,17 @@ class DryViolationDetector:
category = "duplication"
def detect(self, ctx: FileContext) -> list[BiomarkerResult]:
- if not ctx.clones:
+ cross = [p for p in ctx.clones if not p.is_intra_file]
+ if not cross:
return []
- dup_pct = ctx.duplication_pct or 0.0
+ dup_pct = _cross_file_pct(ctx.file_path, ctx.clones, ctx.duplication_pct or 0.0)
if dup_pct < _MIN_DUP_PCT:
return []
# Pick the worst clone — biggest active region wins over a
# slightly larger dormant clone.
worst = max(
- ctx.clones,
+ cross,
key=lambda p: (p.co_change_count, max(p.a_line_count, p.b_line_count)),
)
worst_lines = max(worst.a_line_count, worst.b_line_count)
@@ -64,14 +74,16 @@ class DryViolationDetector:
else worst.b_start_line,
line_end=worst.a_end_line if worst.file_a == ctx.file_path else worst.b_end_line,
details={
- "duplication_pct": dup_pct,
- "clone_pair_count": len(ctx.clones),
+ # Cross-file only; the file metric's ``duplication_pct``
+ # counts clones inside the file too, so the keys differ.
+ "cross_file_duplication_pct": dup_pct,
+ "clone_pair_count": len(cross),
"worst_clone_lines": worst_lines,
"worst_clone_partner": partner,
"worst_clone_co_change": round(worst.co_change_count, 2),
},
reason=(
- f"{dup_pct:.0f}% of file duplicated; worst clone shares "
+ f"{dup_pct:.0f}% of file duplicated in other files; worst clone shares "
f"{worst_lines} lines with {partner}"
+ (
f" (co-changed {round(worst.co_change_count, 2)}x)"
@@ -83,4 +95,17 @@ class DryViolationDetector:
]
+def _cross_file_pct(path: str, clones: list[ClonePair], file_pct: float) -> float:
+ """The share of the file duplicated in other files, scaled from *file_pct*.
+
+ *file_pct* (every clone, intra-file ones included) is what the detector
+ measured against the file's NLOC, which this marker does not see, so the
+ cross-file share is the same percentage scaled by the lines only
+ cross-file clones cover. Exact unless *file_pct* was capped at 100, and
+ then it can only come out lower.
+ """
+ covered = union_line_count(clone_ranges(path, clones))
+ covered_cross = union_line_count(clone_ranges(path, clones, cross_file_only=True))
+ return round(file_pct * covered_cross / covered, 2) if covered else 0.0
+
BIOMARKER = DryViolationDetector()
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
index 0daf3c428..f9c33e11c 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
@@ -884,13 +884,30 @@ def _aggregate(
# Cap at 100% — covered ranges count physical lines (blanks and
# comments included) while the denominator is NLOC, so dense
# clone coverage can still nudge past 100.
- pct = 100.0 * _union_line_count(ranges) / nloc
+ pct = 100.0 * union_line_count(ranges) / nloc
duplication_pct[path] = round(min(100.0, pct), 2)
return dict(pairs_by_file), duplication_pct
-def _union_line_count(ranges: list[tuple[int, int]]) -> int:
+def clone_ranges(
+ path: str, clones: list[ClonePair], *, cross_file_only: bool = False
+) -> list[tuple[int, int]]:
+ """*path*'s line ranges among *clones*: its side of each cross-file pair
+ and, unless *cross_file_only*, both regions of a clone inside the file."""
+ ranges: list[tuple[int, int]] = []
+ for p in clones:
+ if p.is_intra_file:
+ if not cross_file_only:
+ ranges += [(p.a_start_line, p.a_end_line), (p.b_start_line, p.b_end_line)]
+ elif p.file_a == path:
+ ranges.append((p.a_start_line, p.a_end_line))
+ else:
+ ranges.append((p.b_start_line, p.b_end_line))
+ return ranges
+
+
+def union_line_count(ranges: list[tuple[int, int]]) -> int:
"""Total number of distinct lines covered by inclusive ``ranges``."""
merged_total = 0
cur_start, cur_end = -1, -2 # empty sentinel
diff --git a/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py b/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py
index 2360d53dd..2405a18af 100644
--- a/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py
+++ b/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py
@@ -30,8 +30,10 @@ Precision-first, the gate demands a block genuinely worth a helper:
helper is the wrong fix for them;
- the recovered impact is the block's share of the file's ``dry_violation``
finding when it overlaps the block, else ``0``: the finding grades the
- whole file's duplication, so one block is credited the duplicated lines it
- removes here, not the whole finding (:func:`clone_share`).
+ file's cross-file duplication, so one block is credited the lines of it it
+ removes here, not the whole finding (:func:`clone_share`). A block inside
+ one file recovers ``0``: that finding does not grade it, so it stays
+ evidence of the duplication rather than a step that claims a score.
Confidence rides the co-change signal: a clone whose sites are actively
co-modified is real, maintained duplication (``high``); a dormant clone is
@@ -44,6 +46,7 @@ import re
from typing import Any
from ....test_paths import is_test_related_path
+from ..duplication.detector import clone_ranges, union_line_count
from .models import RefactoringContext, RefactoringSuggestion
from .registry import RefactoringDetector, effort_bucket, register
@@ -476,10 +479,13 @@ class ExtractHelperDetector(RefactoringDetector):
if region_lines and _is_declaration_only(region_lines):
return None
- impact = self._impact_for_block(anchor_region, impact_lookup) * clone_share(
- occurrences, ctx.file_path, self._duplication_pct(ctx), ctx.nloc
- )
is_intra = len(occ_files) == 1
+ impact = (
+ 0.0
+ if is_intra
+ else self._impact_for_block(anchor_region, impact_lookup)
+ * clone_share(occurrences, ctx.file_path, _cross_file_lines(ctx))
+ )
suggested_site = self._suggested_site(occ_files)
snippet, snippet_start, snippet_truncated = self._snippet_for(ctx, anchor_region)
@@ -612,16 +618,6 @@ class ExtractHelperDetector(RefactoringDetector):
out.append((int(start), int(end), impact))
return out
- @staticmethod
- def _duplication_pct(ctx: RefactoringContext) -> float | None:
- """The file's duplicated share, as its ``dry_violation`` finding records it."""
- for f in ctx.findings:
- if getattr(f, "biomarker_type", "") == _SOURCE_BIOMARKER:
- pct = (getattr(f, "details", None) or {}).get("duplication_pct")
- if isinstance(pct, (int, float)):
- return float(pct)
- return None
-
@staticmethod
def _impact_for_block(
region: tuple[int, int], impact_lookup: list[tuple[int, int, float]]
@@ -636,19 +632,23 @@ class ExtractHelperDetector(RefactoringDetector):
return best
+def _cross_file_lines(ctx: RefactoringContext) -> int:
+ """Lines of the file covered by clones shared with other files."""
+ return union_line_count(clone_ranges(ctx.file_path, ctx.clones, cross_file_only=True))
+
+
def clone_share(
- occurrences: list[tuple[str, int, int]], file_path: str, dup_pct: float | None, nloc: int
+ occurrences: list[tuple[str, int, int]], file_path: str, duplicated_lines: int
) -> float:
- """Share of *file_path*'s duplicated lines that extracting this block removes.
+ """Share of *file_path*'s cross-file duplicated lines this block removes.
1.0 when the file's duplication is unknown, keeping the finding's own
impact as the other detectors do when they cannot re-measure it.
"""
- duplicated = (dup_pct or 0.0) * nloc / 100.0
- if duplicated <= 0:
+ if duplicated_lines <= 0:
return 1.0
here = sum(end - start + 1 for path, start, end in occurrences if path == file_path)
- return min(1.0, here / duplicated)
+ return min(1.0, here / duplicated_lines)
def _merge_ranges_per_file(
diff --git a/packages/ui/src/health/biomarker-details.tsx b/packages/ui/src/health/biomarker-details.tsx
index 9f781fb81..cd1ba0fa4 100644
--- a/packages/ui/src/health/biomarker-details.tsx
+++ b/packages/ui/src/health/biomarker-details.tsx
@@ -141,9 +141,14 @@ export function BiomarkerDetails({
if (biomarkerType === "dry_violation") {
const partner = str(details.worst_clone_partner);
- const dup = num(details.duplication_pct);
+ // Cross-file share; rows stored before it was split out carry the
+ // whole-file ``duplication_pct``.
+ const crossDup = num(details.cross_file_duplication_pct);
+ const dup = crossDup ?? num(details.duplication_pct);
const stats = joinStats(
- dup != null ? `${dup.toFixed(0)}% duplicated` : null,
+ dup != null
+ ? `${dup.toFixed(0)}% duplicated${crossDup != null ? " in other files" : ""}`
+ : null,
num(details.clone_pair_count) != null
? `${num(details.clone_pair_count)} clone pairs`
: null,
diff --git a/tests/fixtures/refactoring_corpus/golden_opportunities.json b/tests/fixtures/refactoring_corpus/golden_opportunities.json
index 1dceb2177..99f067d3f 100644
--- a/tests/fixtures/refactoring_corpus/golden_opportunities.json
+++ b/tests/fixtures/refactoring_corpus/golden_opportunities.json
@@ -208,7 +208,7 @@
],
"web": [
{
- "addresses_primary_problem": false,
+ "addresses_primary_problem": true,
"affected_files": [
"handlers.py"
],
@@ -244,7 +244,7 @@
],
"file_path": "handlers.py",
"judgment_steps": 0,
- "lead_biomarker": "dry_violation",
+ "lead_biomarker": "complex_method",
"lead_refactoring_type": "extract_method",
"mechanical_steps": 3,
"opportunity_id": "refop4_2b1f7e87189e7adb819c",
@@ -252,10 +252,10 @@
"benefit": 0.249,
"cost": 3.0,
"mechanical_share": 0.5,
- "primary_problem": 0.0,
+ "primary_problem": 0.5,
"risk": 0.0
},
- "rank_score": 0.0934,
+ "rank_score": 0.1245,
"recoverable_health": 0.249,
"refactoring_model_version": 4,
"step_count": 3,
@@ -346,8 +346,8 @@
"value": 0.5
},
{
- "factor": "benefit",
- "value": 0.249
+ "factor": "primary_problem",
+ "value": 0.5
}
]
}
diff --git a/tests/unit/health/test_dry_violation.py b/tests/unit/health/test_dry_violation.py
index ede1e47d1..e017c04b0 100644
--- a/tests/unit/health/test_dry_violation.py
+++ b/tests/unit/health/test_dry_violation.py
@@ -72,3 +72,41 @@ def test_dry_violation_picks_worst_clone():
out = DryViolationDetector().detect(ctx)
# Active clone wins regardless of size.
assert out[0].details["worst_clone_partner"] == "c.py"
+
+
+def test_dry_violation_ignores_a_file_cloned_only_with_itself():
+ ctx = _ctx("a.py", [_pair("a.py", "a.py", lines=30)], dup_pct=60.0)
+ assert DryViolationDetector().detect(ctx) == []
+
+
+def test_dry_violation_grades_and_names_only_cross_file_clones():
+ # Intra-file lines 10-39 and 20-49 plus a cross-file clone on 60-71: 52
+ # covered lines at 52%, of which the cross-file clone holds 12.
+ intra = _pair("a.py", "a.py", lines=30)
+ cross = ClonePair("a.py", "b.py", 60, 71, 5, 16, token_count=80)
+ out = DryViolationDetector().detect(_ctx("a.py", [intra, cross], dup_pct=52.0))
+ assert len(out) == 1
+ assert out[0].details["worst_clone_partner"] == "b.py"
+ assert out[0].details["cross_file_duplication_pct"] == 12.0
+ assert out[0].details["clone_pair_count"] == 1
+ assert out[0].reason.startswith("12% of file duplicated in other files")
+ # The same file below the floor once its own clones stop counting.
+ assert DryViolationDetector().detect(_ctx("a.py", [intra, cross], dup_pct=30.0)) == []
+
+
+def test_dry_violation_intra_clone_overlapping_a_cross_clone_counts_once():
+ # Intra 10-29 / 40-59 and a cross-file clone on 20-39: 50 covered lines at
+ # 50%, of which the cross-file clone covers 20.
+ intra = ClonePair("a.py", "a.py", 10, 29, 40, 59, token_count=80)
+ cross = ClonePair("a.py", "b.py", 20, 39, 5, 24, token_count=80)
+ out = DryViolationDetector().detect(_ctx("a.py", [intra, cross], dup_pct=50.0))
+ assert out[0].details["cross_file_duplication_pct"] == 20.0
+
+
+def test_dry_violation_capped_file_pct_only_scales_down():
+ # The stored % is capped at 100 even when covered lines pass NLOC; the
+ # cross-file share scales that cap, so it can only come out lower.
+ intra = ClonePair("a.py", "a.py", 1, 60, 61, 120, token_count=80)
+ cross = ClonePair("a.py", "b.py", 1, 30, 1, 30, token_count=80)
+ out = DryViolationDetector().detect(_ctx("a.py", [intra, cross], dup_pct=100.0))
+ assert out[0].details["cross_file_duplication_pct"] == 25.0
diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.py
index 0a0f69975..85ca40482 100644
--- a/tests/unit/health/test_duplication.py
+++ b/tests/unit/health/test_duplication.py
@@ -19,7 +19,7 @@ from repowise.core.analysis.health.duplication import (
from repowise.core.analysis.health.duplication.detector import (
ClonePair,
_aggregate,
- _union_line_count,
+ union_line_count,
)
from repowise.core.analysis.health.duplication.rabin_karp import (
index_by_hash,
@@ -331,8 +331,8 @@ def _pair(
([(1, 10), (1, 10), (1, 10)], 10), # identical repeats
],
)
-def test_union_line_count(ranges: list[tuple[int, int]], expected: int):
- assert _union_line_count(ranges) == expected
+def testunion_line_count(ranges: list[tuple[int, int]], expected: int):
+ assert union_line_count(ranges) == expected
def test_aggregate_overlapping_pairs_do_not_double_count():
diff --git a/tests/unit/health/test_refactoring_extract_helper.py b/tests/unit/health/test_refactoring_extract_helper.py
index dc794d287..b325c82ab 100644
--- a/tests/unit/health/test_refactoring_extract_helper.py
+++ b/tests/unit/health/test_refactoring_extract_helper.py
@@ -168,6 +168,19 @@ def test_intra_file_clone():
assert s.blast_radius["files"] == []
+def test_intra_file_clone_recovers_nothing_even_beside_a_cross_file_finding():
+ # dry_violation grades cross-file duplication only, so an intra-file block
+ # that overlaps its line range must not borrow its impact.
+ intra = _pair("pkg/a.py", "pkg/a.py", 100, 115, 160, 175)
+ cross = _pair("pkg/a.py", "pkg/b.py", 12, 27, 40, 55)
+ findings = [_DryFinding(10, 120, 2.0)]
+ sugs = detect_refactorings(_ctx("pkg/a.py", [intra, cross], findings=findings))
+ intra_sug = next(
+ s for s in sugs if s.refactoring_type == "extract_helper" and s.evidence["is_intra_file"]
+ )
+ assert intra_sug.impact_delta == 0.0
+
+
def test_overlapping_windows_collapse_to_one_site():
# The clone detector emits a block as several offset windows; they must
# coalesce into one occurrence per file, not read as many sites.
@@ -305,6 +318,18 @@ def test_impact_from_overlapping_dry_violation():
assert s.impact_delta == 1.8
+def test_impact_share_counts_cross_file_lines_only():
+ # a.py has 32 cross-file duplicated lines and this block removes 16 of
+ # them; the intra-file clone beside them does not dilute the share.
+ block = _pair("pkg/a.py", "pkg/b.py", 10, 25, 40, 55)
+ other = _pair("pkg/a.py", "pkg/c.py", 100, 115, 1, 16)
+ intra = _pair("pkg/a.py", "pkg/a.py", 200, 230, 300, 330)
+ findings = [_DryFinding(10, 25, 2.0)]
+ sugs = detect_refactorings(_ctx("pkg/a.py", [block, other, intra], findings=findings))
+ s = next(s for s in sugs if s.target_symbol == "a.py:10-25")
+ assert s.impact_delta == 1.0
+
+
def test_no_impact_when_finding_disjoint():
pair = _pair("pkg/a.py", "pkg/b.py", 10, 25, 40, 55)
findings = [_DryFinding(200, 210, 1.8)]