Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(health): bound duplication detection so minified bundles can't wedge `init` (#342)
Raghav Chamadiya5ecff5aa00a4
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/__init__.py b/packages/core/src/repowise/core/analysis/health/duplication/__init__.pyindex 6621d5d8f..1d14bfacd 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/__init__.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/__init__.py@@ -7,15 +7,19 @@ from .detector import (DuplicationReport,detect_clones,)+from .limits import DuplicationDiagnostics, DuplicationLimits, looks_minifiedfrom .tokenizer import Token, tokenize_file, tokenize_tree__all__ = ["DEFAULT_MIN_LINES","DEFAULT_WINDOW_TOKENS","ClonePair",+ "DuplicationDiagnostics",+ "DuplicationLimits","DuplicationReport","Token","detect_clones",+ "looks_minified","tokenize_file","tokenize_tree",]diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.pyindex aedbf1e44..7e8063616 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py@@ -21,6 +21,7 @@ size.from __future__ import annotationsimport json+import timefrom collections import defaultdictfrom collections.abc import Iterablefrom dataclasses import dataclass, field@@ -29,6 +30,7 @@ from typing import Anyimport structlog+from .limits import DuplicationDiagnostics, DuplicationLimits, looks_minifiedfrom .rabin_karp import WindowHash, index_by_hash, rolling_hashesfrom .tokenizer import Token, tokenize_file@@ -76,6 +78,10 @@ class DuplicationReport:duplication_pct: dict[str, float] = field(default_factory=dict)# Per-file pair index for fast lookups by biomarker.pairs_by_file: dict[str, list[ClonePair]] = field(default_factory=dict)+ # Flat counters describing how the resource guards behaved (files+ # skipped as minified, degenerate buckets dropped, deadline hit, …).+ # Empty on the no-op fallback path. See ``limits.DuplicationDiagnostics``.+ diagnostics: dict[str, int | bool] = field(default_factory=dict)def _read_source(abs_path: str) -> bytes | None:@@ -186,74 +192,197 @@ def detect_clones(*,window_tokens: int = DEFAULT_WINDOW_TOKENS,min_lines: int = DEFAULT_MIN_LINES,+ limits: DuplicationLimits | None = None,) -> DuplicationReport:"""Run the duplication pipeline over the supplied parsed files.- The detector reads source bytes from ``ParsedFile.file_info.abs_path``- and tokenizes each file once. Skipping files where tokenization- returns an empty list (unsupported language, parse failure) keeps- the rest of the pipeline running.+ Thin orchestrator over four bounded stages:++ 1. :func:`_collect_windows` — read + tokenize each file, skipping+ minified/generated content and over-budget files.+ 2. ``index_by_hash`` — group candidate windows by rolling hash.+ 3. :func:`_pairs_from_buckets` — verify collisions into clone pairs,+ capping degenerate buckets and honouring a wall-clock deadline.+ 4. :func:`_finalize_pairs` / :func:`_aggregate` — merge, filter by+ size, weight by co-change, and roll up per-file metrics.++ Every stage is bounded by :class:`~.limits.DuplicationLimits` so no+ repo shape (minified bundles, generated tables) can wedge the run —+ see issue #341."""meta_map = git_meta_map or {}+ lim = limits or DuplicationLimits()+ diag = DuplicationDiagnostics()++ per_file_tokens, per_file_nloc, all_windows = _collect_windows(+ parsed_files, window_tokens, lim, diag+ )+ if not all_windows:+ return DuplicationReport(diagnostics=diag.as_log_fields())++ bucket = index_by_hash(all_windows)+ raw_pairs = _pairs_from_buckets(bucket, per_file_tokens, window_tokens, lim, diag)++ final = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)+ pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)++ return DuplicationReport(+ pairs=final,+ duplication_pct=duplication_pct,+ pairs_by_file=pairs_by_file,+ diagnostics=diag.as_log_fields(),+ )+++# ---------------------------------------------------------------------------+# Stage 1 — windowing+# ---------------------------------------------------------------------------+++def _collect_windows(+ parsed_files: Iterable[Any],+ window_tokens: int,+ limits: DuplicationLimits,+ diag: DuplicationDiagnostics,+) -> tuple[dict[str, list[Token]], dict[str, int], list[WindowHash]]:+ """Tokenize each file once and emit its rolling-hash windows.++ Files are dropped (and counted in *diag*) when they are unreadable,+ minified/generated, shorter than one window, or exceed the per-file+ token cap. Collection stops cleanly once the repo-wide window budget+ is reached so peak memory stays bounded.+ """per_file_tokens: dict[str, list[Token]] = {}per_file_nloc: dict[str, int] = {}all_windows: list[WindowHash] = []for pf in parsed_files:+ diag.files_considered += 1path = pf.file_info.pathlanguage = pf.file_info.language+source = _read_source(pf.file_info.abs_path)if source is None:+ diag.skipped_unreadable += 1continue+ if looks_minified(source, limits):+ diag.skipped_minified += 1+ continue+toks = tokenize_file(language, source)if len(toks) < window_tokens:continue+ if len(toks) > limits.max_tokens_per_file:+ diag.skipped_token_cap += 1+ continue++ windows = rolling_hashes(path, toks, window_tokens)+ if len(all_windows) + len(windows) > limits.max_total_windows:+ diag.window_budget_hit = True+ break+per_file_tokens[path] = toksper_file_nloc[path] = _nloc(source)- all_windows.extend(rolling_hashes(path, toks, window_tokens))+ all_windows.extend(windows)+ diag.files_tokenized += 1- if not all_windows:- return DuplicationReport()+ diag.total_windows = len(all_windows)+ return per_file_tokens, per_file_nloc, all_windows- bucket = index_by_hash(all_windows)++# ---------------------------------------------------------------------------+# Stage 2 — bucket verification+# ---------------------------------------------------------------------------+++def _pairs_from_buckets(+ bucket: dict[int, list[WindowHash]],+ per_file_tokens: dict[str, list[Token]],+ window_tokens: int,+ limits: DuplicationLimits,+ diag: DuplicationDiagnostics,+) -> list[ClonePair]:+ """Verify each hash bucket into clone pairs.++ Buckets larger than ``limits.max_bucket_windows`` are degenerate+ repetition (boilerplate, generated code) — emitting their O(k²) pairs+ is pure noise, so they are skipped and counted. A soft wall-clock+ deadline guards against any unanticipated pathology: on expiry we+ return the pairs found so far rather than spinning indefinitely.+ """raw_pairs: list[ClonePair] = []seen: set[tuple[str, int, str, int]] = set()+ deadline = (time.monotonic() + limits.time_budget_secs) if limits.time_budget_secs else None- for windows in bucket.values():+ for i, windows in enumerate(bucket.values()):if len(windows) < 2:continue- # All-pairs within a bucket — buckets are small in practice.- for i in range(len(windows)):- for j in range(i + 1, len(windows)):- a, b = windows[i], windows[j]- # Canonicalize so (file_a, file_b) is stable.- if (a.file_path, a.start_index) > (b.file_path, b.start_index):- a, b = b, a- key = (a.file_path, a.start_index, b.file_path, b.start_index)- if key in seen:- continue- seen.add(key)- if not _tokens_equal(- per_file_tokens[a.file_path],- per_file_tokens[b.file_path],- a.start_index,- b.start_index,- window_tokens,- ):- continue- raw_pairs.append(- ClonePair(- file_a=a.file_path,- file_b=b.file_path,- a_start_line=a.start_line,- a_end_line=a.end_line,- b_start_line=b.start_line,- b_end_line=b.end_line,- token_count=window_tokens,- )+ if len(windows) > limits.max_bucket_windows:+ diag.degenerate_buckets += 1+ continue+ # Check the clock occasionally (cheap) rather than per-pair.+ if deadline is not None and (i & 0x3FF) == 0 and time.monotonic() > deadline:+ diag.timed_out = True+ break+ _verify_bucket(windows, per_file_tokens, window_tokens, seen, raw_pairs)++ return raw_pairs+++def _verify_bucket(+ windows: list[WindowHash],+ per_file_tokens: dict[str, list[Token]],+ window_tokens: int,+ seen: set[tuple[str, int, str, int]],+ out: list[ClonePair],+) -> None:+ """Confirm every unordered pair in one (bounded) hash bucket.++ Hash equality is necessary but not sufficient — ``_tokens_equal``+ rejects collisions by comparing the actual token sequences.+ """+ for i in range(len(windows)):+ for j in range(i + 1, len(windows)):+ a, b = windows[i], windows[j]+ # Canonicalize so (file_a, file_b) ordering is stable.+ if (a.file_path, a.start_index) > (b.file_path, b.start_index):+ a, b = b, a+ key = (a.file_path, a.start_index, b.file_path, b.start_index)+ if key in seen:+ continue+ seen.add(key)+ if not _tokens_equal(+ per_file_tokens[a.file_path],+ per_file_tokens[b.file_path],+ a.start_index,+ b.start_index,+ window_tokens,+ ):+ continue+ out.append(+ ClonePair(+ file_a=a.file_path,+ file_b=b.file_path,+ a_start_line=a.start_line,+ a_end_line=a.end_line,+ b_start_line=b.start_line,+ b_end_line=b.end_line,+ token_count=window_tokens,)+ )+++# ---------------------------------------------------------------------------+# Stage 3 — finalize + roll up+# ---------------------------------------------------------------------------+- merged = _merge_adjacent_pairs(raw_pairs)+def _finalize_pairs(+ merged: list[ClonePair],+ min_lines: int,+ meta_map: dict[str, dict[str, Any]],+) -> list[ClonePair]:+ """Drop sub-threshold pairs and attach co-change weight."""final: list[ClonePair] = []for p in merged:if min(p.a_line_count, p.b_line_count) < min_lines:@@ -271,7 +400,14 @@ def detect_clones(co_change_count=score,)final.append(p)+ return final++def _aggregate(+ final: list[ClonePair],+ per_file_nloc: dict[str, int],+) -> tuple[dict[str, list[ClonePair]], dict[str, float]]:+ """Build the per-file pair index and duplication percentages."""pairs_by_file: dict[str, list[ClonePair]] = defaultdict(list)dup_lines: dict[str, int] = defaultdict(int)for p in final:@@ -290,11 +426,7 @@ def detect_clones(# the same physical lines.duplication_pct[path] = round(min(100.0, 100.0 * dup / nloc), 2)- return DuplicationReport(- pairs=final,- duplication_pct=duplication_pct,- pairs_by_file=dict(pairs_by_file),- )+ return dict(pairs_by_file), duplication_pctdef _nloc(source: bytes) -> int:diff --git a/packages/core/src/repowise/core/analysis/health/duplication/limits.py b/packages/core/src/repowise/core/analysis/health/duplication/limits.pynew file mode 100644index 000000000..4cd860782--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/duplication/limits.py@@ -0,0 +1,144 @@+"""Resource guards for duplication detection.++Clone detection is the one health stage whose cost is driven by *input+shape* rather than file count: a single minified/generated bundle can+emit hundreds of thousands of near-identical token windows, and the+all-pairs comparison inside a hash bucket is O(k²) in the bucket size.+Left unbounded, a repo that checks in a build artifact (``*.min.js``,+``storybook-static/``, webpack chunks) can wedge the whole pipeline for+hours — see issue #341.++This module isolates every "how much work is too much" decision so the+detector stays a thin pipeline. Three independent layers, any one of+which is sufficient on its own:++1. **Skip generated/minified files** before tokenizing them at all+ (``looks_minified`` — cheap byte scan, no parse).+2. **Cap per-file tokens** and the **repo-wide window budget** so an+ unusual-but-not-minified file (giant generated lookup table) can't+ dominate memory or bucketing cost.+3. **Cap hash-bucket size** so the O(k²) verifier loop is bounded+ regardless of how the first two layers behaved. A bucket with+ thousands of identical windows is degenerate repetition, not a+ meaningful clone — skipping it improves signal *and* safety.++A soft wall-clock deadline is the final backstop for any pathology none+of the above anticipated.+"""++from __future__ import annotations++from dataclasses import dataclass+++@dataclass(frozen=True)+class DuplicationLimits:+ """Tunable safety bounds for :func:`detect_clones`.++ Defaults are deliberately generous: they never trip on hand-written+ source (human code averages well under 100 bytes/line and rarely+ exceeds a few thousand tokens per file) but cut off generated/minified+ inputs decisively. Override per-call for tests or special repos.+ """++ # --- minified / generated detection (pre-tokenize) ---+ # Average bytes-per-line above which a file is treated as generated.+ # Hand-written code sits around 30-80; bundlers emit 1k+.+ minified_avg_line_bytes: int = 200+ # A single line longer than this flags the file even when the average+ # is dragged down by trailing blank lines (common in partially-minified+ # bundles with one giant IIFE line).+ minified_max_line_bytes: int = 2_000++ # --- per-file and repo-wide window budgets ---+ # Skip files producing more tokens than this. A file with >100k tokens+ # is not a meaningful clone signal; comparing it only adds cost.+ max_tokens_per_file: int = 100_000+ # Stop collecting once the repo-wide window count crosses this. Bounds+ # peak memory and bucketing time on very large repos.+ max_total_windows: int = 5_000_000++ # --- bucket all-pairs guard (the O(k^2) cap) ---+ # Skip hash buckets larger than this. Big buckets are ubiquitous+ # boilerplate (license headers, identical imports), never real clones.+ max_bucket_windows: int = 256++ # --- ultimate backstop ---+ # Soft wall-clock budget for the bucket-comparison phase, in seconds.+ # ``0`` disables the deadline (used by deterministic tests). On expiry+ # the detector returns the clones found so far and flags ``timed_out``.+ time_budget_secs: float = 60.0+++@dataclass+class DuplicationDiagnostics:+ """Counters describing how the guards behaved on one run.++ Surfaced via :attr:`DuplicationReport.diagnostics` so the health+ engine can log *why* a repo produced fewer clone pairs than expected+ (a skipped bundle is invisible otherwise).+ """++ files_considered: int = 0+ files_tokenized: int = 0+ skipped_unreadable: int = 0+ skipped_minified: int = 0+ skipped_token_cap: int = 0+ total_windows: int = 0+ window_budget_hit: bool = False+ degenerate_buckets: int = 0+ timed_out: bool = False++ @property+ def hit_any_limit(self) -> bool:+ """True when any guard actually fired — gates optional logging."""+ return bool(+ self.skipped_minified+ or self.skipped_token_cap+ or self.window_budget_hit+ or self.degenerate_buckets+ or self.timed_out+ )++ def as_log_fields(self) -> dict[str, int | bool]:+ """Flat dict suitable for ``log.debug(**fields)``."""+ return {+ "files_considered": self.files_considered,+ "files_tokenized": self.files_tokenized,+ "skipped_unreadable": self.skipped_unreadable,+ "skipped_minified": self.skipped_minified,+ "skipped_token_cap": self.skipped_token_cap,+ "total_windows": self.total_windows,+ "window_budget_hit": self.window_budget_hit,+ "degenerate_buckets": self.degenerate_buckets,+ "timed_out": self.timed_out,+ }+++def looks_minified(source: bytes, limits: DuplicationLimits) -> bool:+ """Cheap structural test for generated/minified content.++ Scans the raw bytes (no decode, no parse) and flags a file when its+ *average* line length is implausibly long, or when *any single line*+ exceeds the hard cap. Both checks short-circuit, so the cost is at+ most one linear pass and usually far less.+ """+ if not source:+ return False++ newline_count = source.count(b"\n")+ line_count = newline_count + 1+ if len(source) / line_count > limits.minified_avg_line_bytes:+ return True++ # Longest single line — walk newline to newline, bailing the moment we+ # cross the cap so a huge one-line bundle exits almost immediately.+ cap = limits.minified_max_line_bytes+ start = 0+ while True:+ nl = source.find(b"\n", start)+ if nl == -1:+ return (len(source) - start) > cap+ if (nl - start) > cap:+ return True+ start = nl + 1diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.pyindex e3b287ce4..ac3954bb2 100644--- a/packages/core/src/repowise/core/analysis/health/engine.py+++ b/packages/core/src/repowise/core/analysis/health/engine.py@@ -41,6 +41,29 @@ from .scoring import attach_impacts, compute_kpis, score_filelog = structlog.get_logger(__name__)+def _log_duplication_diagnostics(report: DuplicationReport) -> None:+ """Emit a debug line when a duplication guard fired.++ Skipped bundles / capped buckets are otherwise invisible — surfacing+ them explains why a repo produced fewer clone findings than expected+ (and confirms the issue-#341 hang guards are doing their job).+ """+ diag = report.diagnostics+ if not diag:+ return+ if any(+ diag.get(k)+ for k in (+ "skipped_minified",+ "skipped_token_cap",+ "window_budget_hit",+ "degenerate_buckets",+ "timed_out",+ )+ ):+ log.debug("health_duplication_limits", **diag)++def _is_test_file(rel_path: str) -> bool:p = rel_path.lower()return (@@ -268,6 +291,7 @@ class HealthAnalyzer:else:try:dup_report = detect_clones(self.parsed_files, self.git_meta_map)+ _log_duplication_diagnostics(dup_report)except Exception as exc:log.debug("health_duplication_failed", error=str(exc))dup_report = DuplicationReport()@@ -375,6 +399,7 @@ class HealthAnalyzer:dup_report = await asyncio.to_thread(detect_clones, self.parsed_files, self.git_meta_map)+ _log_duplication_diagnostics(dup_report)except Exception as exc:log.debug("health_duplication_failed", error=str(exc))dup_report = DuplicationReport()diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.pyindex cbc7f912a..2bc8ff94d 100644--- a/tests/unit/health/test_duplication.py+++ b/tests/unit/health/test_duplication.py@@ -11,7 +11,9 @@ import pytestfrom repowise.core.analysis.health.duplication import (DEFAULT_MIN_LINES,DEFAULT_WINDOW_TOKENS,+ DuplicationLimits,detect_clones,+ looks_minified,tokenize_file,)from repowise.core.analysis.health.duplication.rabin_karp import (@@ -151,3 +153,112 @@ def test_tokenize_file_returns_empty_for_unsupported_language(language: str):# An obviously invalid language code yields an empty stream rather# than raising.assert tokenize_file("not-a-language", b"x = 1\n") == []+++# ---------------------------------------------------------------------------+# Resource guards (issue #341 — minified bundles wedged the health phase)+# ---------------------------------------------------------------------------+++def test_looks_minified_flags_long_average_line():+ limits = DuplicationLimits()+ # 10 lines, each ~300 bytes → average well over the threshold.+ minified = b"\n".join(b"a" * 300 for _ in range(10))+ assert looks_minified(minified, limits) is True+++def test_looks_minified_flags_single_giant_line():+ limits = DuplicationLimits()+ # Many short lines but one monster line (a bundled IIFE) trips it.+ src = b"const x = 1\n" * 50 + b"y" * 5000 + b"\n"+ assert looks_minified(src, limits) is True+++def test_looks_minified_passes_normal_source():+ limits = DuplicationLimits()+ src = b"\n".join(b" return a + b + c" for _ in range(40))+ assert looks_minified(src, limits) is False+++def test_looks_minified_handles_empty_source():+ assert looks_minified(b"", DuplicationLimits()) is False+++def test_detect_clones_skips_minified_file_but_keeps_real_clones(tmp_path: Path):+ """A checked-in minified bundle must be skipped without preventing+ genuine clone detection between the other files."""+ body = "\n".join(+ [+ "def doit(x, y, z):",+ " if x:",+ " a = x + y",+ " else:",+ " a = x - y",+ " return a + x + y + z",+ "",+ ]+ )+ a = _write(tmp_path, "a.py", body)+ b = _write(tmp_path, "b.py", body.replace("doit", "twin"))+ # One giant single-line "bundle" that previously blew up the detector.+ bundle = _write(tmp_path, "bundle.min.js", "var a=1;" * 50_000 + "\n")+ parsed = [+ _pf("a.py", str(a)),+ _pf("b.py", str(b)),+ _pf("bundle.min.js", str(bundle), language="javascript"),+ ]+ report = detect_clones(parsed, window_tokens=20, min_lines=4)+ # Genuine clone between a.py and b.py still found.+ assert report.pairs+ assert {report.pairs[0].file_a, report.pairs[0].file_b} == {"a.py", "b.py"}+ # The bundle was skipped, not tokenized.+ assert report.diagnostics["skipped_minified"] >= 1+ assert "bundle.min.js" not in report.duplication_pct+++def test_detect_clones_caps_degenerate_bucket(tmp_path: Path):+ """Many identical files produce one oversized hash bucket; it must be+ dropped instead of triggering the O(k^2) all-pairs explosion."""+ snippet = "def f():\n return 1 + 2 + 3 + 4 + 5\n"+ parsed = []+ for i in range(12):+ p = _write(tmp_path, f"f{i}.py", snippet)+ parsed.append(_pf(f"f{i}.py", str(p)))+ limits = DuplicationLimits(max_bucket_windows=4, time_budget_secs=0)+ report = detect_clones(parsed, window_tokens=4, min_lines=1, limits=limits)+ assert report.diagnostics["degenerate_buckets"] >= 1+ # Oversized buckets are skipped wholesale → no pairs emitted.+ assert report.pairs == []+++def test_detect_clones_respects_per_file_token_cap(tmp_path: Path):+ body = "def doit(x, y, z):\n return x + y + z + x + y + z\n"+ a = _write(tmp_path, "a.py", body)+ b = _write(tmp_path, "b.py", body.replace("doit", "twin"))+ parsed = [_pf("a.py", str(a)), _pf("b.py", str(b))]+ # Cap below the token count of these small files → both skipped.+ limits = DuplicationLimits(max_tokens_per_file=5)+ report = detect_clones(parsed, window_tokens=4, min_lines=1, limits=limits)+ assert report.diagnostics["skipped_token_cap"] == 2+ assert report.pairs == []+++def test_detect_clones_window_budget_stops_collection(tmp_path: Path):+ body = "def doit(x, y, z):\n return x + y + z + x + y + z\n"+ parsed = []+ for i in range(5):+ p = _write(tmp_path, f"f{i}.py", body)+ parsed.append(_pf(f"f{i}.py", str(p)))+ limits = DuplicationLimits(max_total_windows=3)+ report = detect_clones(parsed, window_tokens=4, min_lines=1, limits=limits)+ assert report.diagnostics["window_budget_hit"] is True+++def test_detect_clones_reports_diagnostics_on_normal_run(tmp_path: Path):+ a = _write(tmp_path, "a.py", "def f():\n return 1\n")+ parsed = [_pf("a.py", str(a))]+ report = detect_clones(parsed, window_tokens=DEFAULT_WINDOW_TOKENS, min_lines=DEFAULT_MIN_LINES)+ # Diagnostics are always populated, even when nothing tripped.+ assert report.diagnostics["files_considered"] == 1+ assert report.diagnostics["degenerate_buckets"] == 0+ assert report.diagnostics["timed_out"] is False