Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(health): keep clone detection windows within one language (#3414)
Cestercian20673d73a578
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.pyindex f9c33e11c..3ed2c13e1 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py@@ -55,6 +55,10 @@ _MIN_NAME_JACCARD = 0.5# lists are parallel; ``names[i]`` is empty unless token ``i`` is an ``ID``._Stream = tuple[list[str], list[str]]+# Hash buckets are scoped to a grammar tag so Python and TypeScript windows+# with the same normalized token shape never verify together.+_BucketKey = tuple[str, int]+@dataclass(slots=True)class ClonePair:@@ -257,7 +261,7 @@ def detect_clones(# Fall through to the full pipeline; it refreshes the artifact.diag = DuplicationDiagnostics()- per_file_streams, per_file_nloc, all_windows, per_file_hash = _collect_windows(+ per_file_streams, per_file_nloc, windows_by_grammar, per_file_hash = _collect_windows(parsed_list, window_tokens, lim, diag, cache, source_reader)if cache is not None:@@ -272,14 +276,20 @@ def detect_clones(hits=cache.hits,misses=cache.misses,)- if not all_windows:+ if not windows_by_grammar:return DuplicationReport(diagnostics=diag.as_log_fields())- bucket = index_by_hash(all_windows)- # Every WindowHash is now owned by a bucket. Keeping the flat list too- # adds millions of redundant references at the scan's high-water mark.- del all_windows- raw_pairs = _pairs_from_buckets(bucket, per_file_streams, window_tokens, lim, diag)+ raw_pairs: list[ClonePair] = []+ for grammar_windows in windows_by_grammar.values():+ if len(grammar_windows) < 2:+ continue+ bucket = index_by_hash(grammar_windows)+ raw_pairs.extend(+ _pairs_from_buckets(bucket, per_file_streams, window_tokens, lim, diag)+ )+ # Every WindowHash is now owned by a bucket. Drop the per-grammar lists before+ # merge/finalize so freed arenas can be reused.+ del windows_by_grammarif cache is not None and cache_dir is not None:all_paths = {pf.file_info.path for pf in parsed_list}@@ -297,7 +307,7 @@ def detect_clones(# Neither structure participates in merge/finalize/aggregation. Drop# them before those stages allocate their output lists so freed arenas# can be reused inside this phase rather than growing the process again.- del bucket, per_file_streams+ del per_file_streamsfinal = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)@@ -322,7 +332,7 @@ def _collect_windows(diag: DuplicationDiagnostics,cache: Any | None = None,read_source: Any | None = None,-) -> tuple[dict[str, _Stream], dict[str, int], list[WindowHash], dict[str, str]]:+) -> tuple[dict[str, _Stream], dict[str, int], dict[str, list[WindowHash]], dict[str, str]]:"""Tokenize each file once and emit its informative rolling-hash windows.Files are dropped (and counted in *diag*) when they are unreadable,@@ -339,9 +349,11 @@ def _collect_windows("""import hashlib+ from repowise.core.ingestion.parser import grammar_tag_for+per_file_streams: dict[str, _Stream] = {}per_file_nloc: dict[str, int] = {}- all_windows: list[WindowHash] = []+ windows_by_grammar: dict[str, list[WindowHash]] = defaultdict(list)per_file_hash: dict[str, str] = {}for pf in parsed_files:@@ -358,17 +370,16 @@ def _collect_windows(diag.skipped_minified += 1continue+ grammar = grammar_tag_for(language, path)+cached = Nonecontent_hash = ""if cache is not None:- from repowise.core.ingestion.parser import grammar_tag_for-content_hash = hashlib.sha256(source).hexdigest()# The grammar joins the key where it is not the language tag: a# .tsx file and a byte-identical .ts file tokenize differently now,# and this cache and the pair index below both key on this value.# Every other file keys exactly as before and no entry ages out.- grammar = grammar_tag_for(language, path)if grammar != language:content_hash = f"{grammar}:{content_hash}"cached = cache.get(content_hash)@@ -402,19 +413,20 @@ def _collect_windows(diag.skipped_token_cap += 1continue- if len(all_windows) + len(windows) > limits.max_total_windows:+ n_windows_so_far = sum(len(ws) for ws in windows_by_grammar.values())+ if n_windows_so_far + len(windows) > limits.max_total_windows:diag.window_budget_hit = Truebreakper_file_streams[path] = (kinds, names)per_file_nloc[path] = nloc- all_windows.extend(windows)+ windows_by_grammar[grammar].extend(windows)if content_hash:per_file_hash[path] = content_hashdiag.files_tokenized += 1- diag.total_windows = len(all_windows)- return per_file_streams, per_file_nloc, all_windows, per_file_hash+ diag.total_windows = sum(len(ws) for ws in windows_by_grammar.values())+ return per_file_streams, per_file_nloc, dict(windows_by_grammar), per_file_hash# ---------------------------------------------------------------------------@@ -591,9 +603,10 @@ def _detect_clones_incremental(# 1. Collect the changed files live (read, gates, tokenize, windows).# Sorted for deterministic window-budget behaviour.changed_pfs = [current[p] for p in sorted(changed)]- new_streams, new_nloc, new_windows, new_hash = _collect_windows(+ new_streams, new_nloc, new_windows_by_grammar, new_hash = _collect_windows(changed_pfs, window_tokens, lim, diag, cache, read_source)+ new_windows = [w for ws in new_windows_by_grammar.values() for w in ws]if diag.window_budget_hit:return None@@ -612,18 +625,24 @@ def _detect_clones_incremental(if index.total_windows - n_old_aff + len(new_windows) > lim.max_total_windows:return None- # 4. Touched buckets = hashes present in moving windows (old or new).- touched: set[int] = {w.hash_value for w in new_windows}- for _stream, window_tuples in old_aff.values():- touched.update(row[0] for row in window_tuples)+ # 4. Touched buckets = (grammar, hash) pairs present in moving windows.+ path_grammar = {+ p: _grammar_tag_for_path(p, current) for p in set(current) | set(old_files)+ }+ touched: set[_BucketKey] = set()+ for w in new_windows:+ touched.add((path_grammar[w.file_path], w.hash_value))+ for p, (_stream, window_tuples) in old_aff.items():+ g = path_grammar[p]+ touched.update((g, row[0]) for row in window_tuples)# 5. One pass over the unchanged files' cached windows builds the# touched buckets' membership; rows of unchanged files belong to# both the old and the new bucket composition. Stream maps are# split because a modified file verifies with its old stream on# the old side and its new stream on the new side.- old_rows: dict[int, list[WindowHash]] = defaultdict(list)- new_rows: dict[int, list[WindowHash]] = defaultdict(list)+ old_rows: dict[_BucketKey, list[WindowHash]] = defaultdict(list)+ new_rows: dict[_BucketKey, list[WindowHash]] = defaultdict(list)old_streams_map: dict[str, _Stream] = {}new_streams_map: dict[str, _Stream] = dict(new_streams)per_file_nloc: dict[str, int] = dict(new_nloc)@@ -636,8 +655,10 @@ def _detect_clones_incremental(kinds, names, nloc, window_tuples = entryper_file_nloc[p] = nlochit = False+ g = path_grammar[p]for row in window_tuples:- if row[0] in touched:+ key = (g, row[0])+ if key in touched:w = WindowHash(file_path=p,hash_value=row[0],@@ -645,8 +666,8 @@ def _detect_clones_incremental(start_line=row[2],end_line=row[3],)- old_rows[row[0]].append(w)- new_rows[row[0]].append(w)+ old_rows[key].append(w)+ new_rows[key].append(w)hit = Trueif hit:old_streams_map[p] = (kinds, names)@@ -654,9 +675,11 @@ def _detect_clones_incremental(cache.retain(h)for p, (stream, window_tuples) in old_aff.items():+ g = path_grammar[p]for row in window_tuples:- if row[0] in touched:- old_rows[row[0]].append(+ key = (g, row[0])+ if key in touched:+ old_rows[key].append(WindowHash(file_path=p,hash_value=row[0],@@ -667,7 +690,7 @@ def _detect_clones_incremental()old_streams_map[p] = streamfor w in new_windows:- new_rows[w.hash_value].append(w)+ new_rows[(path_grammar[w.file_path], w.hash_value)].append(w)# 6. Per-bucket contributions, old and new.old_contrib = _bucket_contributions(old_rows, old_streams_map, window_tokens, lim, None)@@ -752,8 +775,18 @@ def _pair_key(p: ClonePair) -> _PairKey:return (p.file_a, p.file_b, p.a_start_line, p.a_end_line, p.b_start_line, p.b_end_line)+def _grammar_tag_for_path(path: str, current: dict[str, Any]) -> str:+ """Grammar tag used to tokenize *path* in the current parse set."""+ from repowise.core.analysis.change_health.analyzer import language_of+ from repowise.core.ingestion.parser import grammar_tag_for++ pf = current.get(path)+ language = pf.file_info.language if pf is not None else (language_of(path) or "")+ return grammar_tag_for(language, path)++def _bucket_contributions(- rows_by_hash: dict[int, list[WindowHash]],+ rows_by_hash: dict[_BucketKey, list[WindowHash]],streams_map: dict[str, _Stream],window_tokens: int,lim: DuplicationLimits,diff --git a/packages/core/src/repowise/core/analysis/health/duplication/pair_index.py b/packages/core/src/repowise/core/analysis/health/duplication/pair_index.pyindex 1f0950a5f..f82c72b31 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/pair_index.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/pair_index.py@@ -40,7 +40,7 @@ from .limits import DuplicationLimitslog = structlog.get_logger(__name__)-_INDEX_VERSION = 2+_INDEX_VERSION = 3_INDEX_FILENAME = "duplication_pairs.pkl"# One raw-pair multiset entry as a path-id row:diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.pyindex c38ac3518..74597738b 100644--- a/tests/unit/health/test_duplication.py+++ b/tests/unit/health/test_duplication.py@@ -505,3 +505,34 @@ def test_detect_clones_needs_shared_identifier_names(tmp_path: Path):report = detect_clones(parsed, window_tokens=20, min_lines=4)# Same shape with every name changed is not a copy; a real copy still is.assert {(p.file_a, p.file_b) for p in report.pairs} == {("a.py", "c.py")}+++def _issue_3271_body() -> str:+ return "".join(+ f"result{i} = compute(alpha{i}, beta{i})\nname{i} = lookup(result{i}, gamma)\n"+ f"if name{i} == other:\n done = finish(name{i}, result{i})\n"+ for i in range(6)+ )+++def test_detect_clones_does_not_pair_across_languages(tmp_path: Path):+ """Issue #3271: identical token shapes in Python and TypeScript must not cross-pair."""+ body = _issue_3271_body()+ a = _write(tmp_path, "a.py", body)+ b = _write(tmp_path, "b.ts", body)+ parsed = [+ _pf("a.py", str(a), "python"),+ _pf("b.ts", str(b), "typescript"),+ ]+ report = detect_clones(parsed, {}, window_tokens=DEFAULT_WINDOW_TOKENS, min_lines=3)+ cross = {(p.file_a, p.file_b) for p in report.pairs if p.file_a != p.file_b}+ assert cross == set()+++def test_detect_clones_still_pairs_same_language_duplicates(tmp_path: Path):+ body = _issue_3271_body()+ a = _write(tmp_path, "a.py", body)+ b = _write(tmp_path, "b.py", body)+ parsed = [_pf("a.py", str(a)), _pf("b.py", str(b))]+ report = detect_clones(parsed, {}, window_tokens=DEFAULT_WINDOW_TOKENS, min_lines=3)+ assert any({p.file_a, p.file_b} == {"a.py", "b.py"} for p in report.pairs)diff --git a/tests/unit/health/test_duplication_incremental.py b/tests/unit/health/test_duplication_incremental.pyindex e07af07a3..fd29877a8 100644--- a/tests/unit/health/test_duplication_incremental.py+++ b/tests/unit/health/test_duplication_incremental.py@@ -387,3 +387,49 @@ def test_co_change_weight_applied_live_on_splice(tmp_path: Path):assert report.diagnostics.get("incremental") is Trueab = [p for p in report.pairs if {p.file_a, p.file_b} == {"a.py", "b.py"}]assert ab and all(p.co_change_count == 7 for p in ab)+++def _issue_3271_body() -> str:+ return "".join(+ f"result{i} = compute(alpha{i}, beta{i})\nname{i} = lookup(result{i}, gamma)\n"+ f"if name{i} == other:\n done = finish(name{i}, result{i})\n"+ for i in range(6)+ )+++def _pf_lang(tmp_path: Path, rel: str, language: str) -> SimpleNamespace:+ return SimpleNamespace(+ file_info=SimpleNamespace(path=rel, abs_path=str(tmp_path / rel), language=language),+ symbols=[],+ )+++def test_incremental_does_not_pair_across_languages(tmp_path: Path):+ """Issue #3271: incremental splice must use grammar-scoped hash buckets too."""+ body = _issue_3271_body()+ (tmp_path / "a.py").write_text(body)+ (tmp_path / "b.ts").write_text(body)+ cache_dir = tmp_path / ".repowise"+ seed_parsed = [+ _pf_lang(tmp_path, "a.py", "python"),+ _pf_lang(tmp_path, "b.ts", "typescript"),+ ]+ detect_clones(+ seed_parsed,+ window_tokens=WINDOW,+ min_lines=3,+ limits=LIMITS,+ cache_dir=cache_dir,+ )+ assert (cache_dir / _INDEX_FILENAME).exists()+ # Touch one file so the splice path runs; oracle is a fresh full recompute.+ (tmp_path / "a.py").write_text(body + "\n# touch\n")+ parsed = [+ _pf_lang(tmp_path, "a.py", "python"),+ _pf_lang(tmp_path, "b.ts", "typescript"),+ ]+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert report.diagnostics.get("incremental") is True+ cross = {(p.file_a, p.file_b) for p in report.pairs if p.file_a != p.file_b}+ assert cross == set()+ assert _key(report) == _key(_full(parsed))