Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(health): incremental duplication pair splice for update runs (#460)
Raghav Chamadiyad8d57909d884
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.pyindex 105cecf92..9b64bc460 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py@@ -198,6 +198,7 @@ def detect_clones(min_lines: int = DEFAULT_MIN_LINES,limits: DuplicationLimits | None = None,cache_dir: Path | None = None,+ changed_files: set[str] | None = None,) -> DuplicationReport:"""Run the duplication pipeline over the supplied parsed files.@@ -214,6 +215,12 @@ def detect_clones(Every stage is bounded by :class:`~.limits.DuplicationLimits` so norepo shape (minified bundles, generated tables) can wedge the run —see issue #341.++ When *changed_files* is given (incremental ``repowise update`` runs)+ and a persisted pair index from a previous run validates, the raw+ pair multiset is spliced instead of recomputed: only hash buckets+ touched by a changed/deleted file are re-verified. Any validity miss+ falls back to this full pipeline, which rewrites the artifact."""meta_map = git_meta_map or {}lim = limits or DuplicationLimits()@@ -226,8 +233,32 @@ def detect_clones(cache = DuplicationTokenCache(cache_dir, window_tokens)cache.load()- per_file_kinds, per_file_nloc, all_windows = _collect_windows(- parsed_files, window_tokens, lim, diag, cache+ parsed_list = list(parsed_files)++ if changed_files is not None and cache is not None:+ from .pair_index import load_pair_index++ index = load_pair_index(cache_dir, window_tokens, lim)+ if index is not None:+ report = _detect_clones_incremental(+ parsed_list,+ meta_map,+ set(changed_files),+ window_tokens,+ min_lines,+ lim,+ diag,+ cache,+ index,+ cache_dir,+ )+ if report is not None:+ return report+ # Fall through to the full pipeline; it refreshes the artifact.+ diag = DuplicationDiagnostics()++ per_file_kinds, per_file_nloc, all_windows, per_file_hash = _collect_windows(+ parsed_list, window_tokens, lim, diag, cache)if cache is not None:cache.save()@@ -242,6 +273,19 @@ def detect_clones(bucket = index_by_hash(all_windows)raw_pairs = _pairs_from_buckets(bucket, per_file_kinds, window_tokens, lim, diag)+ if cache is not None and cache_dir is not None:+ all_paths = {pf.file_info.path for pf in parsed_list}+ _persist_pair_index(+ cache_dir,+ window_tokens,+ lim,+ diag,+ per_file_kinds,+ per_file_hash,+ raw_pairs,+ all_paths,+ )+final = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)@@ -264,7 +308,7 @@ def _collect_windows(limits: DuplicationLimits,diag: DuplicationDiagnostics,cache: Any | None = None,-) -> tuple[dict[str, list[str]], dict[str, int], list[WindowHash]]:+) -> tuple[dict[str, list[str]], dict[str, int], list[WindowHash], dict[str, str]]:"""Tokenize each file once and emit its rolling-hash windows.Files are dropped (and counted in *diag*) when they are unreadable,@@ -275,13 +319,16 @@ def _collect_windows(When a :class:`~.token_cache.DuplicationTokenCache` is supplied,unchanged files (by content hash) skip the tokenize + rolling-hashwork and replay their cached kind sequence and window tuples; every- gate above still re-evaluates live against the cached lengths.+ gate above still re-evaluates live against the cached lengths. The+ returned hash map (path -> content hash, gate survivors only) feeds+ the persisted pair index; it is empty when no cache is supplied."""import hashlibper_file_kinds: dict[str, list[str]] = {}per_file_nloc: dict[str, int] = {}all_windows: list[WindowHash] = []+ per_file_hash: dict[str, str] = {}for pf in parsed_files:check_cancelled()@@ -344,10 +391,12 @@ def _collect_windows(per_file_kinds[path] = kindsper_file_nloc[path] = nlocall_windows.extend(windows)+ if content_hash:+ per_file_hash[path] = content_hashdiag.files_tokenized += 1diag.total_windows = len(all_windows)- return per_file_kinds, per_file_nloc, all_windows+ return per_file_kinds, per_file_nloc, all_windows, per_file_hash# ---------------------------------------------------------------------------@@ -434,6 +483,309 @@ def _verify_bucket()+# ---------------------------------------------------------------------------+# Incremental splice (update path)+# ---------------------------------------------------------------------------++# Raw pairs identified by line geometry; multiplicity preserved via Counter+# because the merge stage accumulates token_count per merged region.+_PairKey = tuple[str, str, int, int, int, int]++# An incremental run is only worth it when few files moved — past this+# fraction the full pipeline's flat cost wins (and stays simpler). The+# floor keeps typical small commits incremental even on small repos,+# where a handful of files is a large fraction but splicing is correct+# and still cheap.+_MAX_CHANGED_FRACTION = 0.2+_CHANGED_COUNT_FLOOR = 16+++def _detect_clones_incremental(+ parsed_files: list[Any],+ meta_map: dict[str, dict[str, Any]],+ changed_files: set[str],+ window_tokens: int,+ min_lines: int,+ lim: DuplicationLimits,+ diag: DuplicationDiagnostics,+ cache: Any,+ index: Any,+ cache_dir: Path,+) -> DuplicationReport | None:+ """Splice the persisted raw-pair multiset instead of recomputing it.++ Equivalence argument: raw pairs are a pure function of the gate-+ surviving window set, and each (window, window) pair lives in exactly+ one hash bucket. For every bucket whose membership changes (a hash+ seen in a changed/deleted file's old windows or a changed file's new+ windows), subtract the bucket's old contribution and add its new one+ — both recomputed deterministically, including the degenerate-bucket+ cap on the bucket's *full* membership, so cap transitions in either+ direction are handled uniformly. Untouched buckets keep their pairs+ verbatim. Finalize (merge, min-lines, co-change weighting) always+ runs live against the current ``git_meta_map``.++ Returns ``None`` whenever the persisted state cannot be spliced+ safely (truncated/timed-out runs, missing cache entries, too many+ changes, accounting mismatch); the caller falls back to the full+ pipeline.+ """+ from collections import Counter++ if not index.spliceable:+ return None++ current = {pf.file_info.path: pf for pf in parsed_files}+ old_files: dict[str, str] = index.files+ # Files the previous run considered but gated out are not "new";+ # unchanged ones stay gated (same bytes, same gates), changed ones+ # arrive via changed_files and re-evaluate the gates live.+ new_paths = set(current) - set(old_files) - index.nonsurvivors+ deleted = set(old_files) - set(current)+ changed = (changed_files & set(current)) | new_paths+ moved = len(changed) + len(deleted)+ if moved > max(_CHANGED_COUNT_FLOOR, _MAX_CHANGED_FRACTION * len(current)):+ return None+ # Paths whose OLD windows leave the state (modified or deleted).+ affected_old = (changed | deleted) & set(old_files)+ unchanged = set(old_files) - affected_old - deleted++ # 1. Collect the changed files live (read, gates, tokenize, windows).+ # Sorted for deterministic window-budget behaviour.+ changed_pfs = [current[p] for p in sorted(changed)]+ new_kinds, new_nloc, new_windows, new_hash = _collect_windows(+ changed_pfs, window_tokens, lim, diag, cache+ )+ if diag.window_budget_hit:+ return None++ # 2. Old windows + kinds of affected files, from the token cache.+ old_aff: dict[str, tuple[list[str], list[tuple[int, int, int, int]]]] = {}+ for p in affected_old:+ entry = cache.entry(old_files[p])+ if entry is None:+ return None+ kinds, _nloc, window_tuples = entry+ old_aff[p] = (kinds, window_tuples)++ # 3. Window-budget equivalence: splicing is only valid when the full+ # pipeline would not truncate either state.+ n_old_aff = sum(len(wt) for _k, wt in old_aff.values())+ if index.total_windows - n_old_aff + len(new_windows) > lim.max_total_windows:+ return None++ # 4. Touched buckets = hashes present in moving windows (old or new).+ touched: set[int] = {w.hash_value for w in new_windows}+ for _kinds, window_tuples in old_aff.values():+ touched.update(row[0] for row in window_tuples)++ # 5. One pass over the unchanged files' cached windows builds the+ # touched buckets' membership; rows of unchanged files belong to+ # both the old and the new bucket composition. Kinds maps are+ # split because a modified file verifies with its old kinds on+ # the old side and its new kinds on the new side.+ old_rows: dict[int, list[WindowHash]] = defaultdict(list)+ new_rows: dict[int, list[WindowHash]] = defaultdict(list)+ old_kinds_map: dict[str, list[str]] = {}+ new_kinds_map: dict[str, list[str]] = dict(new_kinds)+ per_file_nloc: dict[str, int] = dict(new_nloc)+ for p in unchanged:+ check_cancelled()+ h = old_files[p]+ entry = cache.entry(h)+ if entry is None:+ return None+ kinds, nloc, window_tuples = entry+ per_file_nloc[p] = nloc+ hit = False+ for row in window_tuples:+ if row[0] in touched:+ w = WindowHash(+ file_path=p,+ hash_value=row[0],+ start_index=row[1],+ start_line=row[2],+ end_line=row[3],+ )+ old_rows[row[0]].append(w)+ new_rows[row[0]].append(w)+ hit = True+ if hit:+ old_kinds_map[p] = kinds+ new_kinds_map[p] = kinds+ cache.retain(h)++ for p, (kinds, window_tuples) in old_aff.items():+ for row in window_tuples:+ if row[0] in touched:+ old_rows[row[0]].append(+ WindowHash(+ file_path=p,+ hash_value=row[0],+ start_index=row[1],+ start_line=row[2],+ end_line=row[3],+ )+ )+ old_kinds_map[p] = kinds+ for w in new_windows:+ new_rows[w.hash_value].append(w)++ # 6. Per-bucket contributions, old and new.+ old_contrib = _bucket_contributions(old_rows, old_kinds_map, window_tokens, lim, None)+ new_contrib = _bucket_contributions(new_rows, new_kinds_map, window_tokens, lim, diag)++ # 7. Splice the multiset. A negative count means the persisted state+ # disagrees with the recomputed old contribution — fall back.+ paths_table: list[str] = index.paths+ pair_counter: Counter[_PairKey] = Counter()+ for pid_a, pid_b, a_sl, a_el, b_sl, b_el, count in index.pairs:+ pair_counter[(paths_table[pid_a], paths_table[pid_b], a_sl, a_el, b_sl, b_el)] += count+ pair_counter.subtract(Counter(_pair_key(p) for p in old_contrib))+ if -pair_counter: # truthy when any count went negative+ return None+ pair_counter.update(_pair_key(p) for p in new_contrib)++ # One ClonePair per distinct key carrying the multiplicity in its+ # token_count: identical raw pairs are guaranteed to merge with each+ # other (same files, same starts), and merging sums token_count, so+ # this is exactly what expanding the multiset would produce.+ raw_pairs = [+ ClonePair(+ file_a=key[0],+ file_b=key[1],+ a_start_line=key[2],+ a_end_line=key[3],+ b_start_line=key[4],+ b_end_line=key[5],+ token_count=window_tokens * count,+ )+ for key, count in pair_counter.items()+ if count > 0+ ]++ # 8. Persist the spliced state and the retained token cache before+ # finalizing, mirroring the full path's ordering.+ from .pair_index import DuplicationPairIndex, limits_fingerprint, save_pair_index++ new_files = {p: old_files[p] for p in unchanged}+ new_files.update(new_hash)+ new_paths_table = sorted(new_files)+ pid = {p: i for i, p in enumerate(new_paths_table)}+ save_pair_index(+ cache_dir,+ DuplicationPairIndex(+ window_tokens=window_tokens,+ limits_key=limits_fingerprint(lim),+ files=new_files,+ nonsurvivors=set(current) - set(new_files),+ paths=new_paths_table,+ pairs=[+ (pid[key[0]], pid[key[1]], key[2], key[3], key[4], key[5], count)+ for key, count in pair_counter.items()+ if count > 0+ ],+ total_windows=index.total_windows - n_old_aff + len(new_windows),+ ),+ )+ cache.save()+ log.debug(+ "duplication_incremental_splice",+ changed=len(changed),+ deleted=len(deleted),+ touched_buckets=len(touched),+ pairs=len(raw_pairs),+ )++ final = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)+ pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)++ diagnostics = diag.as_log_fields()+ diagnostics["incremental"] = True+ return DuplicationReport(+ pairs=final,+ duplication_pct=duplication_pct,+ pairs_by_file=pairs_by_file,+ diagnostics=diagnostics,+ )+++def _pair_key(p: ClonePair) -> _PairKey:+ return (p.file_a, p.file_b, p.a_start_line, p.a_end_line, p.b_start_line, p.b_end_line)+++def _bucket_contributions(+ rows_by_hash: dict[int, list[WindowHash]],+ kinds_map: dict[str, list[str]],+ window_tokens: int,+ lim: DuplicationLimits,+ diag: DuplicationDiagnostics | None,+) -> list[ClonePair]:+ """Verify the touched buckets exactly as the full pipeline would.++ The degenerate-bucket cap applies to each bucket's full membership+ (rows here cover it: every window with a touched hash was gathered),+ so a bucket crossing the cap in either direction contributes pairs on+ exactly one side of the splice. The shared ``seen`` set mirrors the+ full pipeline's; (file, start_index) pairs are unique to one bucket,+ so per-run scoping is equivalent.+ """+ out: list[ClonePair] = []+ seen: set[tuple[str, int, str, int]] = set()+ for rows in rows_by_hash.values():+ if len(rows) < 2:+ continue+ if len(rows) > lim.max_bucket_windows:+ if diag is not None:+ diag.degenerate_buckets += 1+ continue+ check_cancelled()+ _verify_bucket(rows, kinds_map, window_tokens, seen, out)+ return out+++def _persist_pair_index(+ cache_dir: Path,+ window_tokens: int,+ lim: DuplicationLimits,+ diag: DuplicationDiagnostics,+ per_file_kinds: dict[str, list[str]],+ per_file_hash: dict[str, str],+ raw_pairs: list[ClonePair],+ all_paths: set[str],+) -> None:+ """Persist the full run's raw pairs for the next incremental splice."""+ from collections import Counter++ from .pair_index import DuplicationPairIndex, limits_fingerprint, save_pair_index++ files = {p: per_file_hash[p] for p in per_file_kinds if p in per_file_hash}+ if len(files) != len(per_file_kinds):+ # A gate survivor without a content hash should be impossible+ # when the cache is active; don't persist a state we can't trust.+ return+ paths_table = sorted(files)+ pid = {p: i for i, p in enumerate(paths_table)}+ pair_counter = Counter(_pair_key(p) for p in raw_pairs)+ save_pair_index(+ cache_dir,+ DuplicationPairIndex(+ window_tokens=window_tokens,+ limits_key=limits_fingerprint(lim),+ files=files,+ nonsurvivors=all_paths - set(files),+ paths=paths_table,+ pairs=[+ (pid[key[0]], pid[key[1]], key[2], key[3], key[4], key[5], count)+ for key, count in pair_counter.items()+ ],+ total_windows=diag.total_windows,+ window_budget_hit=diag.window_budget_hit,+ timed_out=diag.timed_out,+ ),+ )++# ---------------------------------------------------------------------------# Stage 3 — finalize + roll up# ---------------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/analysis/health/duplication/pair_index.py b/packages/core/src/repowise/core/analysis/health/duplication/pair_index.pynew file mode 100644index 000000000..d61b1f165--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/duplication/pair_index.py@@ -0,0 +1,151 @@+"""Persisted clone-pair index for incremental duplication runs.++A full duplication pass re-derives the repo-wide raw pair set from+scratch even when one file changed, because ``duplication_pct`` is+repo-wide. The raw pairs are a pure function of (file bytes, window+size, limits): pairs between unchanged files cannot change. Persisting+them lets an incremental run splice the pair multiset instead -- drop+the contributions of buckets a changed file touches, re-verify only+those buckets, and keep everything else verbatim.++The artifact stores:++* ``files`` -- path -> content hash for every file that contributed+ windows (the detector's gate survivors). Used to detect deletions,+ to fetch unchanged files' cached token streams, and to keep their+ token-cache entries alive across incremental runs.+* ``pairs`` -- the raw (pre-merge) pair multiset as compact path-id+ rows. Multiset, not set: ``_merge_adjacent_pairs`` accumulates+ ``token_count`` per merged pair, so multiplicity matters.+* ``total_windows`` plus the guard flags, so the incremental path can+ re-evaluate the repo-wide window budget and refuse to splice a+ truncated state.++Validity is keyed on (version, window size, limits fingerprint); any+mismatch -- or any load/save error -- degrades to a full re-detect,+which rewrites the artifact. Best-effort by design, like the token+cache next to it.+"""++from __future__ import annotations++import contextlib+import os+import pickle+import tempfile+from dataclasses import dataclass, field+from pathlib import Path++import structlog++from .limits import DuplicationLimits++log = structlog.get_logger(__name__)++_INDEX_VERSION = 1+_INDEX_FILENAME = "duplication_pairs.pkl"++# One raw-pair multiset entry as a path-id row:+# (pid_a, pid_b, a_start, a_end, b_start, b_end, count).+# Counts compress the multiset: identical raw pairs (same files and line+# geometry) always merge together downstream, so multiplicity is all the+# merge stage needs from them.+PairRow = tuple[int, int, int, int, int, int, int]+++def limits_fingerprint(limits: DuplicationLimits) -> tuple:+ """The limit fields that change which pairs a full run emits."""+ return (+ limits.minified_avg_line_bytes,+ limits.minified_max_line_bytes,+ limits.max_tokens_per_file,+ limits.max_total_windows,+ limits.max_bucket_windows,+ )+++@dataclass+class DuplicationPairIndex:+ """In-memory form of one persisted pair-index artifact."""++ window_tokens: int+ limits_key: tuple+ files: dict[str, str] = field(default_factory=dict) # path -> content hash+ # Paths the detector considered but that contributed no windows+ # (minified, too small, over the token cap, unreadable). Tracked so+ # an incremental run doesn't mistake them for new files every time.+ nonsurvivors: set[str] = field(default_factory=set)+ paths: list[str] = field(default_factory=list)+ pairs: list[PairRow] = field(default_factory=list)+ total_windows: int = 0+ window_budget_hit: bool = False+ timed_out: bool = False++ @property+ def spliceable(self) -> bool:+ """A truncated or deadline-cut state cannot be spliced safely."""+ return not (self.window_budget_hit or self.timed_out)+++def load_pair_index(+ cache_dir: Path,+ window_tokens: int,+ limits: DuplicationLimits,+) -> DuplicationPairIndex | None:+ """Load and validate the artifact; ``None`` on any mismatch/error."""+ path = Path(cache_dir) / _INDEX_FILENAME+ try:+ with path.open("rb") as fh:+ payload = pickle.load(fh)+ if (+ payload.get("version") != _INDEX_VERSION+ or payload.get("window_tokens") != window_tokens+ or tuple(payload.get("limits_key", ())) != limits_fingerprint(limits)+ ):+ return None+ return DuplicationPairIndex(+ window_tokens=window_tokens,+ limits_key=limits_fingerprint(limits),+ files=payload["files"],+ nonsurvivors=payload["nonsurvivors"],+ paths=payload["paths"],+ pairs=payload["pairs"],+ total_windows=payload["total_windows"],+ window_budget_hit=payload["window_budget_hit"],+ timed_out=payload["timed_out"],+ )+ except FileNotFoundError:+ return None+ except Exception as exc: # corrupt / unreadable -> full re-detect+ log.debug("duplication_pair_index_load_failed", error=str(exc))+ return None+++def save_pair_index(cache_dir: Path, index: DuplicationPairIndex) -> None:+ """Atomically persist *index*; failures degrade to a future full run."""+ path = Path(cache_dir) / _INDEX_FILENAME+ try:+ path.parent.mkdir(parents=True, exist_ok=True)+ payload = {+ "version": _INDEX_VERSION,+ "window_tokens": index.window_tokens,+ "limits_key": index.limits_key,+ "files": index.files,+ "nonsurvivors": index.nonsurvivors,+ "paths": index.paths,+ "pairs": index.pairs,+ "total_windows": index.total_windows,+ "window_budget_hit": index.window_budget_hit,+ "timed_out": index.timed_out,+ }+ fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=_INDEX_FILENAME, suffix=".tmp")+ try:+ with os.fdopen(fd, "wb") as fh:+ pickle.dump(payload, fh, protocol=pickle.HIGHEST_PROTOCOL)+ os.replace(tmp_name, path)+ except BaseException:+ with contextlib.suppress(OSError):+ os.unlink(tmp_name)+ raise+ except Exception as exc:+ log.debug("duplication_pair_index_save_failed", error=str(exc))diff --git a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.pyindex ec2a92264..fc6f4ce85 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py@@ -101,6 +101,30 @@ class DuplicationTokenCache:self._fresh[content_hash] = entryreturn entry+ def entry(+ self, content_hash: str+ ) -> tuple[list[str], int, list[tuple[int, int, int, int]]] | None:+ """Read an entry without touching hit/miss stats or freshness.++ The incremental pair-splice path reads unchanged files' cached+ tuples through this so its lookups don't skew the cache-hit+ telemetry that ``get`` feeds.+ """+ return self._entries.get(content_hash)++ def retain(self, content_hash: str) -> bool:+ """Mark an entry as live so ``save`` keeps it.++ ``save`` rewrites the cache with only the entries touched this+ run; an incremental run that never reads unchanged files would+ otherwise evict them. Returns False when the hash is unknown.+ """+ entry = self._entries.get(content_hash)+ if entry is None:+ return False+ self._fresh[content_hash] = entry+ return True+def put(self,content_hash: str,diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.pyindex 91fb8d5cb..0b4299ae1 100644--- a/packages/core/src/repowise/core/analysis/health/engine.py+++ b/packages/core/src/repowise/core/analysis/health/engine.py@@ -319,8 +319,9 @@ class HealthAnalyzer:# Duplication runs once, up-front, so each file biomarker can see# its clone list. Cheap when the repo is small; when disabled# explicitly we skip the work entirely. Even for incremental- # runs we keep the full-repo scan: a changed file's clone partners- # may be unchanged files we still need to compare against.+ # runs the result stays repo-wide: a changed file's clone partners+ # may be unchanged files — passing changed_files lets the detector+ # splice its persisted pair index instead of recomputing it all.if "dry_violation" in disabled:dup_report = DuplicationReport()else:@@ -329,6 +330,7 @@ class HealthAnalyzer:self.parsed_files,self.git_meta_map,cache_dir=self.duplication_cache_dir,+ changed_files=changed_set,)_log_duplication_diagnostics(dup_report)except Exception as exc:@@ -449,6 +451,7 @@ class HealthAnalyzer:self.parsed_files,self.git_meta_map,cache_dir=self.duplication_cache_dir,+ changed_files=changed_set,))diff --git a/tests/unit/health/test_duplication_incremental.py b/tests/unit/health/test_duplication_incremental.pynew file mode 100644index 000000000..1cb8cfc90--- /dev/null+++ b/tests/unit/health/test_duplication_incremental.py@@ -0,0 +1,388 @@+"""Incremental duplication splice must reproduce the full recompute exactly.++Oracle pattern: every scenario seeds a repo, runs a full cached pass+(persisting the pair index), mutates the tree, then asserts the+incremental run (``changed_files=...``) equals a fresh full recompute+of the mutated tree — pairs as a multiset including token_count (the+merge stage accumulates it, so multiplicity drift would surface there),+plus duplication_pct and pairs_by_file.+"""++from __future__ import annotations++from collections import Counter+from pathlib import Path+from types import SimpleNamespace++from repowise.core.analysis.health.duplication import detect_clones+from repowise.core.analysis.health.duplication.limits import DuplicationLimits+from repowise.core.analysis.health.duplication.pair_index import (+ _INDEX_FILENAME,+ load_pair_index,+)++WINDOW = 20+MIN_LINES = 4+# Deterministic limits: no wall-clock deadline in tests.+LIMITS = DuplicationLimits(time_budget_secs=0)+++def _pf(tmp_path: Path, rel: str) -> SimpleNamespace:+ return SimpleNamespace(+ file_info=SimpleNamespace(path=rel, abs_path=str(tmp_path / rel), language="python"),+ symbols=[],+ )+++_BODY = "\n".join(+ [+ "def doit(x, y, z):",+ " if x:",+ " a = x + y",+ " else:",+ " a = x - y",+ " if z:",+ " b = a * 2",+ " else:",+ " b = a - 1",+ " return a + b + x + y + z",+ "",+ ]+)++_OTHER = "\n".join(+ [+ "def other(p, q):",+ " total = 0",+ " for i in range(p):",+ " if i % 2:",+ " total += i * q",+ " else:",+ " total -= i + q",+ " return total",+ "",+ ]+)+++def _write(tmp_path: Path, files: dict[str, str]) -> list[SimpleNamespace]:+ for rel, body in files.items():+ (tmp_path / rel).write_text(body)+ return [_pf(tmp_path, rel) for rel in sorted(files)]+++def _parsed(tmp_path: Path) -> list[SimpleNamespace]:+ return [_pf(tmp_path, p.name) for p in sorted(tmp_path.glob("*.py"))]+++def _key(report):+ return (+ sorted(+ Counter(+ (+ p.file_a,+ p.file_b,+ p.a_start_line,+ p.a_end_line,+ p.b_start_line,+ p.b_end_line,+ p.token_count,+ p.co_change_count,+ )+ for p in report.pairs+ ).items()+ ),+ report.duplication_pct,+ {f: len(ps) for f, ps in report.pairs_by_file.items()},+ )+++def _full(parsed, limits=LIMITS):+ """Fresh full recompute, no cache — the oracle."""+ return detect_clones(parsed, window_tokens=WINDOW, min_lines=MIN_LINES, limits=limits)+++def _incremental(parsed, cache_dir, changed, limits=LIMITS):+ return detect_clones(+ parsed,+ window_tokens=WINDOW,+ min_lines=MIN_LINES,+ limits=limits,+ cache_dir=cache_dir,+ changed_files=set(changed),+ )+++def _seed(tmp_path: Path, files: dict[str, str], limits=LIMITS):+ """Initial full cached run; persists token cache + pair index."""+ parsed = _write(tmp_path, files)+ cache_dir = tmp_path / ".repowise"+ detect_clones(+ parsed, window_tokens=WINDOW, min_lines=MIN_LINES, limits=limits, cache_dir=cache_dir+ )+ assert (cache_dir / _INDEX_FILENAME).exists()+ return cache_dir+++def test_modify_clone_member(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY, "c.py": _OTHER})+ (tmp_path / "a.py").write_text(_OTHER.replace("other", "mutated"))++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert report.diagnostics.get("incremental") is True+ assert _key(report) == _key(_full(parsed))+++def test_add_new_clone_file(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "c.py": _OTHER})+ (tmp_path / "d.py").write_text(_BODY.replace("doit", "added"))++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"d.py"})+ assert report.diagnostics.get("incremental") is True+ assert any({p.file_a, p.file_b} == {"a.py", "d.py"} for p in report.pairs)+ assert _key(report) == _key(_full(parsed))+++def test_delete_clone_file(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY, "c.py": _OTHER})+ (tmp_path / "b.py").unlink()++ parsed = _parsed(tmp_path)+ # Deletions arrive via the parsed set shrinking, not changed_files.+ report = _incremental(parsed, cache_dir, set())+ assert report.diagnostics.get("incremental") is True+ assert not report.pairs+ assert _key(report) == _key(_full(parsed))+++def test_rename_clone_file(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY})+ (tmp_path / "b.py").unlink()+ (tmp_path / "renamed.py").write_text(_BODY)++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"renamed.py"})+ assert report.diagnostics.get("incremental") is True+ assert any({p.file_a, p.file_b} == {"a.py", "renamed.py"} for p in report.pairs)+ assert _key(report) == _key(_full(parsed))+++def test_noop_change_listed_as_changed(tmp_path: Path):+ files = {"a.py": _BODY, "b.py": _BODY, "c.py": _OTHER}+ cache_dir = _seed(tmp_path, files)+ (tmp_path / "a.py").write_text(_BODY) # rewrite identical content++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert report.diagnostics.get("incremental") is True+ assert _key(report) == _key(_full(parsed))+++def test_intra_file_duplication_in_changed_file(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "c.py": _OTHER})+ intra = _BODY + "\n" + _BODY.replace("doit", "again")+ (tmp_path / "a.py").write_text(intra)++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert report.diagnostics.get("incremental") is True+ assert any(p.is_intra_file for p in report.pairs)+ assert _key(report) == _key(_full(parsed))+++def test_degenerate_bucket_shrinks_below_cap(tmp_path: Path):+ """Removing a member can revive a previously capped bucket's pairs."""+ lim = DuplicationLimits(time_budget_secs=0, max_bucket_windows=3)+ files = {f"{n}.py": _BODY for n in "abcd"} # buckets of 4 > cap of 3+ cache_dir = _seed(tmp_path, files, limits=lim)+ baseline = load_pair_index(cache_dir, WINDOW, lim)+ assert baseline is not None and not baseline.pairs # all degenerate++ (tmp_path / "d.py").unlink() # buckets drop to 3 == cap -> pairs emerge+ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, set(), limits=lim)+ assert report.diagnostics.get("incremental") is True+ assert report.pairs+ assert _key(report) == _key(_full(parsed, limits=lim))+++def test_degenerate_bucket_grows_past_cap(tmp_path: Path):+ """Adding a member can cap a bucket, removing unchanged-pair output."""+ lim = DuplicationLimits(time_budget_secs=0, max_bucket_windows=3)+ files = {f"{n}.py": _BODY for n in "abc"} # buckets of 3 == cap -> pairs+ cache_dir = _seed(tmp_path, files, limits=lim)+ baseline = load_pair_index(cache_dir, WINDOW, lim)+ assert baseline is not None and baseline.pairs++ (tmp_path / "d.py").write_text(_BODY) # buckets of 4 > cap+ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"d.py"}, limits=lim)+ assert report.diagnostics.get("incremental") is True+ assert not report.pairs+ assert _key(report) == _key(_full(parsed, limits=lim))+++def test_chained_incremental_updates(tmp_path: Path):+ """The artifact rewritten by one splice must support the next."""+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY, "c.py": _OTHER})++ (tmp_path / "d.py").write_text(_BODY.replace("doit", "fourth"))+ parsed = _parsed(tmp_path)+ first = _incremental(parsed, cache_dir, {"d.py"})+ assert first.diagnostics.get("incremental") is True+ assert _key(first) == _key(_full(parsed))++ (tmp_path / "b.py").write_text(_OTHER.replace("other", "swapped"))+ parsed = _parsed(tmp_path)+ second = _incremental(parsed, cache_dir, {"b.py"})+ assert second.diagnostics.get("incremental") is True+ assert _key(second) == _key(_full(parsed))+++def test_gated_files_are_not_treated_as_new(tmp_path: Path, monkeypatch):+ """Files the seed run gated out (too small to window) must not count+ toward the changed-files guard on every later run."""+ from repowise.core.analysis.health.duplication import detector++ monkeypatch.setattr(detector, "_CHANGED_COUNT_FLOOR", 1)+ files = {"a.py": _BODY, "b.py": _BODY}+ files.update({f"tiny{i}.py": f"x = {i}\n" for i in range(6)})+ cache_dir = _seed(tmp_path, files)+ idx = load_pair_index(cache_dir, WINDOW, LIMITS)+ assert idx is not None and len(idx.nonsurvivors) == 6++ (tmp_path / "a.py").write_text(_BODY.replace("doit", "edited"))+ parsed = _parsed(tmp_path)+ # Only 1 real change; with the tiny files miscounted as new this+ # would exceed the floor of 1 and fall back.+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert report.diagnostics.get("incremental") is True+ assert _key(report) == _key(_full(parsed))+++def test_gated_file_growing_into_survivor(tmp_path: Path):+ """A previously gated file that changes into real content joins in."""+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "tiny.py": "x = 1\n"})+ (tmp_path / "tiny.py").write_text(_BODY.replace("doit", "grown"))++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"tiny.py"})+ assert report.diagnostics.get("incremental") is True+ assert any({p.file_a, p.file_b} == {"a.py", "tiny.py"} for p in report.pairs)+ assert _key(report) == _key(_full(parsed))+++def test_too_many_changes_falls_back_to_full(tmp_path: Path, monkeypatch):+ from repowise.core.analysis.health.duplication import detector++ monkeypatch.setattr(detector, "_CHANGED_COUNT_FLOOR", 0)+ files = {f"f{i}.py": _BODY.replace("doit", f"fn{i}") for i in range(5)}+ cache_dir = _seed(tmp_path, files)+ for i in range(3): # 3 of 5 changed > 20% threshold+ (tmp_path / f"f{i}.py").write_text(_OTHER.replace("other", f"fn{i}"))++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"f0.py", "f1.py", "f2.py"})+ assert "incremental" not in report.diagnostics+ assert _key(report) == _key(_full(parsed))+++def test_limits_change_invalidates_artifact(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY})+ other_limits = DuplicationLimits(time_budget_secs=0, max_bucket_windows=128)+ assert load_pair_index(cache_dir, WINDOW, other_limits) is None++ (tmp_path / "a.py").write_text(_BODY.replace("doit", "edited"))+ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"}, limits=other_limits)+ assert "incremental" not in report.diagnostics+ assert _key(report) == _key(_full(parsed, limits=other_limits))+++def test_missing_artifact_falls_back(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY})+ (cache_dir / _INDEX_FILENAME).unlink()++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert "incremental" not in report.diagnostics+ assert _key(report) == _key(_full(parsed))+ # The fallback full run re-persists the artifact for next time.+ assert (cache_dir / _INDEX_FILENAME).exists()+++def test_corrupt_artifact_falls_back(tmp_path: Path):+ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY})+ (cache_dir / _INDEX_FILENAME).write_bytes(b"not a pickle")++ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert "incremental" not in report.diagnostics+ assert _key(report) == _key(_full(parsed))+++def test_truncated_state_refuses_to_splice(tmp_path: Path):+ """A window-budget-hit run persists a flag that blocks splicing."""+ # Budget admits the first file's windows but trips on the second, so+ # the run persists a truncated state (an empty-window run would+ # return before persisting anything).+ lim = DuplicationLimits(time_budget_secs=0, max_total_windows=35)+ files = {"a.py": _BODY, "b.py": _BODY, "c.py": _BODY}+ cache_dir = _seed(tmp_path, files, limits=lim)+ idx = load_pair_index(cache_dir, WINDOW, lim)+ assert idx is not None and idx.window_budget_hit++ (tmp_path / "a.py").write_text(_BODY.replace("doit", "edited"))+ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"}, limits=lim)+ assert "incremental" not in report.diagnostics+ assert _key(report) == _key(_full(parsed, limits=lim))+++def test_unchanged_token_cache_entries_survive_splice(tmp_path: Path):+ """The splice path must retain unchanged files' cache entries."""+ import hashlib++ from repowise.core.analysis.health.duplication.token_cache import (+ DuplicationTokenCache,+ )++ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY, "c.py": _OTHER})+ (tmp_path / "a.py").write_text(_BODY.replace("doit", "edited"))+ parsed = _parsed(tmp_path)+ report = _incremental(parsed, cache_dir, {"a.py"})+ assert report.diagnostics.get("incremental") is True++ cache = DuplicationTokenCache(cache_dir, WINDOW)+ cache.load()+ for rel in ("a.py", "b.py", "c.py"):+ digest = hashlib.sha256((tmp_path / rel).read_bytes()).hexdigest()+ assert cache.get(digest) is not None, rel+++def test_co_change_weight_applied_live_on_splice(tmp_path: Path):+ """Finalize must consume the CURRENT git_meta_map, not persisted state."""+ import json++ cache_dir = _seed(tmp_path, {"a.py": _BODY, "b.py": _BODY, "c.py": _OTHER})+ (tmp_path / "c.py").write_text(_OTHER.replace("other", "edited"))+ parsed = _parsed(tmp_path)+ meta = {+ "a.py": {+ "co_change_partners_json": json.dumps([{"file_path": "b.py", "co_change_count": 7}])+ }+ }+ report = detect_clones(+ parsed,+ meta,+ window_tokens=WINDOW,+ min_lines=MIN_LINES,+ limits=LIMITS,+ cache_dir=cache_dir,+ changed_files={"c.py"},+ )+ assert report.diagnostics.get("incremental") is True+ ab = [p for p in report.pairs if {p.file_a, p.file_b} == {"a.py", "b.py"}]+ assert ab and all(p.co_change_count == 7 for p in ab)