Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(change-risk): map symbols a change moves across files (#4109)
Raghav Chamadiya23dcc35007bc
diff --git a/packages/core/src/repowise/core/analysis/change_health/__init__.py b/packages/core/src/repowise/core/analysis/change_health/__init__.pyindex ec9157eea..8809f569e 100644--- a/packages/core/src/repowise/core/analysis/change_health/__init__.py+++ b/packages/core/src/repowise/core/analysis/change_health/__init__.py@@ -40,6 +40,7 @@ if TYPE_CHECKING:RevisionId,ScopeCounts,)+ from .moves import MoveMap, SymbolBody, build_move_map, symbol_bodiesfrom .perf_delta import PerfOpportunityView, opportunities_forfrom .service import ChangeHealthDeltaService, DeltaRequest@@ -64,6 +65,10 @@ _LAZY_EXPORTS: dict[str, str] = {"FindingKey": "models","RevisionId": "models","ScopeCounts": "models",+ "MoveMap": "moves",+ "SymbolBody": "moves",+ "build_move_map": "moves",+ "symbol_bodies": "moves","PerfOpportunityView": "perf_delta","opportunities_for": "perf_delta","ChangeHealthDeltaService": "service",@@ -88,6 +93,7 @@ __all__ = ["MappingRevisionSource","MatchResult","MatchedFinding",+ "MoveMap","PerfOpportunityView","RevisionAnalysis","RevisionHealthAnalyzer",@@ -95,10 +101,13 @@ __all__ = ["RevisionPair","RevisionSource","ScopeCounts",+ "SymbolBody",+ "build_move_map","change_finding_id","filter_changes","finding_key","opportunities_for",+ "symbol_bodies",]diff --git a/packages/core/src/repowise/core/analysis/change_health/moves.py b/packages/core/src/repowise/core/analysis/change_health/moves.pynew file mode 100644index 000000000..18feef611--- /dev/null+++ b/packages/core/src/repowise/core/analysis/change_health/moves.py@@ -0,0 +1,175 @@+"""Symbols a change carried from one file to another.++A symbol's body is its token stream with comments and whitespace dropped and+its own name blanked, so a reindent, a comment edit or a rename does not hide a+move. Pairing runs in three passes over the symbols that left their place on+the base side and the ones that appeared on the head side, each pass taking+only keys unique on both sides: same name and body, then same body under a new+name, then same name with an edited body that kept most of its token pairs. A+move is cross-file only; a symbol that stays in its file, or in the file a+rename carried it to, is not a move.+"""++from __future__ import annotations++import hashlib+from bisect import bisect_left, bisect_right+from collections import Counter+from collections.abc import Callable, Hashable, Iterable, Mapping+from dataclasses import dataclass, field, replace+from itertools import pairwise+from types import MappingProxyType++from ...ingestion.models import Symbol+from ..health.duplication.tokenizer import TokenRow, tokenize_file_rows+from .analyzer import language_of++#: ``(path, symbol name)``.+SymbolKey = tuple[str, str]++_SKIPPED_KINDS = frozenset({"module", "impl"})++#: A body pairs under a new name only when it is this many tokens or more;+#: shorter bodies (``return x``, ``X = 1``) match by chance.+MIN_RENAMED_TOKENS = 20++#: An edited move may grow or shrink by at most this factor.+_MAX_EDITED_GROWTH = 2++#: An edited move keeps at least this share of its adjacent token pairs.+_MIN_EDITED_OVERLAP = 0.5++_NAME_MARK = b"\x00"+_SEPARATOR = b"\x1f"+++@dataclass(frozen=True, slots=True)+class SymbolBody:+ """One symbol's normalized body, as the move passes compare it."""++ path: str+ name: str+ digest: str+ tokens: int+ #: Adjacent token pairs, hashed; how much of the body an edit kept.+ pairs: frozenset[int] = field(default=frozenset(), compare=False, repr=False)++ @property+ def key(self) -> SymbolKey:+ return (self.path, self.name)+++@dataclass(frozen=True, slots=True)+class MoveMap:+ """``{(new_path, symbol): (old_path, old_symbol)}`` for the symbols that moved."""++ by_new: Mapping[SymbolKey, SymbolKey] = field(default_factory=lambda: MappingProxyType({}))+ #: Moved keys whose body also changed on the way.+ edited: frozenset[SymbolKey] = frozenset()++ def origin_of(self, path: str, symbol: str | None) -> SymbolKey | None:+ return self.by_new.get((path, symbol)) if symbol else None++ @property+ def moved_total(self) -> int:+ return len(self.by_new)+++def symbol_bodies(path: str, raw: bytes, symbols: Iterable[Symbol]) -> list[SymbolBody]:+ """The normalized body of each named symbol in *path*, read from its *raw* bytes."""+ language = language_of(path)+ rows = tokenize_file_rows(language, raw, path) if language else []+ if not rows:+ return []+ starts = [row[1] for row in rows]+ out: list[SymbolBody] = []+ for symbol in symbols:+ if not symbol.name or symbol.is_declaration or symbol.kind in _SKIPPED_KINDS:+ continue+ # Line-bounded: code sharing a line with the symbol joins its body.+ span = rows[bisect_left(starts, symbol.start_line) : bisect_right(starts, symbol.end_line)]+ if span:+ out.append(_body(path, symbol.name, raw, span))+ return out+++def build_move_map(+ base: Iterable[SymbolBody],+ head: Iterable[SymbolBody],+ *,+ rename_map: Mapping[str, str] | None = None,+) -> MoveMap:+ """Pair the base symbols that left their place with the head symbols that appeared."""+ renamed = dict(rename_map or {})+ based = [(b, replace(b, path=renamed.get(b.path, b.path))) for b in sorted(base, key=_order)]+ head_rows = sorted(head, key=_order)+ base_keys = Counter(on_head.key for _, on_head in based)+ head_keys = Counter(h.key for h in head_rows)+ # A key still present on the other side did not move, and one held twice+ # on its own side cannot name a single symbol.+ removed = [p for p in based if p[1].key not in head_keys and base_keys[p[1].key] == 1]+ added = [h for h in head_rows if h.key not in base_keys and head_keys[h.key] == 1]++ verbatim = _pair_unique(removed, added, lambda b: (b.name, b.digest))+ removed, added = _rest(removed, added, verbatim)+ sized = [h for h in added if h.tokens >= MIN_RENAMED_TOKENS]+ verbatim += _pair_unique(removed, sized, lambda b: b.digest)+ removed, added = _rest(removed, added, verbatim)+ edited = [+ (old, new)+ for old, new in _pair_unique(removed, added, lambda b: b.name)+ if _similar(old, new)+ ]+ by_new = {new.key: old.key for old, new in verbatim + edited}+ return MoveMap(MappingProxyType(by_new), frozenset(new.key for _, new in edited))+++def _pair_unique(+ removed: list[tuple[SymbolBody, SymbolBody]],+ added: list[SymbolBody],+ key: Callable[[SymbolBody], Hashable],+) -> list[tuple[SymbolBody, SymbolBody]]:+ """``(old, new)`` pairs whose *key* occurs once on each side, across files."""+ removed_counts = Counter(key(on_head) for _, on_head in removed)+ added_counts = Counter(key(h) for h in added)+ added_by_key = {key(h): h for h in added}+ pairs = []+ for old, on_head in removed:+ k = key(on_head)+ if removed_counts[k] == 1 and added_counts[k] == 1 and on_head.path != added_by_key[k].path:+ pairs.append((old, added_by_key[k]))+ return pairs+++def _rest(+ removed: list[tuple[SymbolBody, SymbolBody]],+ added: list[SymbolBody],+ pairs: list[tuple[SymbolBody, SymbolBody]],+) -> tuple[list[tuple[SymbolBody, SymbolBody]], list[SymbolBody]]:+ """What *pairs* left unclaimed on each side."""+ old_taken = {id(old) for old, _ in pairs}+ new_taken = {id(new) for _, new in pairs}+ return (+ [p for p in removed if id(p[0]) not in old_taken],+ [h for h in added if id(h) not in new_taken],+ )+++def _similar(old: SymbolBody, new: SymbolBody) -> bool:+ """An edited move keeps most of its body; a same-named stranger does not."""+ small, large = sorted((old.tokens, new.tokens))+ if small < MIN_RENAMED_TOKENS or large > small * _MAX_EDITED_GROWTH:+ return False+ shared = len(old.pairs & new.pairs)+ return shared >= _MIN_EDITED_OVERLAP * max(len(old.pairs), len(new.pairs), 1)+++def _body(path: str, name: str, raw: bytes, rows: list[TokenRow]) -> SymbolBody:+ parts = [_NAME_MARK if row[5] == name else raw[row[3] : row[4]] for row in rows]+ digest = hashlib.blake2b(_SEPARATOR.join(parts), digest_size=16).hexdigest()+ pairs = frozenset(hash(pair) for pair in pairwise(parts))+ return SymbolBody(path, name, digest, len(parts), pairs)+++def _order(body: SymbolBody) -> tuple[str, str, str]:+ return (body.path, body.name, body.digest)diff --git a/tests/unit/change_health/test_moves.py b/tests/unit/change_health/test_moves.pynew file mode 100644index 000000000..e2e537bd0--- /dev/null+++ b/tests/unit/change_health/test_moves.py@@ -0,0 +1,165 @@+"""The move map: which symbols a change carried to another file."""++from __future__ import annotations++from repowise.core.analysis.change_health.analyzer import _file_info, language_of+from repowise.core.analysis.change_health.moves import (+ SymbolBody,+ build_move_map,+ symbol_bodies,+)+from repowise.core.ingestion.parser import parse_file++from .conftest import python_complex+++def bodies(files: dict[str, str]) -> list[SymbolBody]:+ out: list[SymbolBody] = []+ for path, text in files.items():+ raw = text.encode("utf-8")+ parsed = parse_file(_file_info(path, language_of(path) or "", raw), raw)+ out.extend(symbol_bodies(path, raw, parsed.symbols))+ return out+++def moves(base: dict[str, str], head: dict[str, str], rename_map=None):+ return build_move_map(bodies(base), bodies(head), rename_map=rename_map)+++CONSTANTS = 'LIMIT = 30\n_NAMES = frozenset({"alpha", "beta"})\n\n\n'+++def test_a_file_split_into_a_package_maps_every_symbol_to_its_origin():+ base = {"tool.py": CONSTANTS + python_complex("tangle", 6) + "\n\n" + python_complex("knot", 5)}+ head = {+ "tool/__init__.py": "from .a import tangle\nfrom .b import knot\n",+ "tool/a.py": python_complex("tangle", 6),+ "tool/b.py": CONSTANTS + python_complex("knot", 5),+ }++ result = moves(base, head)++ assert result.origin_of("tool/a.py", "tangle") == ("tool.py", "tangle")+ assert result.origin_of("tool/b.py", "knot") == ("tool.py", "knot")+ assert result.origin_of("tool/b.py", "LIMIT") == ("tool.py", "LIMIT")+ assert result.origin_of("tool/b.py", "_NAMES") == ("tool.py", "_NAMES")+ assert result.moved_total == 4+ assert not result.edited+++def test_a_partial_split_maps_only_what_left_the_file():+ base = {"tool.py": python_complex("tangle", 6) + "\n\n" + python_complex("knot", 5)}+ head = {"tool.py": python_complex("tangle", 6), "knots.py": python_complex("knot", 5)}++ result = moves(base, head)++ assert dict(result.by_new) == {("knots.py", "knot"): ("tool.py", "knot")}+++def test_a_reindent_inside_one_file_is_not_a_move():+ method = python_complex("tangle", 6, indent=" ")+ base = {"tool.py": python_complex("tangle", 6)}+ head = {"tool.py": "class Box:\n # gathered here\n" + method}++ assert moves(base, head).moved_total == 0+++def test_a_move_survives_reindenting_and_comment_edits():+ method = python_complex("tangle", 6, indent=" ").replace(+ " value = value + 1\n", " value = value + 1 # bump\n"+ )+ base = {"tool.py": "# old home\n" + python_complex("tangle", 6), "box.py": "X = 1\n"}+ head = {"tool.py": "# emptied\n", "box.py": "X = 1\n\n\nclass Box:\n" + method}++ assert moves(base, head).origin_of("box.py", "tangle") == ("tool.py", "tangle")+++def test_a_renamed_file_is_not_a_move():+ body = python_complex("tangle", 6)+ result = moves({"old.py": body}, {"new.py": body}, rename_map={"old.py": "new.py"})++ assert result.moved_total == 0+++def test_a_symbol_renamed_on_its_way_to_another_file_keeps_its_origin():+ base = {"tool.py": python_complex("tangle", 6)}+ head = {"tool.py": "", "knots.py": python_complex("knot", 6)}++ assert moves(base, head).origin_of("knots.py", "knot") == ("tool.py", "tangle")+++def test_a_copy_beside_the_original_is_not_a_move():+ body = python_complex("tangle", 6)+ result = moves({"tool.py": body}, {"tool.py": body, "copy.py": body})++ assert result.moved_total == 0+++def test_two_candidate_origins_for_one_body_pair_neither():+ body = python_complex("tangle", 6)+ result = moves({"a.py": body, "b.py": body}, {"a.py": "", "b.py": "", "c.py": body})++ assert result.moved_total == 0+++def test_a_moved_and_edited_symbol_pairs_by_its_unique_name():+ base = {"tool.py": python_complex("tangle", 9)}+ head = {"tool.py": "", "knots.py": python_complex("tangle", 6)}++ result = moves(base, head)++ assert result.origin_of("knots.py", "tangle") == ("tool.py", "tangle")+ assert ("knots.py", "tangle") in result.edited+++def test_a_short_body_never_pairs_under_a_new_name():+ base = {"a.py": "LIMIT = 30\n"}+ head = {"a.py": "", "b.py": "CAP = 30\n"}++ assert moves(base, head).moved_total == 0+++def test_an_unrelated_function_of_the_same_name_is_not_an_edited_move():+ base = {"tool.py": python_complex("run", 12)}+ head = {"tool.py": "", "cli.py": "def run(argv):\n return main(argv)\n"}++ assert moves(base, head).moved_total == 0+++def test_a_key_shared_with_a_new_symbol_names_no_origin():+ """Findings look a symbol up by name, so a moved ``run`` beside a new one is unsafe."""+ moved = "class Old:\n" + python_complex("run", 6, indent=" ")+ fresh = "class New:\n" + python_complex("run", 3, indent=" ")+ result = moves({"a.py": moved}, {"a.py": "", "b.py": moved + fresh})++ assert result.origin_of("b.py", "run") is None+ assert result.origin_of("b.py", "Old") == ("a.py", "Old")+++def test_a_copy_beside_an_original_edited_in_place_is_not_a_move():+ base = {"tool.py": python_complex("tangle", 6)}+ head = {"tool.py": python_complex("tangle", 7), "copy.py": python_complex("tangle", 6)}++ assert moves(base, head).moved_total == 0+++def test_a_move_out_of_a_renamed_file_names_the_base_path():+ base = {"old.py": python_complex("tangle", 6) + "\n\n" + python_complex("knot", 5)}+ head = {"new.py": python_complex("tangle", 6), "knots.py": python_complex("knot", 5)}++ result = moves(base, head, rename_map={"old.py": "new.py"})++ assert dict(result.by_new) == {("knots.py", "knot"): ("old.py", "knot")}+++def test_a_same_named_function_with_a_different_body_is_not_an_edited_move():+ """Similar size, unrelated code: the name alone is no evidence of a move."""+ other = "\n".join(+ ["def tangle(rows):", " out = []"]+ + [f" out.append(str(rows[{i}]).strip().lower())" for i in range(12)]+ + [" return out", ""]+ )+ base = {"tool.py": python_complex("tangle", 6)}+ head = {"tool.py": "", "knots.py": other}++ assert moves(base, head).moved_total == 0