Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(change-risk): map symbols a change moves across files (#4109)

Raghav Chamadiya23dcc35007bc
diff --git a/packages/core/src/repowise/core/analysis/change_health/__init__.py b/packages/core/src/repowise/core/analysis/change_health/__init__.py
index ec9157eea..8809f569e 100644
--- a/packages/core/src/repowise/core/analysis/change_health/__init__.py
+++ b/packages/core/src/repowise/core/analysis/change_health/__init__.py
@@ -40,6 +40,7 @@ if TYPE_CHECKING:
RevisionId,
ScopeCounts,
)
+ from .moves import MoveMap, SymbolBody, build_move_map, symbol_bodies
from .perf_delta import PerfOpportunityView, opportunities_for
from .service import ChangeHealthDeltaService, DeltaRequest
@@ -64,6 +65,10 @@ _LAZY_EXPORTS: dict[str, str] = {
"FindingKey": "models",
"RevisionId": "models",
"ScopeCounts": "models",
+ "MoveMap": "moves",
+ "SymbolBody": "moves",
+ "build_move_map": "moves",
+ "symbol_bodies": "moves",
"PerfOpportunityView": "perf_delta",
"opportunities_for": "perf_delta",
"ChangeHealthDeltaService": "service",
@@ -88,6 +93,7 @@ __all__ = [
"MappingRevisionSource",
"MatchResult",
"MatchedFinding",
+ "MoveMap",
"PerfOpportunityView",
"RevisionAnalysis",
"RevisionHealthAnalyzer",
@@ -95,10 +101,13 @@ __all__ = [
"RevisionPair",
"RevisionSource",
"ScopeCounts",
+ "SymbolBody",
+ "build_move_map",
"change_finding_id",
"filter_changes",
"finding_key",
"opportunities_for",
+ "symbol_bodies",
]
diff --git a/packages/core/src/repowise/core/analysis/change_health/moves.py b/packages/core/src/repowise/core/analysis/change_health/moves.py
new file mode 100644
index 000000000..18feef611
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/change_health/moves.py
@@ -0,0 +1,175 @@
+"""Symbols a change carried from one file to another.
+
+A symbol's body is its token stream with comments and whitespace dropped and
+its own name blanked, so a reindent, a comment edit or a rename does not hide a
+move. Pairing runs in three passes over the symbols that left their place on
+the base side and the ones that appeared on the head side, each pass taking
+only keys unique on both sides: same name and body, then same body under a new
+name, then same name with an edited body that kept most of its token pairs. A
+move is cross-file only; a symbol that stays in its file, or in the file a
+rename carried it to, is not a move.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from bisect import bisect_left, bisect_right
+from collections import Counter
+from collections.abc import Callable, Hashable, Iterable, Mapping
+from dataclasses import dataclass, field, replace
+from itertools import pairwise
+from types import MappingProxyType
+
+from ...ingestion.models import Symbol
+from ..health.duplication.tokenizer import TokenRow, tokenize_file_rows
+from .analyzer import language_of
+
+#: ``(path, symbol name)``.
+SymbolKey = tuple[str, str]
+
+_SKIPPED_KINDS = frozenset({"module", "impl"})
+
+#: A body pairs under a new name only when it is this many tokens or more;
+#: shorter bodies (``return x``, ``X = 1``) match by chance.
+MIN_RENAMED_TOKENS = 20
+
+#: An edited move may grow or shrink by at most this factor.
+_MAX_EDITED_GROWTH = 2
+
+#: An edited move keeps at least this share of its adjacent token pairs.
+_MIN_EDITED_OVERLAP = 0.5
+
+_NAME_MARK = b"\x00"
+_SEPARATOR = b"\x1f"
+
+
+@dataclass(frozen=True, slots=True)
+class SymbolBody:
+ """One symbol's normalized body, as the move passes compare it."""
+
+ path: str
+ name: str
+ digest: str
+ tokens: int
+ #: Adjacent token pairs, hashed; how much of the body an edit kept.
+ pairs: frozenset[int] = field(default=frozenset(), compare=False, repr=False)
+
+ @property
+ def key(self) -> SymbolKey:
+ return (self.path, self.name)
+
+
+@dataclass(frozen=True, slots=True)
+class MoveMap:
+ """``{(new_path, symbol): (old_path, old_symbol)}`` for the symbols that moved."""
+
+ by_new: Mapping[SymbolKey, SymbolKey] = field(default_factory=lambda: MappingProxyType({}))
+ #: Moved keys whose body also changed on the way.
+ edited: frozenset[SymbolKey] = frozenset()
+
+ def origin_of(self, path: str, symbol: str | None) -> SymbolKey | None:
+ return self.by_new.get((path, symbol)) if symbol else None
+
+ @property
+ def moved_total(self) -> int:
+ return len(self.by_new)
+
+
+def symbol_bodies(path: str, raw: bytes, symbols: Iterable[Symbol]) -> list[SymbolBody]:
+ """The normalized body of each named symbol in *path*, read from its *raw* bytes."""
+ language = language_of(path)
+ rows = tokenize_file_rows(language, raw, path) if language else []
+ if not rows:
+ return []
+ starts = [row[1] for row in rows]
+ out: list[SymbolBody] = []
+ for symbol in symbols:
+ if not symbol.name or symbol.is_declaration or symbol.kind in _SKIPPED_KINDS:
+ continue
+ # Line-bounded: code sharing a line with the symbol joins its body.
+ span = rows[bisect_left(starts, symbol.start_line) : bisect_right(starts, symbol.end_line)]
+ if span:
+ out.append(_body(path, symbol.name, raw, span))
+ return out
+
+
+def build_move_map(
+ base: Iterable[SymbolBody],
+ head: Iterable[SymbolBody],
+ *,
+ rename_map: Mapping[str, str] | None = None,
+) -> MoveMap:
+ """Pair the base symbols that left their place with the head symbols that appeared."""
+ renamed = dict(rename_map or {})
+ based = [(b, replace(b, path=renamed.get(b.path, b.path))) for b in sorted(base, key=_order)]
+ head_rows = sorted(head, key=_order)
+ base_keys = Counter(on_head.key for _, on_head in based)
+ head_keys = Counter(h.key for h in head_rows)
+ # A key still present on the other side did not move, and one held twice
+ # on its own side cannot name a single symbol.
+ removed = [p for p in based if p[1].key not in head_keys and base_keys[p[1].key] == 1]
+ added = [h for h in head_rows if h.key not in base_keys and head_keys[h.key] == 1]
+
+ verbatim = _pair_unique(removed, added, lambda b: (b.name, b.digest))
+ removed, added = _rest(removed, added, verbatim)
+ sized = [h for h in added if h.tokens >= MIN_RENAMED_TOKENS]
+ verbatim += _pair_unique(removed, sized, lambda b: b.digest)
+ removed, added = _rest(removed, added, verbatim)
+ edited = [
+ (old, new)
+ for old, new in _pair_unique(removed, added, lambda b: b.name)
+ if _similar(old, new)
+ ]
+ by_new = {new.key: old.key for old, new in verbatim + edited}
+ return MoveMap(MappingProxyType(by_new), frozenset(new.key for _, new in edited))
+
+
+def _pair_unique(
+ removed: list[tuple[SymbolBody, SymbolBody]],
+ added: list[SymbolBody],
+ key: Callable[[SymbolBody], Hashable],
+) -> list[tuple[SymbolBody, SymbolBody]]:
+ """``(old, new)`` pairs whose *key* occurs once on each side, across files."""
+ removed_counts = Counter(key(on_head) for _, on_head in removed)
+ added_counts = Counter(key(h) for h in added)
+ added_by_key = {key(h): h for h in added}
+ pairs = []
+ for old, on_head in removed:
+ k = key(on_head)
+ if removed_counts[k] == 1 and added_counts[k] == 1 and on_head.path != added_by_key[k].path:
+ pairs.append((old, added_by_key[k]))
+ return pairs
+
+
+def _rest(
+ removed: list[tuple[SymbolBody, SymbolBody]],
+ added: list[SymbolBody],
+ pairs: list[tuple[SymbolBody, SymbolBody]],
+) -> tuple[list[tuple[SymbolBody, SymbolBody]], list[SymbolBody]]:
+ """What *pairs* left unclaimed on each side."""
+ old_taken = {id(old) for old, _ in pairs}
+ new_taken = {id(new) for _, new in pairs}
+ return (
+ [p for p in removed if id(p[0]) not in old_taken],
+ [h for h in added if id(h) not in new_taken],
+ )
+
+
+def _similar(old: SymbolBody, new: SymbolBody) -> bool:
+ """An edited move keeps most of its body; a same-named stranger does not."""
+ small, large = sorted((old.tokens, new.tokens))
+ if small < MIN_RENAMED_TOKENS or large > small * _MAX_EDITED_GROWTH:
+ return False
+ shared = len(old.pairs & new.pairs)
+ return shared >= _MIN_EDITED_OVERLAP * max(len(old.pairs), len(new.pairs), 1)
+
+
+def _body(path: str, name: str, raw: bytes, rows: list[TokenRow]) -> SymbolBody:
+ parts = [_NAME_MARK if row[5] == name else raw[row[3] : row[4]] for row in rows]
+ digest = hashlib.blake2b(_SEPARATOR.join(parts), digest_size=16).hexdigest()
+ pairs = frozenset(hash(pair) for pair in pairwise(parts))
+ return SymbolBody(path, name, digest, len(parts), pairs)
+
+
+def _order(body: SymbolBody) -> tuple[str, str, str]:
+ return (body.path, body.name, body.digest)
diff --git a/tests/unit/change_health/test_moves.py b/tests/unit/change_health/test_moves.py
new file mode 100644
index 000000000..e2e537bd0
--- /dev/null
+++ b/tests/unit/change_health/test_moves.py
@@ -0,0 +1,165 @@
+"""The move map: which symbols a change carried to another file."""
+
+from __future__ import annotations
+
+from repowise.core.analysis.change_health.analyzer import _file_info, language_of
+from repowise.core.analysis.change_health.moves import (
+ SymbolBody,
+ build_move_map,
+ symbol_bodies,
+)
+from repowise.core.ingestion.parser import parse_file
+
+from .conftest import python_complex
+
+
+def bodies(files: dict[str, str]) -> list[SymbolBody]:
+ out: list[SymbolBody] = []
+ for path, text in files.items():
+ raw = text.encode("utf-8")
+ parsed = parse_file(_file_info(path, language_of(path) or "", raw), raw)
+ out.extend(symbol_bodies(path, raw, parsed.symbols))
+ return out
+
+
+def moves(base: dict[str, str], head: dict[str, str], rename_map=None):
+ return build_move_map(bodies(base), bodies(head), rename_map=rename_map)
+
+
+CONSTANTS = 'LIMIT = 30\n_NAMES = frozenset({"alpha", "beta"})\n\n\n'
+
+
+def test_a_file_split_into_a_package_maps_every_symbol_to_its_origin():
+ base = {"tool.py": CONSTANTS + python_complex("tangle", 6) + "\n\n" + python_complex("knot", 5)}
+ head = {
+ "tool/__init__.py": "from .a import tangle\nfrom .b import knot\n",
+ "tool/a.py": python_complex("tangle", 6),
+ "tool/b.py": CONSTANTS + python_complex("knot", 5),
+ }
+
+ result = moves(base, head)
+
+ assert result.origin_of("tool/a.py", "tangle") == ("tool.py", "tangle")
+ assert result.origin_of("tool/b.py", "knot") == ("tool.py", "knot")
+ assert result.origin_of("tool/b.py", "LIMIT") == ("tool.py", "LIMIT")
+ assert result.origin_of("tool/b.py", "_NAMES") == ("tool.py", "_NAMES")
+ assert result.moved_total == 4
+ assert not result.edited
+
+
+def test_a_partial_split_maps_only_what_left_the_file():
+ base = {"tool.py": python_complex("tangle", 6) + "\n\n" + python_complex("knot", 5)}
+ head = {"tool.py": python_complex("tangle", 6), "knots.py": python_complex("knot", 5)}
+
+ result = moves(base, head)
+
+ assert dict(result.by_new) == {("knots.py", "knot"): ("tool.py", "knot")}
+
+
+def test_a_reindent_inside_one_file_is_not_a_move():
+ method = python_complex("tangle", 6, indent=" ")
+ base = {"tool.py": python_complex("tangle", 6)}
+ head = {"tool.py": "class Box:\n # gathered here\n" + method}
+
+ assert moves(base, head).moved_total == 0
+
+
+def test_a_move_survives_reindenting_and_comment_edits():
+ method = python_complex("tangle", 6, indent=" ").replace(
+ " value = value + 1\n", " value = value + 1 # bump\n"
+ )
+ base = {"tool.py": "# old home\n" + python_complex("tangle", 6), "box.py": "X = 1\n"}
+ head = {"tool.py": "# emptied\n", "box.py": "X = 1\n\n\nclass Box:\n" + method}
+
+ assert moves(base, head).origin_of("box.py", "tangle") == ("tool.py", "tangle")
+
+
+def test_a_renamed_file_is_not_a_move():
+ body = python_complex("tangle", 6)
+ result = moves({"old.py": body}, {"new.py": body}, rename_map={"old.py": "new.py"})
+
+ assert result.moved_total == 0
+
+
+def test_a_symbol_renamed_on_its_way_to_another_file_keeps_its_origin():
+ base = {"tool.py": python_complex("tangle", 6)}
+ head = {"tool.py": "", "knots.py": python_complex("knot", 6)}
+
+ assert moves(base, head).origin_of("knots.py", "knot") == ("tool.py", "tangle")
+
+
+def test_a_copy_beside_the_original_is_not_a_move():
+ body = python_complex("tangle", 6)
+ result = moves({"tool.py": body}, {"tool.py": body, "copy.py": body})
+
+ assert result.moved_total == 0
+
+
+def test_two_candidate_origins_for_one_body_pair_neither():
+ body = python_complex("tangle", 6)
+ result = moves({"a.py": body, "b.py": body}, {"a.py": "", "b.py": "", "c.py": body})
+
+ assert result.moved_total == 0
+
+
+def test_a_moved_and_edited_symbol_pairs_by_its_unique_name():
+ base = {"tool.py": python_complex("tangle", 9)}
+ head = {"tool.py": "", "knots.py": python_complex("tangle", 6)}
+
+ result = moves(base, head)
+
+ assert result.origin_of("knots.py", "tangle") == ("tool.py", "tangle")
+ assert ("knots.py", "tangle") in result.edited
+
+
+def test_a_short_body_never_pairs_under_a_new_name():
+ base = {"a.py": "LIMIT = 30\n"}
+ head = {"a.py": "", "b.py": "CAP = 30\n"}
+
+ assert moves(base, head).moved_total == 0
+
+
+def test_an_unrelated_function_of_the_same_name_is_not_an_edited_move():
+ base = {"tool.py": python_complex("run", 12)}
+ head = {"tool.py": "", "cli.py": "def run(argv):\n return main(argv)\n"}
+
+ assert moves(base, head).moved_total == 0
+
+
+def test_a_key_shared_with_a_new_symbol_names_no_origin():
+ """Findings look a symbol up by name, so a moved ``run`` beside a new one is unsafe."""
+ moved = "class Old:\n" + python_complex("run", 6, indent=" ")
+ fresh = "class New:\n" + python_complex("run", 3, indent=" ")
+ result = moves({"a.py": moved}, {"a.py": "", "b.py": moved + fresh})
+
+ assert result.origin_of("b.py", "run") is None
+ assert result.origin_of("b.py", "Old") == ("a.py", "Old")
+
+
+def test_a_copy_beside_an_original_edited_in_place_is_not_a_move():
+ base = {"tool.py": python_complex("tangle", 6)}
+ head = {"tool.py": python_complex("tangle", 7), "copy.py": python_complex("tangle", 6)}
+
+ assert moves(base, head).moved_total == 0
+
+
+def test_a_move_out_of_a_renamed_file_names_the_base_path():
+ base = {"old.py": python_complex("tangle", 6) + "\n\n" + python_complex("knot", 5)}
+ head = {"new.py": python_complex("tangle", 6), "knots.py": python_complex("knot", 5)}
+
+ result = moves(base, head, rename_map={"old.py": "new.py"})
+
+ assert dict(result.by_new) == {("knots.py", "knot"): ("old.py", "knot")}
+
+
+def test_a_same_named_function_with_a_different_body_is_not_an_edited_move():
+ """Similar size, unrelated code: the name alone is no evidence of a move."""
+ other = "\n".join(
+ ["def tangle(rows):", " out = []"]
+ + [f" out.append(str(rows[{i}]).strip().lower())" for i in range(12)]
+ + [" return out", ""]
+ )
+ base = {"tool.py": python_complex("tangle", 6)}
+ head = {"tool.py": "", "knots.py": other}
+
+ assert moves(base, head).moved_total == 0