Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(health): tokenize and verify clones without per-token objects (#2878)
Raghav Chamadiya99424cefac3f
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.pyindex 6e5d1db47..0daf3c428 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py@@ -36,8 +36,8 @@ from repowise.core.cancellation import check_cancelledfrom repowise.core.co_change import parse_partnersfrom .limits import DuplicationDiagnostics, DuplicationLimits, looks_minified-from .rabin_karp import WindowHash, index_by_hash, rolling_hashes-from .tokenizer import tokenize_file+from .rabin_karp import WindowHash, WindowRow, index_by_hash, window_rows+from .tokenizer import tokenize_file_rowslog = structlog.get_logger(__name__)@@ -136,33 +136,12 @@ def _is_noise(kind: str) -> bool:return kind == "LIT" or not any(ch.isalnum() for ch in kind)-def _informative_windows(- windows: list[WindowHash], kinds: list[str], window: int-) -> list[WindowHash]:+def _informative_windows(windows: list[WindowRow], kinds: list[str], window: int) -> list[WindowRow]:"""Drop windows that are more than ``_MAX_NOISE_SHARE`` literals/punctuation."""- noise = list(accumulate((_is_noise(k) for k in kinds), initial=0))+ noisy = {kind: _is_noise(kind) for kind in set(kinds)}+ noise = list(accumulate((noisy[k] for k in kinds), initial=0))limit = _MAX_NOISE_SHARE * window- return [w for w in windows if noise[w.start_index + window] - noise[w.start_index] <= limit]---def _windows_match(a: _Stream, b: _Stream, a_start: int, b_start: int, window: int) -> bool:- """Confirm a hash match: equal token kinds, then shared identifier names.-- Operates on the per-file streams so cached token streams round-trip- without rebuilding Token objects.- """- a_kinds, a_names = a- b_kinds, b_names = b- if a_start + window > len(a_kinds) or b_start + window > len(b_kinds):- return False- if a_kinds[a_start : a_start + window] != b_kinds[b_start : b_start + window]:- return False- a_set = set(a_names[a_start : a_start + window])- b_set = set(b_names[b_start : b_start + window])- a_set.discard("")- b_set.discard("")- union = len(a_set | b_set)- return union > 0 and len(a_set & b_set) >= _MIN_NAME_JACCARD * union+ return [w for w in windows if noise[w[1] + window] - noise[w[1]] <= limit]def _merge_adjacent_pairs(raw: list[ClonePair]) -> list[ClonePair]:@@ -396,35 +375,26 @@ def _collect_windows(if cached is not None:kinds, names, nloc, window_tuples = cached- windows = [- WindowHash(- file_path=path,- hash_value=h,- start_index=si,- start_line=sl,- end_line=el,- )- for h, si, sl, el in window_tuples- ]else:- toks = tokenize_file(language, source, path)- if len(toks) > limits.max_tokens_per_file:+ rows = tokenize_file_rows(language, source, path)+ if len(rows) > limits.max_tokens_per_file:diag.skipped_token_cap += 1continue- kinds = [t.kind for t in toks]- names = [t.name for t in toks]+ kinds, start_lines, end_lines, names = [], [], [], []+ if rows:+ kinds, start_lines, end_lines, _, _, names = map(list, zip(*rows, strict=True))+ del rowsnloc = _nloc(source)- windows = _informative_windows(- rolling_hashes(path, toks, window_tokens), kinds, window_tokens+ window_tuples = _informative_windows(+ window_rows(kinds, start_lines, end_lines, window_tokens), kinds, window_tokens)+ del start_lines, end_linesif cache is not None:- cache.put(- content_hash,- kinds,- names,- nloc,- [(w.hash_value, w.start_index, w.start_line, w.end_line) for w in windows],- )+ cache.put(content_hash, kinds, names, nloc, window_tuples)+ windows = [+ WindowHash(file_path=path, hash_value=h, start_index=si, start_line=sl, end_line=el)+ for h, si, sl, el in window_tuples+ ]if len(kinds) < window_tokens:continue@@ -468,7 +438,6 @@ def _pairs_from_buckets(return the pairs found so far rather than spinning indefinitely."""raw_pairs: list[ClonePair] = []- seen: set[tuple[str, int, str, int]] = set()deadline = (time.monotonic() + limits.time_budget_secs) if limits.time_budget_secs else Nonefor i, windows in enumerate(bucket.values()):@@ -483,7 +452,7 @@ def _pairs_from_buckets(if deadline is not None and (i & 0x3FF) == 0 and time.monotonic() > deadline:diag.timed_out = Truebreak- _verify_bucket(windows, per_file_streams, window_tokens, seen, raw_pairs)+ _verify_bucket(windows, per_file_streams, window_tokens, raw_pairs)return raw_pairs@@ -492,32 +461,45 @@ def _verify_bucket(windows: list[WindowHash],per_file_streams: dict[str, _Stream],window_tokens: int,- seen: set[tuple[str, int, str, int]],out: list[ClonePair],) -> None:"""Confirm every unordered pair in one (bounded) hash bucket.- Hash equality is necessary but not sufficient — ``_windows_match``- rejects collisions and renamed look-alikes.+ Hash equality is necessary but not sufficient: a pair must have equal+ token kinds (rules out collisions) and share at least+ ``_MIN_NAME_JACCARD`` of their raw identifier names (rules out renamed+ look-alikes). Each window's kind slice and name set is built once per+ bucket rather than once per pair. A window has one hash, so it sits in+ exactly one bucket and no pair is seen twice."""+ kind_slices: list[list[str] | None] = []+ name_sets: list[set[str] | None] = []+ for w in windows:+ kinds = per_file_streams[w.file_path][0]+ end = w.start_index + window_tokens+ kind_slices.append(kinds[w.start_index : end] if end <= len(kinds) else None)+ name_sets.append(None)+for i in range(len(windows)):+ a_kinds = kind_slices[i]+ if a_kinds is None:+ continuefor j in range(i + 1, len(windows)):+ if kind_slices[j] != a_kinds:+ continue+ a_names = name_sets[i]+ if a_names is None:+ a_names = name_sets[i] = _window_names(windows[i], per_file_streams, window_tokens)+ b_names = name_sets[j]+ if b_names is None:+ b_names = name_sets[j] = _window_names(windows[j], per_file_streams, window_tokens)+ union = len(a_names | b_names)+ if not (union > 0 and len(a_names & b_names) >= _MIN_NAME_JACCARD * union):+ continuea, b = windows[i], windows[j]# Canonicalize so (file_a, file_b) ordering is stable.if (a.file_path, a.start_index) > (b.file_path, b.start_index):a, b = b, a- key = (a.file_path, a.start_index, b.file_path, b.start_index)- if key in seen:- continue- seen.add(key)- if not _windows_match(- per_file_streams[a.file_path],- per_file_streams[b.file_path],- a.start_index,- b.start_index,- window_tokens,- ):- continueout.append(ClonePair(file_a=a.file_path,@@ -531,6 +513,13 @@ def _verify_bucket()+def _window_names(w: WindowHash, per_file_streams: dict[str, _Stream], window_tokens: int) -> set[str]:+ """The raw identifier names inside one window."""+ names = set(per_file_streams[w.file_path][1][w.start_index : w.start_index + window_tokens])+ names.discard("")+ return names++# ---------------------------------------------------------------------------# Incremental splice (update path)# ---------------------------------------------------------------------------@@ -775,12 +764,9 @@ def _bucket_contributions(The degenerate-bucket cap applies to each bucket's full membership(rows here cover it: every window with a touched hash was gathered),so a bucket crossing the cap in either direction contributes pairs on- exactly one side of the splice. The shared ``seen`` set mirrors the- full pipeline's; (file, start_index) pairs are unique to one bucket,- so per-run scoping is equivalent.+ exactly one side of the splice."""out: list[ClonePair] = []- seen: set[tuple[str, int, str, int]] = set()for rows in rows_by_hash.values():if len(rows) < 2:continue@@ -789,7 +775,7 @@ def _bucket_contributions(diag.degenerate_buckets += 1continuecheck_cancelled()- _verify_bucket(rows, streams_map, window_tokens, seen, out)+ _verify_bucket(rows, streams_map, window_tokens, out)return outdiff --git a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.pyindex a849443d8..6fd8f2ba4 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py@@ -51,6 +51,46 @@ class WindowHash:end_line: int+# One window as a plain tuple: (hash_value, start_index, start_line, end_line).+WindowRow = tuple[int, int, int, int]+++def window_rows(+ kinds: list[str],+ start_lines: list[int],+ end_lines: list[int],+ window: int,+) -> list[WindowRow]:+ """Rolling hashes for every window of length *window*, as plain tuples.++ The parallel lists describe one token stream. Returns an empty list when+ the stream is shorter than one window. A repository scan rolls millions+ of windows, so the per-kind hash is computed once per distinct spelling+ and no object is built per window.+ """+ n = len(kinds)+ if n < window or window <= 0:+ return []++ base, modulus = _BASE, _MODULUS+ base_pow = pow(base, window - 1, modulus)+ per_kind = {kind: _token_hash(kind) for kind in set(kinds)}+ kind_hashes = [per_kind[kind] for kind in kinds]++ h = 0+ for i in range(window):+ h = (h * base + kind_hashes[i]) % modulus++ out: list[WindowRow] = [(h, 0, start_lines[0], end_lines[window - 1])]+ append = out.append+ # Python's % yields a non-negative result for a positive modulus.+ for i in range(1, n - window + 1):+ last = i + window - 1+ h = ((h - kind_hashes[i - 1] * base_pow) * base + kind_hashes[last]) % modulus+ append((h, i, start_lines[i], end_lines[last]))+ return out++def rolling_hashes(file_path: str,tokens: list[Token],@@ -62,43 +102,16 @@ def rolling_hashes(window size — callers filter clone candidates by minimum sizeupstream so the empty-result case is well-defined."""- n = len(tokens)- if n < window or window <= 0:- return []-- base_pow = pow(_BASE, window - 1, _MODULUS)- kind_hashes = [_token_hash(t.kind) for t in tokens]-- h = 0- for i in range(window):- h = (h * _BASE + kind_hashes[i]) % _MODULUS-- out: list[WindowHash] = []- out.append(- WindowHash(- file_path=file_path,- hash_value=h,- start_index=0,- start_line=tokens[0].start_line,- end_line=tokens[window - 1].end_line,- )+ rows = window_rows(+ [t.kind for t in tokens],+ [t.start_line for t in tokens],+ [t.end_line for t in tokens],+ window,)- for i in range(1, n - window + 1):- h = ((h - kind_hashes[i - 1] * base_pow) * _BASE + kind_hashes[i + window - 1]) % _MODULUS- # Python's % already yields a non-negative result for a positive- # modulus, but we keep the guard for clarity.- if h < 0:- h += _MODULUS- out.append(- WindowHash(- file_path=file_path,- hash_value=h,- start_index=i,- start_line=tokens[i].start_line,- end_line=tokens[i + window - 1].end_line,- )- )- return out+ return [+ WindowHash(file_path=file_path, hash_value=h, start_index=si, start_line=sl, end_line=el)+ for h, si, sl, el in rows+ ]def index_by_hash(hashes: Iterable[WindowHash]) -> dict[int, list[WindowHash]]:diff --git a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.pyindex 1ea0a96e8..90bbd492e 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py@@ -25,7 +25,9 @@ preserve structure. Each identifier token also keeps its raw text infrom __future__ import annotations+import reimport sys+from bisect import bisect_leftfrom dataclasses import dataclassfrom typing import TYPE_CHECKING@@ -113,77 +115,92 @@ def import_node_kinds(language: str) -> frozenset[str]:return frozenset(config.import_node_types) - _NOT_IMPORT_KINDS-def _is_skippable(node: Node) -> bool:- if node.type in _COMMENT_KINDS:- return True- # Tree-sitter exposes ERROR / MISSING for parse errors - drop them- # so a single broken file can't pollute the hash stream.- return bool(getattr(node, "has_error", False) and node.child_count == 0)+_NEWLINE = re.compile(rb"\n")+# One leaf token in ``Token`` field order: (kind, start_line, end_line,+# start_byte, end_byte, name).+TokenRow = tuple[str, int, int, int, int, str]-def _is_reexport(node: Node) -> bool:- # JS/TS ``export { a } from "./b"``: an import in all but name.- return node.type == "export_statement" and node.child_by_field_name("source") is not None++def token_rows(+ root: Node, source: bytes, skip_kinds: frozenset[str] = frozenset()+) -> list[TokenRow]:+ """Walk *root* and return one ``TokenRow`` per kept leaf, in source order.++ Dropped: comments, parse-error leaves, whitespace-only leaves, and whole+ subtrees whose kind is in *skip_kinds* (import statements) or that are+ re-exports. A clone scan walks every node of every file, so this is a+ cursor walk that reads each node's attributes once and emits plain+ tuples; ``tokenize_tree`` wraps them in ``Token`` for callers that want+ objects.+ """+ out: list[TokenRow] = []+ append = out.append+ intern = sys.intern+ # A node's row is the number of newlines before its byte offset, which+ # a C bisect answers faster than building tree-sitter Point objects.+ newlines = [m.start() for m in _NEWLINE.finditer(source)]+ cursor = root.walk()+ while True:+ node = cursor.node+ ntype = node.type+ child_count = node.child_count+ if not (+ ntype in _COMMENT_KINDS+ # Tree-sitter exposes ERROR / MISSING for parse errors - drop them+ # so a single broken file can't pollute the hash stream.+ or (child_count == 0 and node.has_error)+ or ntype in skip_kinds+ # JS/TS ``export { a } from "./b"``: an import in all but name.+ or (ntype == "export_statement" and node.child_by_field_name("source") is not None)+ ):+ if child_count == 0:+ start_byte = node.start_byte+ end_byte = node.end_byte+ name = ""+ if ntype in _IDENTIFIER_KINDS:+ kind = "ID"+ name = intern(source[start_byte:end_byte].decode("utf-8", errors="replace"))+ elif ntype in _LITERAL_KINDS:+ kind = "LIT"+ else:+ # Raw token text for operators / keywords / punctuation.+ # Interned: a repo-sized population of equal spellings+ # becomes one object per distinct spelling.+ text = source[start_byte:end_byte]+ kind = intern(text.decode("utf-8", errors="replace")) if text.strip() else ""+ if kind:+ append(+ (+ kind,+ bisect_left(newlines, start_byte) + 1,+ bisect_left(newlines, end_byte) + 1,+ start_byte,+ end_byte,+ name,+ )+ )+ elif cursor.goto_first_child():+ continue+ while not cursor.goto_next_sibling():+ if not cursor.goto_parent():+ return outdef tokenize_tree(root: Node, source: bytes, skip_kinds: frozenset[str] = frozenset()) -> list[Token]:- """Walk *root* and return the flattened token list.-- Subtrees whose node kind is in *skip_kinds* (import statements) and- re-exports are dropped whole. Iterative DFS — uses a stack rather than- recursion so very deep files don't blow the recursion limit.- """- out: list[Token] = []- stack: list[Node] = [root]- while stack:- node = stack.pop()- if _is_skippable(node) or node.type in skip_kinds or _is_reexport(node):- continue- if node.child_count == 0:- tok = _tokenize_leaf(node, source)- if tok is not None:- out.append(tok)- continue- # Push in reverse so we visit in source order on the next pop.- for child in reversed(node.children):- stack.append(child)- return out---def _tokenize_leaf(node: Node, source: bytes) -> Token | None:- name = ""- if node.type in _IDENTIFIER_KINDS:- kind = "ID"- name = sys.intern(source[node.start_byte : node.end_byte].decode("utf-8", errors="replace"))- elif node.type in _LITERAL_KINDS:- kind = "LIT"- else:- # Use the raw token text for operators / keywords / punctuation.- text = source[node.start_byte : node.end_byte]- if not text.strip():- return None- try:- kind = text.decode("utf-8", errors="replace")- except Exception:- return None- return Token(- # A clone scan holds one Token per leaf and later keeps every kind- # through collision verification. Operators and keywords otherwise- # create one equal Python string per occurrence; interning turns that- # repo-sized string population into one object per distinct spelling.- kind=sys.intern(kind),- start_line=node.start_point[0] + 1,- end_line=node.end_point[0] + 1,- start_byte=node.start_byte,- end_byte=node.end_byte,- name=name,- )+ """Walk *root* and return the flattened token list (see ``token_rows``)."""+ return [Token(*row) for row in token_rows(root, source, skip_kinds)]def tokenize_file(language: str, source: bytes, path: str | None = None) -> list[Token]:- """Parse *source* and return its normalized token stream.+ """Parse *source* and return its normalized token stream (see ``tokenize_file_rows``)."""+ return [Token(*row) for row in tokenize_file_rows(language, source, path)]+++def tokenize_file_rows(language: str, source: bytes, path: str | None = None) -> list[TokenRow]:+ """Parse *source* and return its normalized token rows.Returns an empty list when the language is unsupported or parsingfails — callers treat that as "no clone candidates from this file".@@ -211,4 +228,4 @@ def tokenize_file(language: str, source: bytes, path: str | None = None) -> listtree = parser.parse(source)except Exception:return []- return tokenize_tree(tree.root_node, source, import_node_kinds(language))+ return token_rows(tree.root_node, source, import_node_kinds(language))diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.pyindex 9f91affd2..0a0f69975 100644--- a/tests/unit/health/test_duplication.py+++ b/tests/unit/health/test_duplication.py@@ -426,6 +426,26 @@ def test_tokenize_file_keeps_raw_identifier_names():assert [t.name for t in toks] == ["total", "", "count", "", ""]+def test_tokenize_file_lines_match_tree_sitter_points():+ from tree_sitter import Parser++ from repowise.core.ingestion.parser import _get_language++ source = b'x = 1\r\n\r\ns = """a\r\nb\n"""\nif x:\n y = (s,\n x)\n'+ tree = Parser(_get_language("python")).parse(source)+ leaves = []+ stack = [tree.root_node]+ while stack:+ node = stack.pop()+ if node.child_count == 0 and source[node.start_byte : node.end_byte].strip():+ leaves.append((node.start_point[0] + 1, node.end_point[0] + 1))+ stack.extend(reversed(node.children))++ toks = tokenize_file("python", source)+ assert [(t.start_line, t.end_line) for t in toks] == leaves+ assert (3, 5) in [(t.start_line, t.end_line) for t in toks] # the multi-line string++def test_detect_clones_ignores_identical_import_blocks(tmp_path: Path):imports = "".join(f"from pkg.mod{i} import name{i}, other{i}\n" for i in range(12))a = _write(tmp_path, "a.py", imports + "def f():\n return 1\n")