Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(health): tokenize and verify clones without per-token objects (#2878)

Raghav Chamadiya99424cefac3f
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
index 6e5d1db47..0daf3c428 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
@@ -36,8 +36,8 @@ from repowise.core.cancellation import check_cancelled
from repowise.core.co_change import parse_partners
from .limits import DuplicationDiagnostics, DuplicationLimits, looks_minified
-from .rabin_karp import WindowHash, index_by_hash, rolling_hashes
-from .tokenizer import tokenize_file
+from .rabin_karp import WindowHash, WindowRow, index_by_hash, window_rows
+from .tokenizer import tokenize_file_rows
log = structlog.get_logger(__name__)
@@ -136,33 +136,12 @@ def _is_noise(kind: str) -> bool:
return kind == "LIT" or not any(ch.isalnum() for ch in kind)
-def _informative_windows(
- windows: list[WindowHash], kinds: list[str], window: int
-) -> list[WindowHash]:
+def _informative_windows(windows: list[WindowRow], kinds: list[str], window: int) -> list[WindowRow]:
"""Drop windows that are more than ``_MAX_NOISE_SHARE`` literals/punctuation."""
- noise = list(accumulate((_is_noise(k) for k in kinds), initial=0))
+ noisy = {kind: _is_noise(kind) for kind in set(kinds)}
+ noise = list(accumulate((noisy[k] for k in kinds), initial=0))
limit = _MAX_NOISE_SHARE * window
- return [w for w in windows if noise[w.start_index + window] - noise[w.start_index] <= limit]
-
-
-def _windows_match(a: _Stream, b: _Stream, a_start: int, b_start: int, window: int) -> bool:
- """Confirm a hash match: equal token kinds, then shared identifier names.
-
- Operates on the per-file streams so cached token streams round-trip
- without rebuilding Token objects.
- """
- a_kinds, a_names = a
- b_kinds, b_names = b
- if a_start + window > len(a_kinds) or b_start + window > len(b_kinds):
- return False
- if a_kinds[a_start : a_start + window] != b_kinds[b_start : b_start + window]:
- return False
- a_set = set(a_names[a_start : a_start + window])
- b_set = set(b_names[b_start : b_start + window])
- a_set.discard("")
- b_set.discard("")
- union = len(a_set | b_set)
- return union > 0 and len(a_set & b_set) >= _MIN_NAME_JACCARD * union
+ return [w for w in windows if noise[w[1] + window] - noise[w[1]] <= limit]
def _merge_adjacent_pairs(raw: list[ClonePair]) -> list[ClonePair]:
@@ -396,35 +375,26 @@ def _collect_windows(
if cached is not None:
kinds, names, nloc, window_tuples = cached
- windows = [
- WindowHash(
- file_path=path,
- hash_value=h,
- start_index=si,
- start_line=sl,
- end_line=el,
- )
- for h, si, sl, el in window_tuples
- ]
else:
- toks = tokenize_file(language, source, path)
- if len(toks) > limits.max_tokens_per_file:
+ rows = tokenize_file_rows(language, source, path)
+ if len(rows) > limits.max_tokens_per_file:
diag.skipped_token_cap += 1
continue
- kinds = [t.kind for t in toks]
- names = [t.name for t in toks]
+ kinds, start_lines, end_lines, names = [], [], [], []
+ if rows:
+ kinds, start_lines, end_lines, _, _, names = map(list, zip(*rows, strict=True))
+ del rows
nloc = _nloc(source)
- windows = _informative_windows(
- rolling_hashes(path, toks, window_tokens), kinds, window_tokens
+ window_tuples = _informative_windows(
+ window_rows(kinds, start_lines, end_lines, window_tokens), kinds, window_tokens
)
+ del start_lines, end_lines
if cache is not None:
- cache.put(
- content_hash,
- kinds,
- names,
- nloc,
- [(w.hash_value, w.start_index, w.start_line, w.end_line) for w in windows],
- )
+ cache.put(content_hash, kinds, names, nloc, window_tuples)
+ windows = [
+ WindowHash(file_path=path, hash_value=h, start_index=si, start_line=sl, end_line=el)
+ for h, si, sl, el in window_tuples
+ ]
if len(kinds) < window_tokens:
continue
@@ -468,7 +438,6 @@ def _pairs_from_buckets(
return the pairs found so far rather than spinning indefinitely.
"""
raw_pairs: list[ClonePair] = []
- seen: set[tuple[str, int, str, int]] = set()
deadline = (time.monotonic() + limits.time_budget_secs) if limits.time_budget_secs else None
for i, windows in enumerate(bucket.values()):
@@ -483,7 +452,7 @@ def _pairs_from_buckets(
if deadline is not None and (i & 0x3FF) == 0 and time.monotonic() > deadline:
diag.timed_out = True
break
- _verify_bucket(windows, per_file_streams, window_tokens, seen, raw_pairs)
+ _verify_bucket(windows, per_file_streams, window_tokens, raw_pairs)
return raw_pairs
@@ -492,32 +461,45 @@ def _verify_bucket(
windows: list[WindowHash],
per_file_streams: dict[str, _Stream],
window_tokens: int,
- seen: set[tuple[str, int, str, int]],
out: list[ClonePair],
) -> None:
"""Confirm every unordered pair in one (bounded) hash bucket.
- Hash equality is necessary but not sufficient — ``_windows_match``
- rejects collisions and renamed look-alikes.
+ Hash equality is necessary but not sufficient: a pair must have equal
+ token kinds (rules out collisions) and share at least
+ ``_MIN_NAME_JACCARD`` of their raw identifier names (rules out renamed
+ look-alikes). Each window's kind slice and name set is built once per
+ bucket rather than once per pair. A window has one hash, so it sits in
+ exactly one bucket and no pair is seen twice.
"""
+ kind_slices: list[list[str] | None] = []
+ name_sets: list[set[str] | None] = []
+ for w in windows:
+ kinds = per_file_streams[w.file_path][0]
+ end = w.start_index + window_tokens
+ kind_slices.append(kinds[w.start_index : end] if end <= len(kinds) else None)
+ name_sets.append(None)
+
for i in range(len(windows)):
+ a_kinds = kind_slices[i]
+ if a_kinds is None:
+ continue
for j in range(i + 1, len(windows)):
+ if kind_slices[j] != a_kinds:
+ continue
+ a_names = name_sets[i]
+ if a_names is None:
+ a_names = name_sets[i] = _window_names(windows[i], per_file_streams, window_tokens)
+ b_names = name_sets[j]
+ if b_names is None:
+ b_names = name_sets[j] = _window_names(windows[j], per_file_streams, window_tokens)
+ union = len(a_names | b_names)
+ if not (union > 0 and len(a_names & b_names) >= _MIN_NAME_JACCARD * union):
+ continue
a, b = windows[i], windows[j]
# Canonicalize so (file_a, file_b) ordering is stable.
if (a.file_path, a.start_index) > (b.file_path, b.start_index):
a, b = b, a
- key = (a.file_path, a.start_index, b.file_path, b.start_index)
- if key in seen:
- continue
- seen.add(key)
- if not _windows_match(
- per_file_streams[a.file_path],
- per_file_streams[b.file_path],
- a.start_index,
- b.start_index,
- window_tokens,
- ):
- continue
out.append(
ClonePair(
file_a=a.file_path,
@@ -531,6 +513,13 @@ def _verify_bucket(
)
+def _window_names(w: WindowHash, per_file_streams: dict[str, _Stream], window_tokens: int) -> set[str]:
+ """The raw identifier names inside one window."""
+ names = set(per_file_streams[w.file_path][1][w.start_index : w.start_index + window_tokens])
+ names.discard("")
+ return names
+
+
# ---------------------------------------------------------------------------
# Incremental splice (update path)
# ---------------------------------------------------------------------------
@@ -775,12 +764,9 @@ def _bucket_contributions(
The degenerate-bucket cap applies to each bucket's full membership
(rows here cover it: every window with a touched hash was gathered),
so a bucket crossing the cap in either direction contributes pairs on
- exactly one side of the splice. The shared ``seen`` set mirrors the
- full pipeline's; (file, start_index) pairs are unique to one bucket,
- so per-run scoping is equivalent.
+ exactly one side of the splice.
"""
out: list[ClonePair] = []
- seen: set[tuple[str, int, str, int]] = set()
for rows in rows_by_hash.values():
if len(rows) < 2:
continue
@@ -789,7 +775,7 @@ def _bucket_contributions(
diag.degenerate_buckets += 1
continue
check_cancelled()
- _verify_bucket(rows, streams_map, window_tokens, seen, out)
+ _verify_bucket(rows, streams_map, window_tokens, out)
return out
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py
index a849443d8..6fd8f2ba4 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py
@@ -51,6 +51,46 @@ class WindowHash:
end_line: int
+# One window as a plain tuple: (hash_value, start_index, start_line, end_line).
+WindowRow = tuple[int, int, int, int]
+
+
+def window_rows(
+ kinds: list[str],
+ start_lines: list[int],
+ end_lines: list[int],
+ window: int,
+) -> list[WindowRow]:
+ """Rolling hashes for every window of length *window*, as plain tuples.
+
+ The parallel lists describe one token stream. Returns an empty list when
+ the stream is shorter than one window. A repository scan rolls millions
+ of windows, so the per-kind hash is computed once per distinct spelling
+ and no object is built per window.
+ """
+ n = len(kinds)
+ if n < window or window <= 0:
+ return []
+
+ base, modulus = _BASE, _MODULUS
+ base_pow = pow(base, window - 1, modulus)
+ per_kind = {kind: _token_hash(kind) for kind in set(kinds)}
+ kind_hashes = [per_kind[kind] for kind in kinds]
+
+ h = 0
+ for i in range(window):
+ h = (h * base + kind_hashes[i]) % modulus
+
+ out: list[WindowRow] = [(h, 0, start_lines[0], end_lines[window - 1])]
+ append = out.append
+ # Python's % yields a non-negative result for a positive modulus.
+ for i in range(1, n - window + 1):
+ last = i + window - 1
+ h = ((h - kind_hashes[i - 1] * base_pow) * base + kind_hashes[last]) % modulus
+ append((h, i, start_lines[i], end_lines[last]))
+ return out
+
+
def rolling_hashes(
file_path: str,
tokens: list[Token],
@@ -62,43 +102,16 @@ def rolling_hashes(
window size — callers filter clone candidates by minimum size
upstream so the empty-result case is well-defined.
"""
- n = len(tokens)
- if n < window or window <= 0:
- return []
-
- base_pow = pow(_BASE, window - 1, _MODULUS)
- kind_hashes = [_token_hash(t.kind) for t in tokens]
-
- h = 0
- for i in range(window):
- h = (h * _BASE + kind_hashes[i]) % _MODULUS
-
- out: list[WindowHash] = []
- out.append(
- WindowHash(
- file_path=file_path,
- hash_value=h,
- start_index=0,
- start_line=tokens[0].start_line,
- end_line=tokens[window - 1].end_line,
- )
+ rows = window_rows(
+ [t.kind for t in tokens],
+ [t.start_line for t in tokens],
+ [t.end_line for t in tokens],
+ window,
)
- for i in range(1, n - window + 1):
- h = ((h - kind_hashes[i - 1] * base_pow) * _BASE + kind_hashes[i + window - 1]) % _MODULUS
- # Python's % already yields a non-negative result for a positive
- # modulus, but we keep the guard for clarity.
- if h < 0:
- h += _MODULUS
- out.append(
- WindowHash(
- file_path=file_path,
- hash_value=h,
- start_index=i,
- start_line=tokens[i].start_line,
- end_line=tokens[i + window - 1].end_line,
- )
- )
- return out
+ return [
+ WindowHash(file_path=file_path, hash_value=h, start_index=si, start_line=sl, end_line=el)
+ for h, si, sl, el in rows
+ ]
def index_by_hash(hashes: Iterable[WindowHash]) -> dict[int, list[WindowHash]]:
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py
index 1ea0a96e8..90bbd492e 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py
@@ -25,7 +25,9 @@ preserve structure. Each identifier token also keeps its raw text in
from __future__ import annotations
+import re
import sys
+from bisect import bisect_left
from dataclasses import dataclass
from typing import TYPE_CHECKING
@@ -113,77 +115,92 @@ def import_node_kinds(language: str) -> frozenset[str]:
return frozenset(config.import_node_types) - _NOT_IMPORT_KINDS
-def _is_skippable(node: Node) -> bool:
- if node.type in _COMMENT_KINDS:
- return True
- # Tree-sitter exposes ERROR / MISSING for parse errors - drop them
- # so a single broken file can't pollute the hash stream.
- return bool(getattr(node, "has_error", False) and node.child_count == 0)
+_NEWLINE = re.compile(rb"\n")
+# One leaf token in ``Token`` field order: (kind, start_line, end_line,
+# start_byte, end_byte, name).
+TokenRow = tuple[str, int, int, int, int, str]
-def _is_reexport(node: Node) -> bool:
- # JS/TS ``export { a } from "./b"``: an import in all but name.
- return node.type == "export_statement" and node.child_by_field_name("source") is not None
+
+def token_rows(
+ root: Node, source: bytes, skip_kinds: frozenset[str] = frozenset()
+) -> list[TokenRow]:
+ """Walk *root* and return one ``TokenRow`` per kept leaf, in source order.
+
+ Dropped: comments, parse-error leaves, whitespace-only leaves, and whole
+ subtrees whose kind is in *skip_kinds* (import statements) or that are
+ re-exports. A clone scan walks every node of every file, so this is a
+ cursor walk that reads each node's attributes once and emits plain
+ tuples; ``tokenize_tree`` wraps them in ``Token`` for callers that want
+ objects.
+ """
+ out: list[TokenRow] = []
+ append = out.append
+ intern = sys.intern
+ # A node's row is the number of newlines before its byte offset, which
+ # a C bisect answers faster than building tree-sitter Point objects.
+ newlines = [m.start() for m in _NEWLINE.finditer(source)]
+ cursor = root.walk()
+ while True:
+ node = cursor.node
+ ntype = node.type
+ child_count = node.child_count
+ if not (
+ ntype in _COMMENT_KINDS
+ # Tree-sitter exposes ERROR / MISSING for parse errors - drop them
+ # so a single broken file can't pollute the hash stream.
+ or (child_count == 0 and node.has_error)
+ or ntype in skip_kinds
+ # JS/TS ``export { a } from "./b"``: an import in all but name.
+ or (ntype == "export_statement" and node.child_by_field_name("source") is not None)
+ ):
+ if child_count == 0:
+ start_byte = node.start_byte
+ end_byte = node.end_byte
+ name = ""
+ if ntype in _IDENTIFIER_KINDS:
+ kind = "ID"
+ name = intern(source[start_byte:end_byte].decode("utf-8", errors="replace"))
+ elif ntype in _LITERAL_KINDS:
+ kind = "LIT"
+ else:
+ # Raw token text for operators / keywords / punctuation.
+ # Interned: a repo-sized population of equal spellings
+ # becomes one object per distinct spelling.
+ text = source[start_byte:end_byte]
+ kind = intern(text.decode("utf-8", errors="replace")) if text.strip() else ""
+ if kind:
+ append(
+ (
+ kind,
+ bisect_left(newlines, start_byte) + 1,
+ bisect_left(newlines, end_byte) + 1,
+ start_byte,
+ end_byte,
+ name,
+ )
+ )
+ elif cursor.goto_first_child():
+ continue
+ while not cursor.goto_next_sibling():
+ if not cursor.goto_parent():
+ return out
def tokenize_tree(
root: Node, source: bytes, skip_kinds: frozenset[str] = frozenset()
) -> list[Token]:
- """Walk *root* and return the flattened token list.
-
- Subtrees whose node kind is in *skip_kinds* (import statements) and
- re-exports are dropped whole. Iterative DFS — uses a stack rather than
- recursion so very deep files don't blow the recursion limit.
- """
- out: list[Token] = []
- stack: list[Node] = [root]
- while stack:
- node = stack.pop()
- if _is_skippable(node) or node.type in skip_kinds or _is_reexport(node):
- continue
- if node.child_count == 0:
- tok = _tokenize_leaf(node, source)
- if tok is not None:
- out.append(tok)
- continue
- # Push in reverse so we visit in source order on the next pop.
- for child in reversed(node.children):
- stack.append(child)
- return out
-
-
-def _tokenize_leaf(node: Node, source: bytes) -> Token | None:
- name = ""
- if node.type in _IDENTIFIER_KINDS:
- kind = "ID"
- name = sys.intern(source[node.start_byte : node.end_byte].decode("utf-8", errors="replace"))
- elif node.type in _LITERAL_KINDS:
- kind = "LIT"
- else:
- # Use the raw token text for operators / keywords / punctuation.
- text = source[node.start_byte : node.end_byte]
- if not text.strip():
- return None
- try:
- kind = text.decode("utf-8", errors="replace")
- except Exception:
- return None
- return Token(
- # A clone scan holds one Token per leaf and later keeps every kind
- # through collision verification. Operators and keywords otherwise
- # create one equal Python string per occurrence; interning turns that
- # repo-sized string population into one object per distinct spelling.
- kind=sys.intern(kind),
- start_line=node.start_point[0] + 1,
- end_line=node.end_point[0] + 1,
- start_byte=node.start_byte,
- end_byte=node.end_byte,
- name=name,
- )
+ """Walk *root* and return the flattened token list (see ``token_rows``)."""
+ return [Token(*row) for row in token_rows(root, source, skip_kinds)]
def tokenize_file(language: str, source: bytes, path: str | None = None) -> list[Token]:
- """Parse *source* and return its normalized token stream.
+ """Parse *source* and return its normalized token stream (see ``tokenize_file_rows``)."""
+ return [Token(*row) for row in tokenize_file_rows(language, source, path)]
+
+
+def tokenize_file_rows(language: str, source: bytes, path: str | None = None) -> list[TokenRow]:
+ """Parse *source* and return its normalized token rows.
Returns an empty list when the language is unsupported or parsing
fails — callers treat that as "no clone candidates from this file".
@@ -211,4 +228,4 @@ def tokenize_file(language: str, source: bytes, path: str | None = None) -> list
tree = parser.parse(source)
except Exception:
return []
- return tokenize_tree(tree.root_node, source, import_node_kinds(language))
+ return token_rows(tree.root_node, source, import_node_kinds(language))
diff --git a/tests/unit/health/test_duplication.py b/tests/unit/health/test_duplication.py
index 9f91affd2..0a0f69975 100644
--- a/tests/unit/health/test_duplication.py
+++ b/tests/unit/health/test_duplication.py
@@ -426,6 +426,26 @@ def test_tokenize_file_keeps_raw_identifier_names():
assert [t.name for t in toks] == ["total", "", "count", "", ""]
+def test_tokenize_file_lines_match_tree_sitter_points():
+ from tree_sitter import Parser
+
+ from repowise.core.ingestion.parser import _get_language
+
+ source = b'x = 1\r\n\r\ns = """a\r\nb\n"""\nif x:\n y = (s,\n x)\n'
+ tree = Parser(_get_language("python")).parse(source)
+ leaves = []
+ stack = [tree.root_node]
+ while stack:
+ node = stack.pop()
+ if node.child_count == 0 and source[node.start_byte : node.end_byte].strip():
+ leaves.append((node.start_point[0] + 1, node.end_point[0] + 1))
+ stack.extend(reversed(node.children))
+
+ toks = tokenize_file("python", source)
+ assert [(t.start_line, t.end_line) for t in toks] == leaves
+ assert (3, 5) in [(t.start_line, t.end_line) for t in toks] # the multi-line string
+
+
def test_detect_clones_ignores_identical_import_blocks(tmp_path: Path):
imports = "".join(f"from pkg.mod{i} import name{i}, other{i}\n" for i in range(12))
a = _write(tmp_path, "a.py", imports + "def f():\n return 1\n")