Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(index): bound repository-scale native memory (#1778)
Raghav Chamadiyae989355c6153
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.pyindex da11d9ca4..92485477e 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py@@ -44,7 +44,7 @@ DEFAULT_WINDOW_TOKENS = 50DEFAULT_MIN_LINES = 6-@dataclass+@dataclass(slots=True)class ClonePair:"""One verified clone region between two files (or two regions inthe same file)."""@@ -253,6 +253,11 @@ def detect_clones()if cache is not None:cache.save()+ # The persisted form holds another representation of every token+ # stream and window. Collision verification only needs the returned+ # per-file kinds, so release the cache's owning dictionaries before+ # allocating the repo-wide hash index and clone-pair lists.+ cache.release_memory()log.debug("duplication_token_cache",hits=cache.hits,@@ -262,6 +267,9 @@ def detect_clones(return DuplicationReport(diagnostics=diag.as_log_fields())bucket = index_by_hash(all_windows)+ # Every WindowHash is now owned by a bucket. Keeping the flat list too+ # adds millions of redundant references at the scan's high-water mark.+ del all_windowsraw_pairs = _pairs_from_buckets(bucket, per_file_kinds, window_tokens, lim, diag)if cache is not None and cache_dir is not None:@@ -277,6 +285,11 @@ def detect_clones(all_paths,)+ # Neither structure participates in merge/finalize/aggregation. Drop+ # them before those stages allocate their output lists so freed arenas+ # can be reused inside this phase rather than growing the process again.+ del bucket, per_file_kinds+final = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)diff --git a/packages/core/src/repowise/core/analysis/health/duplication/isolation.py b/packages/core/src/repowise/core/analysis/health/duplication/isolation.pynew file mode 100644index 000000000..80d318a8e--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/duplication/isolation.py@@ -0,0 +1,179 @@+"""Process isolation for repository-scale duplicate detection.++The full scan creates millions of short-lived Python objects. CPython frees+them when the scan finishes, but allocator fragmentation can keep gigabytes of+pages committed in the indexing process. Running only large scans in a child+lets the operating system reclaim those pages before wiki generation starts.+"""++from __future__ import annotations++import multiprocessing+import traceback+from dataclasses import dataclass+from pathlib import Path+from typing import Any++from repowise.core.cancellation import check_cancelled++from ..source_reader import disk_source_reader+from .detector import (+ DEFAULT_MIN_LINES,+ DEFAULT_WINDOW_TOKENS,+ DuplicationReport,+ detect_clones,+)+from .limits import DuplicationLimits++# Process startup is needless overhead for the small repositories common in+# unit tests and editor workflows. Repositories above this size are where the+# detector's transient arenas become material compared with the base process.+_ISOLATION_FILE_THRESHOLD = 1_000+++@dataclass(frozen=True, slots=True)+class _FileInfo:+ path: str+ abs_path: str+ language: str+++@dataclass(frozen=True, slots=True)+class _ParsedFile:+ file_info: _FileInfo+++def _worker(+ sender: Any,+ files: list[tuple[str, str, str]],+ git_meta_map: dict[str, dict[str, Any]],+ window_tokens: int,+ min_lines: int,+ limits: DuplicationLimits | None,+ cache_dir: Path | None,+ changed_files: set[str] | None,+) -> None:+ """Run in a fresh interpreter and return only the compact report."""+ try:+ parsed_files = [_ParsedFile(_FileInfo(*file_info)) for file_info in files]+ report = detect_clones(+ parsed_files,+ git_meta_map,+ window_tokens=window_tokens,+ min_lines=min_lines,+ limits=limits,+ cache_dir=cache_dir,+ changed_files=changed_files,+ )+ sender.send((True, report))+ except BaseException:+ sender.send((False, traceback.format_exc()))+ finally:+ sender.close()+++def detect_clones_with_isolation(+ parsed_files: list[Any],+ git_meta_map: dict[str, dict[str, Any]] | None = None,+ *,+ window_tokens: int = DEFAULT_WINDOW_TOKENS,+ min_lines: int = DEFAULT_MIN_LINES,+ limits: DuplicationLimits | None = None,+ cache_dir: Path | None = None,+ changed_files: set[str] | None = None,+ source_reader: Any | None = None,+ isolation_file_threshold: int = _ISOLATION_FILE_THRESHOLD,+) -> DuplicationReport:+ """Detect clones, isolating repository-scale transient allocations.++ Only the three file-info strings used by the detector cross the process+ boundary. The full parsed AST/model graph remains in the parent, and the+ child returns the comparatively small final report.++ Isolation is taken only for a working-tree read, which is the case it was+ measured on and the only one whose reader survives a spawn. Any other+ *source_reader* runs the detector in process; see the comment below.+ """+ # A reader that is not the working tree cannot cross the spawn boundary.+ # ``MappingSourceReader`` holds every file's bytes, so shipping it would+ # send the exact memory this isolation exists to avoid; and letting the+ # worker fall back to disk would report working-tree findings as the+ # revision's, which that reader documents as the thing it must not do.+ # Those callers (the base side of a diff, a historical commit) analyse a+ # changed-file set, not a repository, so the bound is not what they need.+ reader_is_working_tree = source_reader is None or source_reader is disk_source_reader++ if len(parsed_files) < isolation_file_threshold or not reader_is_working_tree:+ return detect_clones(+ parsed_files,+ git_meta_map,+ window_tokens=window_tokens,+ min_lines=min_lines,+ limits=limits,+ cache_dir=cache_dir,+ changed_files=changed_files,+ source_reader=source_reader,+ )++ files = [+ (pf.file_info.path, str(pf.file_info.abs_path), pf.file_info.language)+ for pf in parsed_files+ ]+ # Clone weighting only reads this field. Avoid serializing the much larger+ # Git metadata records into the worker.+ compact_git_meta = {+ path: {"co_change_partners_json": meta.get("co_change_partners_json")}+ for path, meta in (git_meta_map or {}).items()+ if meta.get("co_change_partners_json")+ }++ context = multiprocessing.get_context("spawn")+ receiver, sender = context.Pipe(duplex=False)+ process = context.Process(+ target=_worker,+ args=(+ sender,+ files,+ compact_git_meta,+ window_tokens,+ min_lines,+ limits,+ cache_dir,+ changed_files,+ ),+ name="repowise-duplication",+ )+ try:+ process.start()+ except BaseException:+ receiver.close()+ sender.close()+ raise+ sender.close()++ try:+ while not receiver.poll(0.1):+ check_cancelled()+ if not process.is_alive():+ process.join()+ raise RuntimeError(+ f"duplication worker exited without a result (exit code {process.exitcode})"+ )+ try:+ succeeded, result = receiver.recv()+ except EOFError as exc:+ raise RuntimeError(+ f"duplication worker closed without a result (exit code {process.exitcode})"+ ) from exc+ process.join()+ if not succeeded:+ raise RuntimeError(f"duplication worker failed:\n{result}")+ return result+ finally:+ receiver.close()+ if process.is_alive():+ process.terminate()+ process.join(timeout=5)+ if process.is_alive():+ process.kill()+ process.join()diff --git a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.pyindex 7bbd42f0a..a849443d8 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py@@ -40,7 +40,7 @@ def _token_hash(tok_kind: str) -> int:return h-@dataclass(frozen=True)+@dataclass(frozen=True, slots=True)class WindowHash:"""One rolling-hash window with origin file + line span."""diff --git a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.pyindex 968e23ec2..b88613bff 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py@@ -73,6 +73,16 @@ class DuplicationTokenCache:except Exception as exc:log.debug("duplication_cache_save_failed", error=str(exc))+ def release_memory(self) -> None:+ """Drop cached representations after a full scan has persisted them.++ Callers retain the kind lists they still need for collision+ verification. The cache-owned window tuples are otherwise a second+ repo-sized copy of the live ``WindowHash`` population.+ """+ self._entries.clear()+ self._fresh.clear()+# -- access ------------------------------------------------------------def get(@@ -118,8 +128,11 @@ class DuplicationTokenCache:nloc: int,windows: list[tuple[int, int, int, int]],) -> None:- # Interned kinds keep the pickle compact (each distinct kind is- # memoized once by identity) and make later equality checks cheap.- entry = ([sys.intern(k) for k in kinds], nloc, windows)+ # Intern in place so the detector and cache share one kind list rather+ # than holding two repo-sized lists of references during a full scan.+ # Identity reuse also keeps the pickle compact and comparisons cheap.+ for index, kind in enumerate(kinds):+ kinds[index] = sys.intern(kind)+ entry = (kinds, nloc, windows)self._entries[content_hash] = entryself._fresh[content_hash] = entrydiff --git a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.pyindex 7473ae5d4..4501c0aeb 100644--- a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py+++ b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py@@ -22,6 +22,7 @@ preserve structure.from __future__ import annotations+import sysfrom dataclasses import dataclassfrom typing import TYPE_CHECKING@@ -82,7 +83,7 @@ _LITERAL_KINDS = frozenset()-@dataclass(frozen=True)+@dataclass(frozen=True, slots=True)class Token:"""One AST token with the source location of its origin node."""@@ -139,7 +140,11 @@ def _tokenize_leaf(node: Node, source: bytes) -> Token | None:except Exception:return Nonereturn Token(- kind=kind,+ # A clone scan holds one Token per leaf and later keeps every kind+ # through collision verification. Operators and keywords otherwise+ # create one equal Python string per occurrence; interning turns that+ # repo-sized string population into one object per distinct spelling.+ kind=sys.intern(kind),start_line=node.start_point[0] + 1,end_line=node.end_point[0] + 1,start_byte=node.start_byte,diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.pyindex 618982d2c..d34a27cb2 100644--- a/packages/core/src/repowise/core/analysis/health/engine.py+++ b/packages/core/src/repowise/core/analysis/health/engine.py@@ -43,7 +43,8 @@ from .biomarkers import FileContext, detect_allfrom .complexity import FileComplexity, FunctionComplexity, walk_filefrom .coverage import is_test_file as _coverage_is_test_filefrom .dataflow import FileDataflowCache-from .duplication import DuplicationReport, detect_clones+from .duplication import DuplicationReport+from .duplication.isolation import detect_clones_with_isolation as detect_clonesfrom .models import HealthFileMetricData, HealthFindingData, HealthReport, Severityfrom .perf import (CallGraphIndex,diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.pyindex 2775aaec3..21d5ce742 100644--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py@@ -9,6 +9,11 @@ from ._base import STORED_SNIPPET_CHARS, VectorStore, iter_embed_chunks__all__ = ["STORED_SNIPPET_CHARS", "LanceDBVectorStore"]+# DataFusion expands every literal in a large ``IN`` filter into native query+# state. A several-thousand-path generation level can otherwise commit+# gigabytes before returning even though the selected result is small.+_SUMMARY_PATH_BATCH_SIZE = 100+# ``STORED_SNIPPET_CHARS`` — how much of a page's content each row keeps — is# defined with the embed recipe and re-exported here so the historical import# path keeps working.@@ -356,10 +361,11 @@ class LanceDBVectorStore(VectorStore):return {"summary": summary, "key_exports": []}async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:- """One ``IN``-filtered scan instead of one filtered query per path.+ """Read summaries with bounded ``IN``-filtered scans.Mirrors the single-path semantics (first row per path wins, empty- summaries dropped, ``key_exports`` not stored in this schema).+ summaries dropped, ``key_exports`` not stored in this schema). Batches+ bound DataFusion's native query-plan allocation for large levels."""if not paths:return {}@@ -367,22 +373,25 @@ class LanceDBVectorStore(VectorStore):if self._table is None:return {}- try:- rows = (- await self._table.query() # type: ignore[union-attr]- .where(_paths_in_filter(paths))- .select(["target_path", "content_snippet"])- .to_list()- )- except Exception:- return {}-out: dict[str, dict] = {}- for r in rows:- tp = str(r.get("target_path") or "")- if not tp or tp in out:- continue- summary = str(r.get("content_snippet") or "")[:_SNIPPET_LEN]- if summary:- out[tp] = {"summary": summary, "key_exports": []}+ unique_paths = list(dict.fromkeys(paths))+ for index in range(0, len(unique_paths), _SUMMARY_PATH_BATCH_SIZE):+ batch = unique_paths[index : index + _SUMMARY_PATH_BATCH_SIZE]+ try:+ rows = (+ await self._table.query() # type: ignore[union-attr]+ .where(_paths_in_filter(batch))+ .select(["target_path", "content_snippet"])+ .to_list()+ )+ except Exception:+ return {}++ for row in rows:+ target_path = str(row.get("target_path") or "")+ if not target_path or target_path in out:+ continue+ summary = str(row.get("content_snippet") or "")[:_SNIPPET_LEN]+ if summary:+ out[target_path] = {"summary": summary, "key_exports": []}return outdiff --git a/tests/unit/health/test_duplication_cache.py b/tests/unit/health/test_duplication_cache.pyindex c1d950c3f..9337e03a6 100644--- a/tests/unit/health/test_duplication_cache.py+++ b/tests/unit/health/test_duplication_cache.py@@ -2,14 +2,18 @@from __future__ import annotations+import sysfrom pathlib import Pathfrom types import SimpleNamespacefrom repowise.core.analysis.health.duplication import detect_clones+from repowise.core.analysis.health.duplication.detector import ClonePair+from repowise.core.analysis.health.duplication.rabin_karp import WindowHashfrom repowise.core.analysis.health.duplication.token_cache import (_CACHE_FILENAME,DuplicationTokenCache,)+from repowise.core.analysis.health.duplication.tokenizer import Tokendef _pf(path: str, abs_path: str) -> SimpleNamespace:@@ -113,3 +117,32 @@ def test_corrupt_cache_degrades_to_full_run(tmp_path: Path):baseline = detect_clones(parsed, window_tokens=20, min_lines=4)report = detect_clones(parsed, window_tokens=20, min_lines=4, cache_dir=cache_dir)assert _report_key(report) == _report_key(baseline)+++def test_cache_shares_interned_kinds_and_can_release_memory(tmp_path: Path):+ cache = DuplicationTokenCache(tmp_path, 20)+ kinds = [bytearray(b"identifier").decode() for _ in range(2)]++ cache.put("digest", kinds, 1, [(1, 0, 1, 1)])+ cached = cache.get("digest")++ assert cached is not None+ cached_kinds, _, _ = cached+ assert cached_kinds is kinds+ assert all(kind is sys.intern(kind) for kind in kinds)++ cache.release_memory()+ assert cache.get("digest") is None+ # Releasing the owning dictionaries must not invalidate the detector's+ # reference used for collision verification.+ assert kinds == ["identifier", "identifier"]+++def test_high_volume_duplication_records_are_slotted():+ token = Token("ID", 1, 1, 0, 1)+ window = WindowHash("a.py", 1, 0, 1, 1)+ pair = ClonePair("a.py", "b.py", 1, 2, 1, 2, 20)++ assert not hasattr(token, "__dict__")+ assert not hasattr(window, "__dict__")+ assert not hasattr(pair, "__dict__")diff --git a/tests/unit/health/test_duplication_isolation.py b/tests/unit/health/test_duplication_isolation.pynew file mode 100644index 000000000..76659824c--- /dev/null+++ b/tests/unit/health/test_duplication_isolation.py@@ -0,0 +1,154 @@+"""Large duplication scans release their transient allocator arenas."""++from __future__ import annotations++from pathlib import Path+from types import SimpleNamespace++from repowise.core.analysis.health.duplication import detect_clones+from repowise.core.analysis.health.duplication.isolation import (+ detect_clones_with_isolation,+)+++def _parsed(path: Path) -> SimpleNamespace:+ return SimpleNamespace(+ file_info=SimpleNamespace(+ path=path.name,+ abs_path=str(path),+ language="python",+ )+ )+++def _key(report):+ return sorted(+ (+ pair.file_a,+ pair.file_b,+ pair.a_start_line,+ pair.a_end_line,+ pair.b_start_line,+ pair.b_end_line,+ )+ for pair in report.pairs+ )+++def test_isolated_detection_matches_in_process(tmp_path: Path):+ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))+ first = tmp_path / "a.py"+ second = tmp_path / "b.py"+ first.write_text(body)+ second.write_text(body)+ parsed = [_parsed(first), _parsed(second)]++ expected = detect_clones(parsed, window_tokens=20, min_lines=4)+ actual = detect_clones_with_isolation(+ parsed,+ window_tokens=20,+ min_lines=4,+ cache_dir=tmp_path / ".repowise",+ isolation_file_threshold=0,+ )++ assert _key(actual) == _key(expected)+ assert actual.duplication_pct == expected.duplication_pct+ assert (tmp_path / ".repowise" / "duplication_cache.pkl").exists()+++def test_a_custom_reader_is_honoured_not_ignored(tmp_path: Path):+ """The reader's bytes win over what is on disk.++ The engine routes every health read through a ``SourceReader`` so a pass+ can analyse content that is not the working tree. If the isolation wrapper+ dropped the reader, those callers would silently get working-tree results+ labelled as the revision's.+ """+ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))+ first = tmp_path / "a.py"+ second = tmp_path / "b.py"+ # On disk the two files differ, so a disk read finds no clone at all.+ first.write_text(body)+ second.write_text("unique = 0\n")+ parsed = [_parsed(first), _parsed(second)]++ served = {str(first): body.encode(), str(second): body.encode()}+ report = detect_clones_with_isolation(+ parsed,+ window_tokens=20,+ min_lines=4,+ source_reader=lambda abs_path: served.get(abs_path),+ isolation_file_threshold=0,+ )++ assert report.pairs, "the reader's identical bodies should clone-match"+ assert _key(report) == _key(detect_clones(parsed, window_tokens=20, min_lines=4,+ source_reader=lambda p: served.get(p)))+++def test_a_custom_reader_stays_in_process(tmp_path: Path, monkeypatch):+ """A non-working-tree reader must not be shipped to a spawned worker.++ It cannot be pickled in the general case, and the one such reader in the+ tree holds every file's bytes, which is the memory the isolation exists to+ avoid moving. Falling back to a disk read in the worker would be wrong+ rather than merely slow, so the whole scan stays here.+ """+ import repowise.core.analysis.health.duplication.isolation as isolation++ def _fail(*args, **kwargs):+ raise AssertionError("a custom reader must not reach the spawned worker")++ monkeypatch.setattr(isolation.multiprocessing, "get_context", _fail)++ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))+ first = tmp_path / "a.py"+ second = tmp_path / "b.py"+ first.write_text(body)+ second.write_text(body)+ parsed = [_parsed(first), _parsed(second)]++ served = {str(first): body.encode(), str(second): body.encode()}+ report = isolation.detect_clones_with_isolation(+ parsed,+ window_tokens=20,+ min_lines=4,+ source_reader=lambda abs_path: served.get(abs_path),+ isolation_file_threshold=0,+ )+ assert report.pairs+++def test_the_default_reader_still_isolates(tmp_path: Path, monkeypatch):+ """The working-tree case keeps the spawn, which is where the win was measured."""+ import repowise.core.analysis.health.duplication.isolation as isolation+ from repowise.core.analysis.health.source_reader import disk_source_reader++ spawned: list[bool] = []+ real = isolation.multiprocessing.get_context++ def _record(*args, **kwargs):+ spawned.append(True)+ return real(*args, **kwargs)++ monkeypatch.setattr(isolation.multiprocessing, "get_context", _record)++ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))+ first = tmp_path / "a.py"+ second = tmp_path / "b.py"+ first.write_text(body)+ second.write_text(body)+ parsed = [_parsed(first), _parsed(second)]++ for reader in (None, disk_source_reader):+ spawned.clear()+ report = isolation.detect_clones_with_isolation(+ parsed,+ window_tokens=20,+ min_lines=4,+ source_reader=reader,+ isolation_file_threshold=0,+ )+ assert spawned, f"reader {reader!r} should still take the isolated path"+ assert report.pairsdiff --git a/tests/unit/persistence/test_vector_store_batching.py b/tests/unit/persistence/test_vector_store_batching.pyindex c99caa615..f40225402 100644--- a/tests/unit/persistence/test_vector_store_batching.py+++ b/tests/unit/persistence/test_vector_store_batching.py@@ -7,6 +7,8 @@ import pytestfrom repowise.core.persistence.vector_store.in_memory import InMemoryVectorStorefrom repowise.core.persistence.vector_store.lancedb_store import (+ _SUMMARY_PATH_BATCH_SIZE,+ LanceDBVectorStore,_page_ids_in_filter,_paths_in_filter,)@@ -163,6 +165,44 @@ async def test_lancedb_batch_summaries_roundtrip(tmp_path) -> None:await store.close()+async def test_lancedb_batch_summaries_bounds_in_filter_size(tmp_path) -> None:+ class Query:+ def __init__(self, filters: list[str]) -> None:+ self.filters = filters++ def where(self, expression: str):+ self.filters.append(expression)+ return self++ def select(self, _columns: list[str]):+ return self++ async def to_list(self):+ return []++ class Table:+ def __init__(self) -> None:+ self.filters: list[str] = []++ def query(self):+ return Query(self.filters)++ store = LanceDBVectorStore(str(tmp_path / "lance"), MockEmbedder())+ table = Table()+ store._table = table++ async def connected() -> None:+ return None++ store._ensure_connected = connected # type: ignore[method-assign]+ count = _SUMMARY_PATH_BATCH_SIZE * 2 + 7+ await store.get_page_summaries_by_paths([f"src/{index}.py" for index in range(count)])++ assert len(table.filters) == 3+ assert all(expression.count("'src/") <= _SUMMARY_PATH_BATCH_SIZE for expression in table.filters)++# ---------------------------------------------------------------------------# delete_many — bulk removal of swept page embeddings# ---------------------------------------------------------------------------