Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(index): bound repository-scale native memory (#1778)

Raghav Chamadiyae989355c6153
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/detector.py b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
index da11d9ca4..92485477e 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/detector.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/detector.py
@@ -44,7 +44,7 @@ DEFAULT_WINDOW_TOKENS = 50
DEFAULT_MIN_LINES = 6
-@dataclass
+@dataclass(slots=True)
class ClonePair:
"""One verified clone region between two files (or two regions in
the same file)."""
@@ -253,6 +253,11 @@ def detect_clones(
)
if cache is not None:
cache.save()
+ # The persisted form holds another representation of every token
+ # stream and window. Collision verification only needs the returned
+ # per-file kinds, so release the cache's owning dictionaries before
+ # allocating the repo-wide hash index and clone-pair lists.
+ cache.release_memory()
log.debug(
"duplication_token_cache",
hits=cache.hits,
@@ -262,6 +267,9 @@ def detect_clones(
return DuplicationReport(diagnostics=diag.as_log_fields())
bucket = index_by_hash(all_windows)
+ # Every WindowHash is now owned by a bucket. Keeping the flat list too
+ # adds millions of redundant references at the scan's high-water mark.
+ del all_windows
raw_pairs = _pairs_from_buckets(bucket, per_file_kinds, window_tokens, lim, diag)
if cache is not None and cache_dir is not None:
@@ -277,6 +285,11 @@ def detect_clones(
all_paths,
)
+ # Neither structure participates in merge/finalize/aggregation. Drop
+ # them before those stages allocate their output lists so freed arenas
+ # can be reused inside this phase rather than growing the process again.
+ del bucket, per_file_kinds
+
final = _finalize_pairs(_merge_adjacent_pairs(raw_pairs), min_lines, meta_map)
pairs_by_file, duplication_pct = _aggregate(final, per_file_nloc)
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/isolation.py b/packages/core/src/repowise/core/analysis/health/duplication/isolation.py
new file mode 100644
index 000000000..80d318a8e
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/duplication/isolation.py
@@ -0,0 +1,179 @@
+"""Process isolation for repository-scale duplicate detection.
+
+The full scan creates millions of short-lived Python objects. CPython frees
+them when the scan finishes, but allocator fragmentation can keep gigabytes of
+pages committed in the indexing process. Running only large scans in a child
+lets the operating system reclaim those pages before wiki generation starts.
+"""
+
+from __future__ import annotations
+
+import multiprocessing
+import traceback
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from repowise.core.cancellation import check_cancelled
+
+from ..source_reader import disk_source_reader
+from .detector import (
+ DEFAULT_MIN_LINES,
+ DEFAULT_WINDOW_TOKENS,
+ DuplicationReport,
+ detect_clones,
+)
+from .limits import DuplicationLimits
+
+# Process startup is needless overhead for the small repositories common in
+# unit tests and editor workflows. Repositories above this size are where the
+# detector's transient arenas become material compared with the base process.
+_ISOLATION_FILE_THRESHOLD = 1_000
+
+
+@dataclass(frozen=True, slots=True)
+class _FileInfo:
+ path: str
+ abs_path: str
+ language: str
+
+
+@dataclass(frozen=True, slots=True)
+class _ParsedFile:
+ file_info: _FileInfo
+
+
+def _worker(
+ sender: Any,
+ files: list[tuple[str, str, str]],
+ git_meta_map: dict[str, dict[str, Any]],
+ window_tokens: int,
+ min_lines: int,
+ limits: DuplicationLimits | None,
+ cache_dir: Path | None,
+ changed_files: set[str] | None,
+) -> None:
+ """Run in a fresh interpreter and return only the compact report."""
+ try:
+ parsed_files = [_ParsedFile(_FileInfo(*file_info)) for file_info in files]
+ report = detect_clones(
+ parsed_files,
+ git_meta_map,
+ window_tokens=window_tokens,
+ min_lines=min_lines,
+ limits=limits,
+ cache_dir=cache_dir,
+ changed_files=changed_files,
+ )
+ sender.send((True, report))
+ except BaseException:
+ sender.send((False, traceback.format_exc()))
+ finally:
+ sender.close()
+
+
+def detect_clones_with_isolation(
+ parsed_files: list[Any],
+ git_meta_map: dict[str, dict[str, Any]] | None = None,
+ *,
+ window_tokens: int = DEFAULT_WINDOW_TOKENS,
+ min_lines: int = DEFAULT_MIN_LINES,
+ limits: DuplicationLimits | None = None,
+ cache_dir: Path | None = None,
+ changed_files: set[str] | None = None,
+ source_reader: Any | None = None,
+ isolation_file_threshold: int = _ISOLATION_FILE_THRESHOLD,
+) -> DuplicationReport:
+ """Detect clones, isolating repository-scale transient allocations.
+
+ Only the three file-info strings used by the detector cross the process
+ boundary. The full parsed AST/model graph remains in the parent, and the
+ child returns the comparatively small final report.
+
+ Isolation is taken only for a working-tree read, which is the case it was
+ measured on and the only one whose reader survives a spawn. Any other
+ *source_reader* runs the detector in process; see the comment below.
+ """
+ # A reader that is not the working tree cannot cross the spawn boundary.
+ # ``MappingSourceReader`` holds every file's bytes, so shipping it would
+ # send the exact memory this isolation exists to avoid; and letting the
+ # worker fall back to disk would report working-tree findings as the
+ # revision's, which that reader documents as the thing it must not do.
+ # Those callers (the base side of a diff, a historical commit) analyse a
+ # changed-file set, not a repository, so the bound is not what they need.
+ reader_is_working_tree = source_reader is None or source_reader is disk_source_reader
+
+ if len(parsed_files) < isolation_file_threshold or not reader_is_working_tree:
+ return detect_clones(
+ parsed_files,
+ git_meta_map,
+ window_tokens=window_tokens,
+ min_lines=min_lines,
+ limits=limits,
+ cache_dir=cache_dir,
+ changed_files=changed_files,
+ source_reader=source_reader,
+ )
+
+ files = [
+ (pf.file_info.path, str(pf.file_info.abs_path), pf.file_info.language)
+ for pf in parsed_files
+ ]
+ # Clone weighting only reads this field. Avoid serializing the much larger
+ # Git metadata records into the worker.
+ compact_git_meta = {
+ path: {"co_change_partners_json": meta.get("co_change_partners_json")}
+ for path, meta in (git_meta_map or {}).items()
+ if meta.get("co_change_partners_json")
+ }
+
+ context = multiprocessing.get_context("spawn")
+ receiver, sender = context.Pipe(duplex=False)
+ process = context.Process(
+ target=_worker,
+ args=(
+ sender,
+ files,
+ compact_git_meta,
+ window_tokens,
+ min_lines,
+ limits,
+ cache_dir,
+ changed_files,
+ ),
+ name="repowise-duplication",
+ )
+ try:
+ process.start()
+ except BaseException:
+ receiver.close()
+ sender.close()
+ raise
+ sender.close()
+
+ try:
+ while not receiver.poll(0.1):
+ check_cancelled()
+ if not process.is_alive():
+ process.join()
+ raise RuntimeError(
+ f"duplication worker exited without a result (exit code {process.exitcode})"
+ )
+ try:
+ succeeded, result = receiver.recv()
+ except EOFError as exc:
+ raise RuntimeError(
+ f"duplication worker closed without a result (exit code {process.exitcode})"
+ ) from exc
+ process.join()
+ if not succeeded:
+ raise RuntimeError(f"duplication worker failed:\n{result}")
+ return result
+ finally:
+ receiver.close()
+ if process.is_alive():
+ process.terminate()
+ process.join(timeout=5)
+ if process.is_alive():
+ process.kill()
+ process.join()
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py
index 7bbd42f0a..a849443d8 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/rabin_karp.py
@@ -40,7 +40,7 @@ def _token_hash(tok_kind: str) -> int:
return h
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
class WindowHash:
"""One rolling-hash window with origin file + line span."""
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py
index 968e23ec2..b88613bff 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/token_cache.py
@@ -73,6 +73,16 @@ class DuplicationTokenCache:
except Exception as exc:
log.debug("duplication_cache_save_failed", error=str(exc))
+ def release_memory(self) -> None:
+ """Drop cached representations after a full scan has persisted them.
+
+ Callers retain the kind lists they still need for collision
+ verification. The cache-owned window tuples are otherwise a second
+ repo-sized copy of the live ``WindowHash`` population.
+ """
+ self._entries.clear()
+ self._fresh.clear()
+
# -- access ------------------------------------------------------------
def get(
@@ -118,8 +128,11 @@ class DuplicationTokenCache:
nloc: int,
windows: list[tuple[int, int, int, int]],
) -> None:
- # Interned kinds keep the pickle compact (each distinct kind is
- # memoized once by identity) and make later equality checks cheap.
- entry = ([sys.intern(k) for k in kinds], nloc, windows)
+ # Intern in place so the detector and cache share one kind list rather
+ # than holding two repo-sized lists of references during a full scan.
+ # Identity reuse also keeps the pickle compact and comparisons cheap.
+ for index, kind in enumerate(kinds):
+ kinds[index] = sys.intern(kind)
+ entry = (kinds, nloc, windows)
self._entries[content_hash] = entry
self._fresh[content_hash] = entry
diff --git a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py
index 7473ae5d4..4501c0aeb 100644
--- a/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py
+++ b/packages/core/src/repowise/core/analysis/health/duplication/tokenizer.py
@@ -22,6 +22,7 @@ preserve structure.
from __future__ import annotations
+import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING
@@ -82,7 +83,7 @@ _LITERAL_KINDS = frozenset(
)
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
class Token:
"""One AST token with the source location of its origin node."""
@@ -139,7 +140,11 @@ def _tokenize_leaf(node: Node, source: bytes) -> Token | None:
except Exception:
return None
return Token(
- kind=kind,
+ # A clone scan holds one Token per leaf and later keeps every kind
+ # through collision verification. Operators and keywords otherwise
+ # create one equal Python string per occurrence; interning turns that
+ # repo-sized string population into one object per distinct spelling.
+ kind=sys.intern(kind),
start_line=node.start_point[0] + 1,
end_line=node.end_point[0] + 1,
start_byte=node.start_byte,
diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.py
index 618982d2c..d34a27cb2 100644
--- a/packages/core/src/repowise/core/analysis/health/engine.py
+++ b/packages/core/src/repowise/core/analysis/health/engine.py
@@ -43,7 +43,8 @@ from .biomarkers import FileContext, detect_all
from .complexity import FileComplexity, FunctionComplexity, walk_file
from .coverage import is_test_file as _coverage_is_test_file
from .dataflow import FileDataflowCache
-from .duplication import DuplicationReport, detect_clones
+from .duplication import DuplicationReport
+from .duplication.isolation import detect_clones_with_isolation as detect_clones
from .models import HealthFileMetricData, HealthFindingData, HealthReport, Severity
from .perf import (
CallGraphIndex,
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
index 2775aaec3..21d5ce742 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
@@ -9,6 +9,11 @@ from ._base import STORED_SNIPPET_CHARS, VectorStore, iter_embed_chunks
__all__ = ["STORED_SNIPPET_CHARS", "LanceDBVectorStore"]
+# DataFusion expands every literal in a large ``IN`` filter into native query
+# state. A several-thousand-path generation level can otherwise commit
+# gigabytes before returning even though the selected result is small.
+_SUMMARY_PATH_BATCH_SIZE = 100
+
# ``STORED_SNIPPET_CHARS`` — how much of a page's content each row keeps — is
# defined with the embed recipe and re-exported here so the historical import
# path keeps working.
@@ -356,10 +361,11 @@ class LanceDBVectorStore(VectorStore):
return {"summary": summary, "key_exports": []}
async def get_page_summaries_by_paths(self, paths: list[str]) -> dict[str, dict]:
- """One ``IN``-filtered scan instead of one filtered query per path.
+ """Read summaries with bounded ``IN``-filtered scans.
Mirrors the single-path semantics (first row per path wins, empty
- summaries dropped, ``key_exports`` not stored in this schema).
+ summaries dropped, ``key_exports`` not stored in this schema). Batches
+ bound DataFusion's native query-plan allocation for large levels.
"""
if not paths:
return {}
@@ -367,22 +373,25 @@ class LanceDBVectorStore(VectorStore):
if self._table is None:
return {}
- try:
- rows = (
- await self._table.query() # type: ignore[union-attr]
- .where(_paths_in_filter(paths))
- .select(["target_path", "content_snippet"])
- .to_list()
- )
- except Exception:
- return {}
-
out: dict[str, dict] = {}
- for r in rows:
- tp = str(r.get("target_path") or "")
- if not tp or tp in out:
- continue
- summary = str(r.get("content_snippet") or "")[:_SNIPPET_LEN]
- if summary:
- out[tp] = {"summary": summary, "key_exports": []}
+ unique_paths = list(dict.fromkeys(paths))
+ for index in range(0, len(unique_paths), _SUMMARY_PATH_BATCH_SIZE):
+ batch = unique_paths[index : index + _SUMMARY_PATH_BATCH_SIZE]
+ try:
+ rows = (
+ await self._table.query() # type: ignore[union-attr]
+ .where(_paths_in_filter(batch))
+ .select(["target_path", "content_snippet"])
+ .to_list()
+ )
+ except Exception:
+ return {}
+
+ for row in rows:
+ target_path = str(row.get("target_path") or "")
+ if not target_path or target_path in out:
+ continue
+ summary = str(row.get("content_snippet") or "")[:_SNIPPET_LEN]
+ if summary:
+ out[target_path] = {"summary": summary, "key_exports": []}
return out
diff --git a/tests/unit/health/test_duplication_cache.py b/tests/unit/health/test_duplication_cache.py
index c1d950c3f..9337e03a6 100644
--- a/tests/unit/health/test_duplication_cache.py
+++ b/tests/unit/health/test_duplication_cache.py
@@ -2,14 +2,18 @@
from __future__ import annotations
+import sys
from pathlib import Path
from types import SimpleNamespace
from repowise.core.analysis.health.duplication import detect_clones
+from repowise.core.analysis.health.duplication.detector import ClonePair
+from repowise.core.analysis.health.duplication.rabin_karp import WindowHash
from repowise.core.analysis.health.duplication.token_cache import (
_CACHE_FILENAME,
DuplicationTokenCache,
)
+from repowise.core.analysis.health.duplication.tokenizer import Token
def _pf(path: str, abs_path: str) -> SimpleNamespace:
@@ -113,3 +117,32 @@ def test_corrupt_cache_degrades_to_full_run(tmp_path: Path):
baseline = detect_clones(parsed, window_tokens=20, min_lines=4)
report = detect_clones(parsed, window_tokens=20, min_lines=4, cache_dir=cache_dir)
assert _report_key(report) == _report_key(baseline)
+
+
+def test_cache_shares_interned_kinds_and_can_release_memory(tmp_path: Path):
+ cache = DuplicationTokenCache(tmp_path, 20)
+ kinds = [bytearray(b"identifier").decode() for _ in range(2)]
+
+ cache.put("digest", kinds, 1, [(1, 0, 1, 1)])
+ cached = cache.get("digest")
+
+ assert cached is not None
+ cached_kinds, _, _ = cached
+ assert cached_kinds is kinds
+ assert all(kind is sys.intern(kind) for kind in kinds)
+
+ cache.release_memory()
+ assert cache.get("digest") is None
+ # Releasing the owning dictionaries must not invalidate the detector's
+ # reference used for collision verification.
+ assert kinds == ["identifier", "identifier"]
+
+
+def test_high_volume_duplication_records_are_slotted():
+ token = Token("ID", 1, 1, 0, 1)
+ window = WindowHash("a.py", 1, 0, 1, 1)
+ pair = ClonePair("a.py", "b.py", 1, 2, 1, 2, 20)
+
+ assert not hasattr(token, "__dict__")
+ assert not hasattr(window, "__dict__")
+ assert not hasattr(pair, "__dict__")
diff --git a/tests/unit/health/test_duplication_isolation.py b/tests/unit/health/test_duplication_isolation.py
new file mode 100644
index 000000000..76659824c
--- /dev/null
+++ b/tests/unit/health/test_duplication_isolation.py
@@ -0,0 +1,154 @@
+"""Large duplication scans release their transient allocator arenas."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+
+from repowise.core.analysis.health.duplication import detect_clones
+from repowise.core.analysis.health.duplication.isolation import (
+ detect_clones_with_isolation,
+)
+
+
+def _parsed(path: Path) -> SimpleNamespace:
+ return SimpleNamespace(
+ file_info=SimpleNamespace(
+ path=path.name,
+ abs_path=str(path),
+ language="python",
+ )
+ )
+
+
+def _key(report):
+ return sorted(
+ (
+ pair.file_a,
+ pair.file_b,
+ pair.a_start_line,
+ pair.a_end_line,
+ pair.b_start_line,
+ pair.b_end_line,
+ )
+ for pair in report.pairs
+ )
+
+
+def test_isolated_detection_matches_in_process(tmp_path: Path):
+ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))
+ first = tmp_path / "a.py"
+ second = tmp_path / "b.py"
+ first.write_text(body)
+ second.write_text(body)
+ parsed = [_parsed(first), _parsed(second)]
+
+ expected = detect_clones(parsed, window_tokens=20, min_lines=4)
+ actual = detect_clones_with_isolation(
+ parsed,
+ window_tokens=20,
+ min_lines=4,
+ cache_dir=tmp_path / ".repowise",
+ isolation_file_threshold=0,
+ )
+
+ assert _key(actual) == _key(expected)
+ assert actual.duplication_pct == expected.duplication_pct
+ assert (tmp_path / ".repowise" / "duplication_cache.pkl").exists()
+
+
+def test_a_custom_reader_is_honoured_not_ignored(tmp_path: Path):
+ """The reader's bytes win over what is on disk.
+
+ The engine routes every health read through a ``SourceReader`` so a pass
+ can analyse content that is not the working tree. If the isolation wrapper
+ dropped the reader, those callers would silently get working-tree results
+ labelled as the revision's.
+ """
+ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))
+ first = tmp_path / "a.py"
+ second = tmp_path / "b.py"
+ # On disk the two files differ, so a disk read finds no clone at all.
+ first.write_text(body)
+ second.write_text("unique = 0\n")
+ parsed = [_parsed(first), _parsed(second)]
+
+ served = {str(first): body.encode(), str(second): body.encode()}
+ report = detect_clones_with_isolation(
+ parsed,
+ window_tokens=20,
+ min_lines=4,
+ source_reader=lambda abs_path: served.get(abs_path),
+ isolation_file_threshold=0,
+ )
+
+ assert report.pairs, "the reader's identical bodies should clone-match"
+ assert _key(report) == _key(detect_clones(parsed, window_tokens=20, min_lines=4,
+ source_reader=lambda p: served.get(p)))
+
+
+def test_a_custom_reader_stays_in_process(tmp_path: Path, monkeypatch):
+ """A non-working-tree reader must not be shipped to a spawned worker.
+
+ It cannot be pickled in the general case, and the one such reader in the
+ tree holds every file's bytes, which is the memory the isolation exists to
+ avoid moving. Falling back to a disk read in the worker would be wrong
+ rather than merely slow, so the whole scan stays here.
+ """
+ import repowise.core.analysis.health.duplication.isolation as isolation
+
+ def _fail(*args, **kwargs):
+ raise AssertionError("a custom reader must not reach the spawned worker")
+
+ monkeypatch.setattr(isolation.multiprocessing, "get_context", _fail)
+
+ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))
+ first = tmp_path / "a.py"
+ second = tmp_path / "b.py"
+ first.write_text(body)
+ second.write_text(body)
+ parsed = [_parsed(first), _parsed(second)]
+
+ served = {str(first): body.encode(), str(second): body.encode()}
+ report = isolation.detect_clones_with_isolation(
+ parsed,
+ window_tokens=20,
+ min_lines=4,
+ source_reader=lambda abs_path: served.get(abs_path),
+ isolation_file_threshold=0,
+ )
+ assert report.pairs
+
+
+def test_the_default_reader_still_isolates(tmp_path: Path, monkeypatch):
+ """The working-tree case keeps the spawn, which is where the win was measured."""
+ import repowise.core.analysis.health.duplication.isolation as isolation
+ from repowise.core.analysis.health.source_reader import disk_source_reader
+
+ spawned: list[bool] = []
+ real = isolation.multiprocessing.get_context
+
+ def _record(*args, **kwargs):
+ spawned.append(True)
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(isolation.multiprocessing, "get_context", _record)
+
+ body = "\n".join(f"value_{index} = source_{index} + 1" for index in range(30))
+ first = tmp_path / "a.py"
+ second = tmp_path / "b.py"
+ first.write_text(body)
+ second.write_text(body)
+ parsed = [_parsed(first), _parsed(second)]
+
+ for reader in (None, disk_source_reader):
+ spawned.clear()
+ report = isolation.detect_clones_with_isolation(
+ parsed,
+ window_tokens=20,
+ min_lines=4,
+ source_reader=reader,
+ isolation_file_threshold=0,
+ )
+ assert spawned, f"reader {reader!r} should still take the isolated path"
+ assert report.pairs
diff --git a/tests/unit/persistence/test_vector_store_batching.py b/tests/unit/persistence/test_vector_store_batching.py
index c99caa615..f40225402 100644
--- a/tests/unit/persistence/test_vector_store_batching.py
+++ b/tests/unit/persistence/test_vector_store_batching.py
@@ -7,6 +7,8 @@ import pytest
from repowise.core.persistence.vector_store.in_memory import InMemoryVectorStore
from repowise.core.persistence.vector_store.lancedb_store import (
+ _SUMMARY_PATH_BATCH_SIZE,
+ LanceDBVectorStore,
_page_ids_in_filter,
_paths_in_filter,
)
@@ -163,6 +165,44 @@ async def test_lancedb_batch_summaries_roundtrip(tmp_path) -> None:
await store.close()
+async def test_lancedb_batch_summaries_bounds_in_filter_size(tmp_path) -> None:
+ class Query:
+ def __init__(self, filters: list[str]) -> None:
+ self.filters = filters
+
+ def where(self, expression: str):
+ self.filters.append(expression)
+ return self
+
+ def select(self, _columns: list[str]):
+ return self
+
+ async def to_list(self):
+ return []
+
+ class Table:
+ def __init__(self) -> None:
+ self.filters: list[str] = []
+
+ def query(self):
+ return Query(self.filters)
+
+ store = LanceDBVectorStore(str(tmp_path / "lance"), MockEmbedder())
+ table = Table()
+ store._table = table
+
+ async def connected() -> None:
+ return None
+
+ store._ensure_connected = connected # type: ignore[method-assign]
+ count = _SUMMARY_PATH_BATCH_SIZE * 2 + 7
+ await store.get_page_summaries_by_paths([f"src/{index}.py" for index in range(count)])
+
+ assert len(table.filters) == 3
+ assert all(expression.count("'src/") <= _SUMMARY_PATH_BATCH_SIZE for expression in table.filters)
+
+
# ---------------------------------------------------------------------------
# delete_many — bulk removal of swept page embeddings
# ---------------------------------------------------------------------------