Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(cuvs): improve recovery scalability and filtered-search efficiency (#3311)
Yuanqing ZHAO2d84a6f13cfe
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 3f5e5fcb2..7bf80d019 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -1295,9 +1295,6 @@ class PersistCollection(LocalCollection):passif index_count > 0:logger.info("Recovering %d index(es) from %s", index_count, self.index_dir)- recovery_candidates = (- self.store_mgr.get_all_cands_data() if self.store_mgr is not None else []- )for folder in os.listdir(self.index_dir):try:index_dir = safe_join(self.index_dir, folder)@@ -1329,6 +1326,12 @@ class PersistCollection(LocalCollection):# all data from the delta log (CandidateData) regardless of when that data was created.# If we used the default (current time), the index would ignore older data in the log.# Keep the dense worker stopped until the native snapshot has replayed those deltas.+ # Create a fresh lazy stream for each index. In particular, cuVS can pack+ # one candidate at a time instead of retaining every deserialized FP32+ # vector alongside its compact host shadow during collection recovery.+ recovery_candidates = (+ self.store_mgr.iter_all_cands_data() if self.store_mgr is not None else iter(())+ )index = PersistentIndex(name=index_name,path=self.index_dir,diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.pyindex 92a951368..9ae9f1d3e 100644--- a/openviking/storage/vectordb/engine/_python_api.py+++ b/openviking/storage/vectordb/engine/_python_api.py@@ -126,6 +126,7 @@ class Schema:def get_field_order(self) -> list[FieldMeta]:return self.field_orders+def _get_row_value(row_data: Any, field_name: str, default_value: Any) -> Any:if isinstance(row_data, dict):return row_data.get(field_name, default_value)@@ -350,11 +351,11 @@ class FilterResult:self,*,eligible_count: int = 0,- bitset_words: list[int] | None = None,+ bitset_words: list[int] | bytes | None = None,native_filter_token: int = 0,):self.eligible_count = eligible_count- self.bitset_words = bitset_words or []+ self.bitset_words = [] if bitset_words is None else bitset_wordsself.native_filter_token = native_filter_token@classmethod@@ -365,6 +366,19 @@ class FilterResult:native_filter_token=int(payload.get("native_filter_token", 0)),)+ @classmethod+ def from_packed_backend(cls, payload: dict[str, Any]) -> "FilterResult":+ if "bitset_words_le" not in payload:+ raise KeyError("Packed filter ABI response is missing bitset_words_le")+ bitset_words = payload["bitset_words_le"]+ if not isinstance(bitset_words, bytes):+ raise TypeError("Packed filter ABI must return bitset_words_le as bytes")+ return cls(+ eligible_count=int(payload.get("eligible_count", 0)),+ bitset_words=bitset_words,+ native_filter_token=int(payload.get("native_filter_token", 0)),+ )+class StateResult:__slots__ = ("update_timestamp", "element_count")@@ -481,6 +495,66 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:payload = self._backend._index_engine_evaluate_filter(self._handle, dsl)return FilterResult.from_backend(payload)+ def evaluate_filter_packed(self, dsl: str, max_cached_candidates: int = 0) -> FilterResult:+ symbol_name = (+ "_index_engine_evaluate_filter_cached_packed"+ if max_cached_candidates > 0+ else "_index_engine_evaluate_filter_packed"+ )+ evaluate_packed = getattr(self._backend, symbol_name, None)+ if evaluate_packed is None:+ return self.evaluate_filter(+ dsl,+ max_cached_candidates=max_cached_candidates,+ )+ if max_cached_candidates > 0:+ payload = evaluate_packed(+ self._handle,+ dsl,+ max_cached_candidates,+ )+ else:+ payload = evaluate_packed(self._handle, dsl)+ return FilterResult.from_packed_backend(payload)++ def evaluate_filter_for_routing(self, dsl: str, native_threshold: int) -> FilterResult:+ evaluate_for_routing = getattr(+ self._backend,+ "_index_engine_evaluate_filter_for_routing",+ None,+ )+ if evaluate_for_routing is None:+ return self.evaluate_filter(+ dsl,+ max_cached_candidates=native_threshold,+ )+ payload = evaluate_for_routing(+ self._handle,+ dsl,+ native_threshold,+ )+ return FilterResult.from_backend(payload)++ def evaluate_filter_for_routing_packed(+ self, dsl: str, native_threshold: int+ ) -> FilterResult:+ evaluate_packed = getattr(+ self._backend,+ "_index_engine_evaluate_filter_for_routing_packed",+ None,+ )+ if evaluate_packed is None:+ return self.evaluate_filter_for_routing(+ dsl,+ native_threshold=native_threshold,+ )+ payload = evaluate_packed(+ self._handle,+ dsl,+ native_threshold,+ )+ return FilterResult.from_packed_backend(payload)+def dump(self, path: str) -> int:return int(self._backend._index_engine_dump(self._handle, path))@@ -510,6 +584,28 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:def seek_range(self, start_key: str, end_key: str) -> list[tuple[str, bytes]]:return list(self._backend._store_seek_range(self._handle, start_key, end_key))+ def seek_range_page(+ self,+ start_key: str,+ end_key: str,+ limit: int,+ max_bytes: int,+ start_exclusive: bool = False,+ ) -> list[tuple[str, bytes]]:+ seek_page = getattr(self._backend, "_store_seek_range_page", None)+ if seek_page is None:+ raise NotImplementedError("This native engine does not support paged store scans")+ return list(+ seek_page(+ self._handle,+ start_key,+ end_key,+ limit,+ max_bytes,+ start_exclusive,+ )+ )+class PersistStore(_StoreBase):def __init__(self, path: str):super().__init__(backend._new_persist_store(path))diff --git a/openviking/storage/vectordb/index/cuvs_index.py b/openviking/storage/vectordb/index/cuvs_index.pyindex c6c173078..303b1a15c 100644--- a/openviking/storage/vectordb/index/cuvs_index.py+++ b/openviking/storage/vectordb/index/cuvs_index.py@@ -20,21 +20,42 @@ import jsonimport loggingimport mathimport re+import structimport threadingimport timeimport traceback+from array import arrayfrom collections import OrderedDictfrom contextlib import nullcontext-from dataclasses import dataclass-from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union+from dataclasses import dataclass, field+from typing import (+ Any,+ Callable,+ Dict,+ Iterable,+ Iterator,+ List,+ Mapping,+ Optional,+ Sequence,+ Tuple,+ Union,+ overload,+)from openviking.storage.vectordb.store.data import CandidateData, DeltaRecordlogger = logging.getLogger(__name__)+_FP32_BYTES = 4+_U32_BYTES = 4+_FP32_UPLOAD_BATCH_BYTES = 64 * 1024 * 1024++NativeFilterWords = Union[Sequence[int], bytes]+StoredNativeFilterWords = Union[Tuple[int, ...], bytes]NativeFilterEvaluation = Union[- Tuple[Sequence[int], int],- Tuple[Sequence[int], int, int],+ Tuple[NativeFilterWords, int],+ Tuple[NativeFilterWords, int, int],]NativeFilterResolver = Callable[[Mapping[str, Any]], NativeFilterEvaluation]@@ -55,6 +76,10 @@ class UnsupportedCuVSFilterError(ValueError):"""Raised when a filter cannot be translated to a cuVS prefilter."""+class _StalePreparedFilter(RuntimeError):+ """Internal retry signal for a host filter invalidated before admission."""++@dataclass(frozen=True)class CuVSMemoryEstimate:"""Conservative GPU-memory estimate used only for auto-admission."""@@ -77,6 +102,8 @@ class CuVSSearchTelemetry:route_reason: str = "pending"filter_kind: str = "none"filter_cache_hit: bool = False+ filter_cache_eviction_fallback: bool = False+ filter_words_packed: bool = Falsenative_filter_reused: bool = Falsebuild_performed: bool = Falseeligible_count: Optional[int] = None@@ -88,6 +115,7 @@ class CuVSSearchTelemetry:total_ms: float = 0.0preflight_ms: float = 0.0queue_ms: float = 0.0+ gpu_gate_queue_ms: float = 0.0build_ms: float = 0.0filter_prepare_ms: float = 0.0gpu_search_ms: float = 0.0@@ -102,6 +130,8 @@ class CuVSSearchTelemetry:"route_reason": self.route_reason,"filter_kind": self.filter_kind,"filter_cache_hit": self.filter_cache_hit,+ "filter_cache_eviction_fallback": self.filter_cache_eviction_fallback,+ "filter_words_packed": self.filter_words_packed,"native_filter_reused": self.native_filter_reused,"build_performed": self.build_performed,"eligible_count": self.eligible_count,@@ -113,6 +143,7 @@ class CuVSSearchTelemetry:"total_ms": round(self.total_ms, 3),"preflight_ms": round(self.preflight_ms, 3),"queue_ms": round(self.queue_ms, 3),+ "gpu_gate_queue_ms": round(self.gpu_gate_queue_ms, 3),"build_ms": round(self.build_ms, 3),"filter_prepare_ms": round(self.filter_prepare_ms, 3),"gpu_search_ms": round(self.gpu_search_ms, 3),@@ -355,6 +386,60 @@ def matches_filter(raise UnsupportedCuVSFilterError(f"Unsupported cuVS filter operation: {op!r}")+@dataclass(frozen=True)+class _PackedFP32Rows(Sequence[Sequence[float]]):+ """Immutable FP32 rows captured for one cuVS rebuild.++ The row blobs keep mutation snapshots cheap and safe: an upsert replaces a+ blob instead of modifying storage which a background rebuild may still be+ reading. Runtime uploads concatenate only a bounded number of rows at a+ time, so a rebuild does not require another full host-side dataset copy.+ """++ rows: Tuple[bytes, ...]+ dimension: int++ def __len__(self) -> int:+ return len(self.rows)++ @overload+ def __getitem__(self, index: int) -> Sequence[float]: ...++ @overload+ def __getitem__(self, index: slice) -> Sequence[Sequence[float]]: ...++ def __getitem__(+ self, index: Union[int, slice]+ ) -> Union[Sequence[float], Sequence[Sequence[float]]]:+ if isinstance(index, slice):+ return tuple(memoryview(row).cast("f") for row in self.rows[index])+ return memoryview(self.rows[index]).cast("f")++ def __iter__(self) -> Iterator[Sequence[float]]:+ for row in self.rows:+ yield memoryview(row).cast("f")++ @property+ def nbytes(self) -> int:+ return len(self.rows) * self.dimension * _FP32_BYTES++ def iter_packed_batches(+ self,+ max_bytes: int,+ ) -> Iterator[Tuple[int, int, bytes]]:+ """Yield row-aligned host buffers bounded by ``max_bytes`` when possible."""++ if max_bytes <= 0:+ raise ValueError("cuVS FP32 upload batch size must be positive")+ row_bytes = self.dimension * _FP32_BYTES+ if row_bytes <= 0:+ return+ rows_per_batch = max(1, max_bytes // row_bytes)+ for start in range(0, len(self.rows), rows_per_batch):+ end = min(start + rows_per_batch, len(self.rows))+ yield start, end, b"".join(self.rows[start:end])++class _CuVSRuntime:"""Small adapter around the public cuVS Python API."""@@ -401,13 +486,11 @@ class _CuVSRuntime:self._owned_resources: List[Any] = []self._resource_limit = 1self._resources_closed = False- self._use_explicit_resources = Falsedef set_max_concurrent_searches(self, value: int) -> None:limit = max(1, int(value))with self._resource_condition:self._resource_limit = limit- self._use_explicit_resources = limit > 1self._resource_condition.notify_all()def device_scope(self):@@ -442,9 +525,36 @@ class _CuVSRuntime:def build(self, dataset: Sequence[Sequence[float]]):with self.device_scope():device_dataset = None+ packed_batch = None+ host_batch = Noneindex = Nonetry:- device_dataset = self.cp.asarray(dataset, dtype=self.device_dtype)+ if isinstance(dataset, _PackedFP32Rows):+ # CuPy already depends on NumPy, but keep the import on the+ # GPU-only path so native OpenViking users do not gain a+ # new import-time dependency through this module.+ import numpy as np++ device_dataset = self.cp.empty(+ (len(dataset), dataset.dimension),+ dtype=self.device_dtype,+ )+ for start, end, packed_batch in dataset.iter_packed_batches(+ _FP32_UPLOAD_BATCH_BYTES+ ):+ host_batch = np.frombuffer(packed_batch, dtype=np.float32).reshape(+ end - start, dataset.dimension+ )+ if self.dtype == "float16":+ host_batch = host_batch.astype(np.float16)+ # ndarray.set() without an explicit stream performs a+ # synchronous host-to-device copy. The bounded host+ # buffer can therefore be released before the next batch.+ device_dataset[start:end].set(host_batch)+ host_batch = None+ packed_batch = None+ else:+ device_dataset = self.cp.asarray(dataset, dtype=self.device_dtype)if self.algorithm == "brute_force":index = self.brute_force.build(device_dataset, metric=self.metric)return _CuVSRuntimeIndex(index=index, dataset=device_dataset)@@ -456,6 +566,8 @@ class _CuVSRuntime:# Drop them before Device.__exit__ restores the caller's device.index = Nonedevice_dataset = None+ host_batch = None+ packed_batch = None# Python/Cython exception frames can retain CUDA arguments even# after the locals above are cleared. Preserve stack locations# while releasing frame locals under the captured device.@@ -504,10 +616,33 @@ class _CuVSRuntime:words[index // 32] |= 1 << (index % 32)return self.cp.asarray(words, dtype=self.cp.uint32)- def prepare_filter_words(self, words: Sequence[int]):+ def prepare_filter_words(self, words: NativeFilterWords):"""Copy an already packed native filter bitmap to the device."""+ if isinstance(words, bytes) and len(words) % _U32_BYTES != 0:+ raise ValueError("Packed native filter bitmap length must be a multiple of 4 bytes")with self.device_scope():+ if isinstance(words, bytes):+ # Keep NumPy on the GPU-only path. The explicit little-endian+ # dtype matches the additive native ABI without constructing a+ # Python int for every bitmap word.+ import numpy as np++ host_words = np.frombuffer(words, dtype="<u4")+ device_words = None+ try:+ device_words = self.cp.empty(host_words.shape, dtype=self.cp.uint32)+ # ndarray.set() without an explicit stream completes the host+ # copy before the immutable bytes owner can leave the cache.+ device_words.set(host_words)+ return device_words+ except BaseException as exc:+ # Drop the last device reference before restoring the+ # caller's CUDA device; exception frames can otherwise+ # retain it beyond Device.__exit__.+ device_words = None+ traceback.clear_frames(exc.__traceback__)+ raisereturn self.cp.asarray(words, dtype=self.cp.uint32)def search(@@ -527,8 +662,14 @@ class _CuVSRuntime:resource_reusable = Falsetry:index = runtime_index.index- resources = self._acquire_resources() if self._use_explicit_resources else None- resource_kwargs = {"resources": resources} if resources is not None else {}+ # Always provide an explicit resource, including the serialized+ # max_concurrent_gpu_searches=1 case. cuVS synchronizes an+ # implicit resource before returning, but an owned reusable+ # resource also pins worker-thread churn to a known stream and+ # gives filter/result lifetimes an explicit synchronization+ # boundary inside this admitted search.+ resources = self._acquire_resources()+ resource_kwargs = {"resources": resources}queries = self.cp.asarray([query], dtype=self.device_dtype)if mask is None:prefilter = None@@ -558,14 +699,18 @@ class _CuVSRuntime:filter=prefilter,**resource_kwargs,)- if resources is not None:- resources.sync()- resource_reusable = True+ resources.sync()host_neighbors = self.cp.asnumpy(neighbors)[0].tolist()host_distances = self.cp.asnumpy(distances)[0].tolist()- return [int(item) for item in host_neighbors], [- float(item) for item in host_distances- ]+ result = (+ [int(item) for item in host_neighbors],+ [float(item) for item in host_distances],+ )+ # Reuse only after the complete call, including host result+ # materialization, succeeds. Any exception discards a+ # potentially poisoned resource from the pool.+ resource_reusable = True+ return resultexcept Exception as exc:traceback.clear_frames(exc.__traceback__)raise@@ -597,7 +742,7 @@ class _CuVSRuntime:@dataclass(frozen=True)class _Record:- vector: Tuple[float, ...]+ vector: bytesfields: Mapping[str, Any]@@ -629,15 +774,57 @@ class _CachedFilter:route_native: bool = Falsenative_threshold: int = 0native_filter_token: int = 0+ filter_words_packed: bool = False@dataclass(frozen=True)class _ResolvedNativeFilter:- bitset_words: Tuple[int, ...]+ bitset_words: StoredNativeFilterWordseligible_count: introute_native: boolnative_threshold: intnative_filter_token: int = 0+ filter_words_packed: bool = False+++@dataclass(frozen=True)+class _CachedFilterMetadata:+ """Host-only copy of route metadata from a potentially device-backed cache entry."""++ eligible_count: int+ route_native: bool+ native_threshold: int+ native_filter_token: int+ filter_words_packed: bool+++@dataclass(frozen=True)+class _PreparedHostFilter:+ """Generation-bound host work completed before GPU admission.++ This context deliberately never owns a device allocation. A cache hit is+ represented only by copied route metadata; the device entry is borrowed+ again after admission. Otherwise ``resolved_native_filter`` contains host+ words whose conversion to a device bitset remains inside the GPU gate.+ """++ generation: int+ cache_key: Optional[str]+ cached_metadata: Optional[_CachedFilterMetadata] = None+ resolved_native_filter: Optional[_ResolvedNativeFilter] = None+++@dataclass+class _NativeFilterPreflightFlight:+ generation: int+ done: threading.Event = field(default_factory=threading.Event)+ route_count: Optional[int] = None+ eligible_count: int = 0+ cached_metadata: Optional[_CachedFilterMetadata] = None+ resolved_native_filter: Optional[_ResolvedNativeFilter] = None+ error: Optional[BaseException] = None+ stale: bool = False+ participants: int = 1class CuVSDenseIndex:@@ -724,6 +911,7 @@ class CuVSDenseIndex:self._dirty = Trueself._filter_cache: OrderedDict[str, _CachedFilter] = OrderedDict()self._preflight_filter_cache: OrderedDict[str, _ResolvedNativeFilter] = OrderedDict()+ self._preflight_flights: Dict[Tuple[int, str], _NativeFilterPreflightFlight] = {}self._lock = threading.RLock()self._idle_condition = threading.Condition(self._lock)self._active_searches = 0@@ -731,6 +919,8 @@ class CuVSDenseIndex:self._filter_layout_lock = threading.Lock()self._records_generation = 0self._filter_layout_generation = -1+ self._closed = False+ self._close_complete = Falselogger.info("Initialized cuVS dense index: algorithm=%s metric=%s dimension=%d",self.algorithm,@@ -747,18 +937,33 @@ class CuVSDenseIndex:with self._lock:return len(self._records)+ @property+ def host_shadow_nbytes(self) -> int:+ """Return the compact FP32 vector payload size, excluding Python metadata."""++ with self._lock:+ return len(self._records) * self.dimension * _FP32_BYTES+@propertydef needs_rebuild(self) -> bool:with self._lock:return self._dirty- def _prepare_vector(self, vector: Sequence[float]) -> Tuple[float, ...]:+ def _prepare_vector_values(self, vector: Sequence[float]) -> List[float]:if len(vector) != self.dimension:raise ValueError(f"cuVS vector dimension mismatch: expected {self.dimension}, got {len(vector)}")- prepared = _normalize(vector) if self.normalize_vectors else [float(v) for v in vector]- return tuple(prepared)+ return _normalize(vector) if self.normalize_vectors else [float(v) for v in vector]++ def _prepare_vector(self, vector: Sequence[float]) -> Tuple[float, ...]:+ return tuple(self._prepare_vector_values(vector))++ def _pack_vector(self, vector: Sequence[float]) -> bytes:+ packed = array("f", self._prepare_vector_values(vector))+ if packed.itemsize != _FP32_BYTES:+ raise RuntimeError("cuVS host vector storage requires 4-byte IEEE FP32 values")+ return packed.tobytes()@staticmethoddef _parse_fields(value: str) -> Mapping[str, Any]:@@ -776,7 +981,7 @@ class CuVSDenseIndex:if not candidate.vector:continueself._records[int(candidate.label)] = _Record(- vector=self._prepare_vector(candidate.vector),+ vector=self._pack_vector(candidate.vector),fields=self._parse_fields(candidate.fields),)self._invalidate()@@ -788,7 +993,7 @@ class CuVSDenseIndex:if not record.vector:continueself._records[int(record.label)] = _Record(- vector=self._prepare_vector(record.vector),+ vector=self._pack_vector(record.vector),fields=self._parse_fields(record.fields),)changed = True@@ -807,10 +1012,20 @@ class CuVSDenseIndex:def _invalidate(self) -> None:self._records_generation += 1self._dirty = True+ self._cancel_preflight_flights_()with self._runtime_device_scope():self._filter_cache.clear()self._preflight_filter_cache.clear()+ def _cancel_preflight_flights_(self) -> None:+ """Mark in-flight projections stale and wake waiters under ``_lock``."""++ flights = tuple(self._preflight_flights.values())+ self._preflight_flights.clear()+ for flight in flights:+ flight.stale = True+ flight.done.set()+@staticmethoddef _filter_cache_key(filters: Mapping[str, Any]) -> Optional[str]:try:@@ -826,6 +1041,16 @@ class CuVSDenseIndex:self._filter_cache[cache_key] = cachedreturn cached+ @staticmethod+ def _cached_filter_metadata(cached: _CachedFilter) -> _CachedFilterMetadata:+ return _CachedFilterMetadata(+ eligible_count=cached.eligible_count,+ route_native=cached.route_native,+ native_threshold=cached.native_threshold,+ native_filter_token=cached.native_filter_token,+ filter_words_packed=cached.filter_words_packed,+ )+def _cache_filter(self, cache_key: Optional[str], cached: _CachedFilter) -> None:if cache_key is None or self.filter_cache_size <= 0:return@@ -855,6 +1080,51 @@ class CuVSDenseIndex:while len(self._preflight_filter_cache) > capacity:self._preflight_filter_cache.popitem(last=False)+ def _lookup_preflight_route_(+ self,+ cache_key: Optional[str],+ ) -> Optional[Tuple[Optional[int], int]]:+ cached = self._get_cached_filter(cache_key)+ if cached is not None:+ return (+ cached.eligible_count if cached.route_native else None,+ cached.eligible_count,+ )+ resolved = self._get_preflight_filter(cache_key)+ if resolved is None:+ return None+ return (+ resolved.eligible_count if resolved.route_native else None,+ resolved.eligible_count,+ )++ def _store_preflight_route_(+ self,+ cache_key: Optional[str],+ resolved: _ResolvedNativeFilter,+ ) -> Tuple[Optional[int], int, bool]:+ existing = self._lookup_preflight_route_(cache_key)+ if existing is not None:+ route_count, eligible_count = existing+ return route_count, eligible_count, True+ if not resolved.route_native and resolved.eligible_count != 0:+ self._cache_preflight_filter(cache_key, resolved)+ return None, resolved.eligible_count, False+ cached = _CachedFilter(+ prepared=None,+ eligible_count=resolved.eligible_count,+ route_native=resolved.route_native,+ native_threshold=resolved.native_threshold,+ native_filter_token=resolved.native_filter_token,+ filter_words_packed=resolved.filter_words_packed,+ )+ self._cache_filter(cache_key, cached)+ return (+ cached.eligible_count if cached.route_native else None,+ cached.eligible_count,+ False,+ )+def native_filter_threshold(self, filters: Mapping[str, Any]) -> int:return (self.auto_path_filter_native_threshold@@ -877,6 +1147,8 @@ class CuVSDenseIndex:with self._filter_layout_lock:with self._lock:generation = self._records_generation+ if self._closed:+ return Noneif self._filter_layout_generation == generation:return generationordered_labels = list(self._records)@@ -884,7 +1156,7 @@ class CuVSDenseIndex:native_filter_layout_registrar(ordered_labels)with self._lock:- if self._records_generation != generation:+ if self._closed or self._records_generation != generation:return Noneself._filter_layout_generation = generationreturn generation@@ -902,8 +1174,9 @@ class CuVSDenseIndex:try:if not self.auto_memory or not filters:return None- cache_key = self._filter_cache_key(filters)with self._lock:+ if self._closed:+ return Noneif telemetry is not None:telemetry.filter_kind = ("path"@@ -912,77 +1185,253 @@ class CuVSDenseIndex:)telemetry.records_generation = self._records_generationtelemetry.index_size = len(self._records)- cached = self._get_cached_filter(cache_key)- if cached is not None:- if telemetry is not None:- telemetry.filter_cache_hit = True- telemetry.eligible_count = cached.eligible_count- return cached.eligible_count if cached.route_native else Nonenative_threshold = self.native_filter_threshold(filters)if native_threshold <= 0:return None- preflight_cached = self._get_preflight_filter(cache_key)- if preflight_cached is not None:- if telemetry is not None:- telemetry.filter_cache_hit = True- telemetry.eligible_count = preflight_cached.eligible_count- return (- preflight_cached.eligible_count if preflight_cached.route_native else None- )+ prepared = self._prepare_host_filter(+ filters,+ native_filter_resolver,+ native_filter_layout_registrar,+ telemetry,+ )+ if prepared is None:+ return None+ cached = prepared.cached_metadata+ resolved = prepared.resolved_native_filter+ if cached is None and resolved is None:+ raise RuntimeError("Native filter preflight completed without a result")+ eligible_count = (+ cached.eligible_count if cached is not None else resolved.eligible_count+ )+ route_native = cached.route_native if cached is not None else resolved.route_native+ return eligible_count if route_native else None+ finally:+ if telemetry is not None:+ telemetry.preflight_ms += (time.perf_counter() - started) * 1000.0++ def _prepare_host_filter(+ self,+ filters: Mapping[str, Any],+ native_filter_resolver: NativeFilterResolver,+ native_filter_layout_registrar: Callable[[Sequence[int]], None],+ telemetry: Optional[CuVSSearchTelemetry] = None,+ ) -> Optional[_PreparedHostFilter]:+ """Resolve and cache a generation-bound native bitmap without GPU admission."""- generation = self._ensure_native_filter_layout(native_filter_layout_registrar)- if generation is None:+ cache_key = self._filter_cache_key(filters)+ with self._lock:+ if self._closed:return None- resolved = self._resolve_native_filter(filters, native_filter_resolver)+ generation = self._records_generationif telemetry is not None:- telemetry.eligible_count = resolved.eligible_count+ telemetry.filter_kind = (+ "path"+ if _filter_uses_field_type(filters, self.field_types, "path")+ else "scalar"+ )+ telemetry.records_generation = generation+ telemetry.index_size = len(self._records)+ cached = self._get_cached_filter(cache_key)+ if cached is not None:+ if telemetry is not None:+ telemetry.filter_cache_hit = True+ telemetry.eligible_count = cached.eligible_count+ telemetry.filter_words_packed = cached.filter_words_packed+ return _PreparedHostFilter(+ generation,+ cache_key,+ cached_metadata=self._cached_filter_metadata(cached),+ )+ resolved = self._get_preflight_filter(cache_key)+ if resolved is not None:+ if telemetry is not None:+ telemetry.filter_cache_hit = True+ telemetry.eligible_count = resolved.eligible_count+ telemetry.filter_words_packed = resolved.filter_words_packed+ return _PreparedHostFilter(+ generation,+ cache_key,+ resolved_native_filter=resolved,+ )++ # Keep the established _filter_layout_lock -> _lock order. The native+ # resolver can then run unlocked against this registered row layout.+ generation = self._ensure_native_filter_layout(native_filter_layout_registrar)+ if generation is None:+ return None+ flight_key: Optional[Tuple[int, str]] = None+ flight: Optional[_NativeFilterPreflightFlight] = None+ is_owner = True+ try:with self._lock:- if self._records_generation != generation:+ if self._closed or self._records_generation != generation:return Nonecached = self._get_cached_filter(cache_key)if cached is not None:if telemetry is not None:telemetry.filter_cache_hit = Truetelemetry.eligible_count = cached.eligible_count- return cached.eligible_count if cached.route_native else None- if not resolved.route_native and resolved.eligible_count != 0:- self._cache_preflight_filter(cache_key, resolved)+ telemetry.filter_words_packed = cached.filter_words_packed+ return _PreparedHostFilter(+ generation,+ cache_key,+ cached_metadata=self._cached_filter_metadata(cached),+ )+ resolved = self._get_preflight_filter(cache_key)+ if resolved is not None:+ if telemetry is not None:+ telemetry.filter_cache_hit = True+ telemetry.eligible_count = resolved.eligible_count+ telemetry.filter_words_packed = resolved.filter_words_packed+ return _PreparedHostFilter(+ generation,+ cache_key,+ resolved_native_filter=resolved,+ )+ if cache_key is not None:+ flight_key = (generation, cache_key)+ flight = self._preflight_flights.get(flight_key)+ if flight is None:+ flight = _NativeFilterPreflightFlight(generation=generation)+ self._preflight_flights[flight_key] = flight+ else:+ flight.participants += 1+ is_owner = False++ if not is_owner:+ # Mutation and close wake waiters through _cancel_preflight_flights_.+ # Never wait while holding _lock.+ flight.done.wait()+ with self._lock:+ stale = flight.stale or self._closed or self._records_generation != generation+ error = flight.error+ cached = flight.cached_metadata+ resolved = flight.resolved_native_filter+ if stale:return None- cached = _CachedFilter(- prepared=None,- eligible_count=resolved.eligible_count,- route_native=resolved.route_native,- native_threshold=resolved.native_threshold,- native_filter_token=resolved.native_filter_token,+ if error is not None:+ raise error+ if telemetry is not None:+ telemetry.filter_cache_hit = True+ telemetry.eligible_count = flight.eligible_count+ telemetry.filter_words_packed = bool(+ cached.filter_words_packed+ if cached is not None+ else resolved is not None and resolved.filter_words_packed+ )+ return _PreparedHostFilter(+ generation,+ cache_key,+ cached_metadata=cached,+ resolved_native_filter=resolved,)- self._cache_filter(cache_key, cached)- return cached.eligible_count if cached.route_native else None- finally:- if telemetry is not None:- telemetry.preflight_ms += (time.perf_counter() - started) * 1000.0++ resolved = self._resolve_native_filter(filters, native_filter_resolver)+ with self._lock:+ stale = (+ self._closed+ or self._records_generation != generation+ or (flight is not None and flight.stale)+ )+ if flight is not None and self._preflight_flights.get(flight_key) is flight:+ self._preflight_flights.pop(flight_key, None)+ if stale:+ if flight is not None:+ flight.stale = True+ flight.done.set()+ return None+ route_count, eligible_count, cache_hit = self._store_preflight_route_(+ cache_key,+ resolved,+ )+ cached = self._get_cached_filter(cache_key)+ cached_metadata = (+ self._cached_filter_metadata(cached) if cached is not None else None+ )+ if flight is not None:+ flight.route_count = route_count+ flight.eligible_count = eligible_count+ flight.cached_metadata = cached_metadata+ flight.resolved_native_filter = None if cached is not None else resolved+ flight.done.set()+ except BaseException as exc:+ if is_owner and flight is not None:+ with self._lock:+ if self._preflight_flights.get(flight_key) is flight:+ self._preflight_flights.pop(flight_key, None)+ if not flight.stale:+ flight.error = exc+ flight.done.set()+ raise++ if telemetry is not None:+ telemetry.filter_cache_hit = cache_hit+ telemetry.eligible_count = eligible_count+ telemetry.filter_words_packed = (+ cached_metadata.filter_words_packed+ if cached_metadata is not None+ else resolved.filter_words_packed+ )+ return _PreparedHostFilter(+ generation,+ cache_key,+ cached_metadata=cached_metadata,+ resolved_native_filter=None if cached is not None else resolved,+ )def _resolve_native_filter(self,filters: Mapping[str, Any],native_filter_resolver: NativeFilterResolver,) -> _ResolvedNativeFilter:+ with self._lock:+ projection_generation = self._records_generation+ projection_layout_generation = self._filter_layout_generation+ projection_row_count = len(self._records)evaluation = native_filter_resolver(filters)if len(evaluation) == 2:words, eligible_count = evaluationnative_filter_token = 0else:words, eligible_count, native_filter_token = evaluation+ if isinstance(words, bytes):+ if len(words) % _U32_BYTES != 0:+ raise ValueError("Packed native filter bitmap length must be a multiple of 4 bytes")+ bitset_words: StoredNativeFilterWords = words+ bitset_word_count = len(words) // _U32_BYTES+ else:+ bitset_words = tuple(int(word) for word in words)+ bitset_word_count = len(bitset_words)+ eligible_count = int(eligible_count)native_threshold = self.native_filter_threshold(filters)route_native = (self.auto_memory and native_threshold > 0 and eligible_count <= native_threshold)+ if not route_native and eligible_count > 0:+ # Resolver work intentionally runs outside the records lock. Only+ # enforce the ABI against the same registered layout snapshot;+ # callers discard the result when a concurrent mutation wins.+ with self._lock:+ projection_is_stable = (+ projection_layout_generation == projection_generation+ and self._records_generation == projection_generation+ and self._filter_layout_generation == projection_generation+ )+ required_words = (projection_row_count + 31) // 32+ if projection_is_stable and bitset_word_count < required_words:+ raise RuntimeError(+ "Native filter resolver returned an incomplete bitset for GPU routing: "+ f"got {bitset_word_count} words for {projection_row_count} rows "+ f"(expected at least {required_words})"+ )return _ResolvedNativeFilter(- bitset_words=tuple(int(word) for word in words),- eligible_count=int(eligible_count),+ bitset_words=bitset_words,+ eligible_count=eligible_count,route_native=route_native,native_threshold=native_threshold,native_filter_token=int(native_filter_token),+ filter_words_packed=isinstance(bitset_words, bytes),)def _prepare_filter(@@ -997,6 +1446,7 @@ class CuVSDenseIndex:if cached is not None:return cached+ resolved: Optional[_ResolvedNativeFilter] = Noneif native_filter_resolver is not None:resolved = resolved_native_filter or self._resolve_native_filter(filters, native_filter_resolver@@ -1012,7 +1462,18 @@ class CuVSDenseIndex:prepared = prepare_filter_words(resolved.bitset_words)else:prepared = tuple(- bool(resolved.bitset_words[row // 32] & (1 << (row % 32)))+ bool(+ (+ struct.unpack_from(+ "<I",+ resolved.bitset_words,+ (row // 32) * _U32_BYTES,+ )[0]+ if isinstance(resolved.bitset_words, bytes)+ else resolved.bitset_words[row // 32]+ )+ & (1 << (row % 32))+ )for row in range(len(labels)))else:@@ -1035,11 +1496,8 @@ class CuVSDenseIndex:eligible_count=eligible_count,route_native=route_native,native_threshold=native_threshold,- native_filter_token=(- resolved_native_filter.native_filter_token- if resolved_native_filter is not None- else 0- ),+ native_filter_token=(resolved.native_filter_token if resolved is not None else 0),+ filter_words_packed=(resolved.filter_words_packed if resolved is not None else False),)self._cache_filter(cache_key, cached)return cached@@ -1119,7 +1577,10 @@ class CuVSDenseIndex:generation=generation,)labels = tuple(self._records)- dataset = [self._records[label].vector for label in labels]+ dataset = _PackedFP32Rows(+ tuple(self._records[label].vector for label in labels),+ self.dimension,+ )with _CUVS_MEMORY_COORDINATOR.build_lock(self._runtime):# Admission is checked again while the per-device build lock is@@ -1228,35 +1689,95 @@ class CuVSDenseIndex:) -> Tuple[List[int], List[float]]:if limit <= 0:return [], []- queue_started = time.perf_counter()- self._gpu_search_gate.acquire()- if telemetry is not None:- telemetry.queue_ms += (time.perf_counter() - queue_started) * 1000.0- try:- return self._search_admitted(- query_vector,- limit,- filters,- native_filter_resolver,- native_filter_layout_registrar,- telemetry,- )- finally:- self._gpu_search_gate.release()+ query = self._prepare_vector(query_vector)+ while True:+ prepared_filter: Optional[_PreparedHostFilter] = None+ if (+ filters+ and native_filter_resolver is not None+ and native_filter_layout_registrar is not None+ ):+ filter_started = time.perf_counter()+ try:+ prepared_filter = self._prepare_host_filter(+ filters,+ native_filter_resolver,+ native_filter_layout_registrar,+ telemetry,+ )+ finally:+ if telemetry is not None:+ telemetry.filter_prepare_ms += (+ time.perf_counter() - filter_started+ ) * 1000.0+ if prepared_filter is None:+ with self._lock:+ if self._closed:+ raise RuntimeError("cuVS dense index is closed")+ # A mutation invalidated the registered native layout while+ # the resolver ran. Retry before consuming GPU admission.+ continue+ cached = prepared_filter.cached_metadata+ resolved = prepared_filter.resolved_native_filter+ if cached is None and resolved is None:+ raise RuntimeError("Native filter preparation completed without a result")+ eligible_count = (+ cached.eligible_count if cached is not None else resolved.eligible_count+ )+ route_native = cached.route_native if cached is not None else resolved.route_native+ native_threshold = (+ cached.native_threshold if cached is not None else resolved.native_threshold+ )+ if eligible_count == 0:+ return [], []+ if route_native:+ raise CuVSNativeRouteError(+ "cuVS auto mode routed a selective filter to native search "+ f"({eligible_count} candidates <= {native_threshold})"+ )++ queue_started = time.perf_counter()+ self._gpu_search_gate.acquire()+ waited_ms = (time.perf_counter() - queue_started) * 1000.0+ if telemetry is not None:+ # queue_ms remains the aggregate of all lock/admission waits;+ # gpu_gate_queue_ms isolates only this device-search permit.+ telemetry.queue_ms += waited_ms+ telemetry.gpu_gate_queue_ms += waited_ms+ try:+ try:+ return self._search_admitted(+ query,+ limit,+ filters,+ native_filter_resolver,+ native_filter_layout_registrar,+ prepared_filter,+ telemetry,+ )+ except _StalePreparedFilter:+ continue+ finally:+ self._gpu_search_gate.release()def _search_admitted(self,- query_vector: Sequence[float],+ query: Sequence[float],limit: int,filters: Optional[Mapping[str, Any]],native_filter_resolver: Optional[NativeFilterResolver] = None,native_filter_layout_registrar: Optional[Callable[[Sequence[int]], None]] = None,+ prepared_filter: Optional[_PreparedHostFilter] = None,telemetry: Optional[CuVSSearchTelemetry] = None,) -> Tuple[List[int], List[float]]:- query = self._prepare_vector(query_vector)- if self.auto_memory and native_filter_layout_registrar is not None:- self._ensure_native_filter_layout(native_filter_layout_registrar)with self._lock:+ if self._closed:+ raise RuntimeError("cuVS dense index is closed")+ if (+ prepared_filter is not None+ and prepared_filter.generation != self._records_generation+ ):+ raise _StalePreparedFilterif telemetry is not None:telemetry.filter_kind = ("path"@@ -1268,24 +1789,45 @@ class CuVSDenseIndex:telemetry.records_generation = self._records_generationtelemetry.index_size = len(self._records)cached_filter: Optional[_CachedFilter] = None- resolved_native_filter: Optional[_ResolvedNativeFilter] = None+ resolved_native_filter = (+ prepared_filter.resolved_native_filter if prepared_filter is not None else None+ )+ if prepared_filter is not None and prepared_filter.cached_metadata is not None:+ cached_filter = self._get_cached_filter(prepared_filter.cache_key)+ if cached_filter is None:+ # The device LRU entry was evicted after host preparation.+ # This is the rare guaranteed-progress fallback: leave the+ # cached/resolved values empty so _prepare_filter resolves+ # and materializes this filter once inside the gate. The+ # common cache-miss path still resolves before admission.+ resolved_native_filter = None+ if telemetry is not None:+ telemetry.filter_cache_hit = False+ telemetry.filter_cache_eviction_fallback = Truefilter_layout_is_current = self._filter_layout_generation == self._records_generation# Auto mode decides whether a selective filter should remain native# before paying GPU admission or rebuild costs. A dirty native# layout is refreshed against the pending cuVS row order only; the# live GPU row mapping is not changed until a build actually runs.- if filters and self.auto_memory and native_filter_resolver is not None:+ if (+ filters+ and self.auto_memory+ and native_filter_resolver is not None+ and prepared_filter is None+ ):cache_key = self._filter_cache_key(filters)cached_filter = self._get_cached_filter(cache_key)if cached_filter is not None and telemetry is not None:telemetry.filter_cache_hit = Truetelemetry.eligible_count = cached_filter.eligible_count+ telemetry.filter_words_packed = cached_filter.filter_words_packedif cached_filter is None:resolved_native_filter = self._get_preflight_filter(cache_key)if resolved_native_filter is not None and telemetry is not None:telemetry.filter_cache_hit = Truetelemetry.eligible_count = resolved_native_filter.eligible_count+ telemetry.filter_words_packed = resolved_native_filter.filter_words_packedif resolved_native_filter is None:if (self._dirty@@ -1298,6 +1840,10 @@ class CuVSDenseIndex:resolved_native_filter = self._resolve_native_filter(filters, native_filter_resolver)+ if telemetry is not None:+ telemetry.filter_words_packed = (+ resolved_native_filter.filter_words_packed+ )if (resolved_native_filter.route_nativeor resolved_native_filter.eligible_count == 0@@ -1308,6 +1854,7 @@ class CuVSDenseIndex:route_native=resolved_native_filter.route_native,native_threshold=resolved_native_filter.native_threshold,native_filter_token=resolved_native_filter.native_filter_token,+ filter_words_packed=resolved_native_filter.filter_words_packed,)self._cache_filter(cache_key, cached_filter)@@ -1336,6 +1883,7 @@ class CuVSDenseIndex:cached_filter = self._get_cached_filter(self._filter_cache_key(filters))if cached_filter is not None and telemetry is not None:telemetry.filter_cache_hit = True+ telemetry.filter_words_packed = cached_filter.filter_words_packedcached_filter = cached_filter or self._prepare_filter(filters,snapshot.labels,@@ -1345,6 +1893,7 @@ class CuVSDenseIndex:if telemetry is not None:telemetry.filter_prepare_ms += (time.perf_counter() - filter_started) * 1000.0telemetry.eligible_count = cached_filter.eligible_count+ telemetry.filter_words_packed = cached_filter.filter_words_packedmask = cached_filter.preparedeligible_count = cached_filter.eligible_countif eligible_count == 0:@@ -1399,10 +1948,23 @@ class CuVSDenseIndex:def close(self) -> None:with self._lock:+ if self._close_complete:+ return+ if not self._closed:+ self._closed = True+ self._records_generation += 1+ self._filter_layout_generation = -1+ self._cancel_preflight_flights_()while self._active_searches:self._idle_condition.wait()with self._runtime_device_scope():self._snapshot = Noneself._filter_cache.clear()self._preflight_filter_cache.clear()- self._runtime.close()+ try:+ self._runtime.close()+ finally:+ # A failed recovery can otherwise keep this partially packed+ # host shadow alive through the constructor traceback.+ self._records.clear()+ self._close_complete = Truediff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex 9f7af96cd..f33949099 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -9,7 +9,7 @@ import threadingimport timefrom contextlib import contextmanagerfrom pathlib import Path-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Unionimport openviking.storage.vectordb.engine as enginefrom openviking.storage.vectordb.index.cuvs_index import (@@ -279,6 +279,51 @@ class IndexEngineProxy:)return result.bitset_words, result.eligible_count, result.native_filter_token+ def evaluate_filter_for_routing(+ self,+ filters: Dict[str, Any],+ native_threshold: int,+ ) -> Tuple[List[int], int, int]:+ """Evaluate only the projection needed for an adaptive route decision."""++ if not self.index_engine:+ raise RuntimeError("Index engine not initialized")+ result = self.index_engine.evaluate_filter_for_routing(+ json.dumps(filters),+ native_threshold=native_threshold,+ )+ return result.bitset_words, result.eligible_count, result.native_filter_token++ def evaluate_filter_packed(+ self,+ filters: Dict[str, Any],+ max_cached_candidates: int = 0,+ ) -> Tuple[Union[List[int], bytes], int, int]:+ """Evaluate a cuVS filter using packed words when the engine supports it."""++ if not self.index_engine:+ raise RuntimeError("Index engine not initialized")+ result = self.index_engine.evaluate_filter_packed(+ json.dumps(filters),+ max_cached_candidates=max_cached_candidates,+ )+ return result.bitset_words, result.eligible_count, result.native_filter_token++ def evaluate_filter_for_routing_packed(+ self,+ filters: Dict[str, Any],+ native_threshold: int,+ ) -> Tuple[Union[List[int], bytes], int, int]:+ """Evaluate a cuVS route using packed words when available."""++ if not self.index_engine:+ raise RuntimeError("Index engine not initialized")+ result = self.index_engine.evaluate_filter_for_routing_packed(+ json.dumps(filters),+ native_threshold=native_threshold,+ )+ return result.bitset_words, result.eligible_count, result.native_filter_token+def drop(self):"""Release the index engine resources.@@ -310,7 +355,7 @@ class LocalIndex(IIndex):index_path_or_json: str,meta: Any,dense_search_config: Optional[Dict[str, Any]] = None,- initial_candidates: Optional[List[CandidateData]] = None,+ initial_candidates: Optional[Iterable[CandidateData]] = None,defer_dense_rebuild_start: bool = False,):"""Initialize a local index instance.@@ -319,7 +364,7 @@ class LocalIndex(IIndex):index_path_or_json (str): Path to index files or JSON configurationmeta: Index metadata object containing configurationdense_search_config: Optional dense-search backend configuration.- initial_candidates: Records used to initialize the dense-search shadow state.+ initial_candidates: Records consumed to initialize the dense-search shadow state.defer_dense_rebuild_start: Delay the background rebuild worker until thecaller has finished initializing the native index."""@@ -352,6 +397,7 @@ class LocalIndex(IIndex):dense_search_backend = dense_search_config.get("backend")if dense_search_backend in {"cuvs", "auto_cuvs"}:self._auto_cuvs = dense_search_backend == "auto_cuvs"+ candidate_iterable = initial_candidates if initial_candidates is not None else ()vector_meta = meta.inner_meta.get("VectorIndex", {})field_types = {name: DataProcessor.normalize_field_type(field_meta.get("FieldType", ""))@@ -366,7 +412,7 @@ class LocalIndex(IIndex):config=dense_search_config,auto_memory=self._auto_cuvs,)- self.dense_search.add_candidates(initial_candidates or [])+ self.dense_search.add_candidates(candidate_iterable)self._auto_background_rebuild = self._auto_cuvs and bool(dense_search_config.get("auto_background_rebuild", False))@@ -374,10 +420,34 @@ class LocalIndex(IIndex):max(0, int(dense_search_config.get("auto_rebuild_debounce_ms", 500))) / 1000.0)except CuVSUnavailableError:+ failed_dense_search = self.dense_search+ self.dense_search = None+ if failed_dense_search is not None:+ try:+ failed_dense_search.close()+ except Exception:+ logger.warning(+ "Failed to close unavailable cuVS dense search", exc_info=True+ )if not self._auto_cuvs:raiselogger.info("cuVS auto mode unavailable; keeping native dense search")+ except Exception:+ failed_dense_search = self.dense_searchself.dense_search = None+ if failed_dense_search is not None:+ try:+ failed_dense_search.close()+ except Exception:+ logger.warning("Failed to close partial cuVS dense search", exc_info=True)+ raise+ finally:+ close_candidates = getattr(candidate_iterable, "close", None)+ if callable(close_candidates):+ try:+ close_candidates()+ except Exception:+ logger.warning("Failed to close dense recovery iterator", exc_info=True)if not defer_dense_rebuild_start:self._start_dense_rebuild_worker()@@ -783,7 +853,7 @@ class LocalIndex(IIndex):) * 1000.0native_count = self.dense_search.preflight_native_count(filters,- self._evaluate_cuvs_filter,+ self._evaluate_cuvs_filter_for_routing,self.engine_proxy.set_filter_layout,telemetry=cuvs_telemetry,)@@ -879,6 +949,11 @@ class LocalIndex(IIndex):) -> Tuple[List[int], List[float]]:if self.dense_search is None or self.engine_proxy is None:raise RuntimeError("cuVS search requires an initialized index")+ filter_resolver = (+ self._evaluate_cuvs_filter_for_routing+ if self._auto_cuvs+ else self._evaluate_cuvs_filter+ )if self._auto_background_rebuild:queue_started = time.perf_counter()with self._dense_search_lock.read():@@ -892,7 +967,7 @@ class LocalIndex(IIndex):query_vector,limit,filters,- self._evaluate_cuvs_filter,+ filter_resolver,self.engine_proxy.set_filter_layout,telemetry=telemetry,)@@ -906,7 +981,7 @@ class LocalIndex(IIndex):query_vector,limit,filters,- self._evaluate_cuvs_filter,+ filter_resolver,self.engine_proxy.set_filter_layout,telemetry=telemetry,)@@ -920,7 +995,7 @@ class LocalIndex(IIndex):query_vector,limit,filters,- self._evaluate_cuvs_filter,+ filter_resolver,self.engine_proxy.set_filter_layout,telemetry=telemetry,)@@ -962,14 +1037,26 @@ class LocalIndex(IIndex):if telemetry.route_reason == "pending":telemetry.route_reason = "native_fallback"- def _evaluate_cuvs_filter(self, filters: Dict[str, Any]) -> Tuple[List[int], int, int]:+ def _evaluate_cuvs_filter(+ self, filters: Dict[str, Any]+ ) -> Tuple[Union[List[int], bytes], int, int]:if self.dense_search is None or self.engine_proxy is None:raise RuntimeError("cuVS filter evaluation requires an initialized index")- return self.engine_proxy.evaluate_filter(+ return self.engine_proxy.evaluate_filter_packed(filters,max_cached_candidates=self.dense_search.native_filter_threshold(filters),)+ def _evaluate_cuvs_filter_for_routing(+ self, filters: Dict[str, Any]+ ) -> Tuple[Union[List[int], bytes], int, int]:+ if self.dense_search is None or self.engine_proxy is None:+ raise RuntimeError("cuVS filter evaluation requires an initialized index")+ return self.engine_proxy.evaluate_filter_for_routing_packed(+ filters,+ native_threshold=self.dense_search.native_filter_threshold(filters),+ )+@staticmethoddef _cuvs_telemetry_enabled() -> bool:if logger.isEnabledFor(logging.DEBUG):@@ -1247,7 +1334,7 @@ class PersistentIndex(LocalIndex):name: str,meta: Any,path: str,- cands_list: Optional[List[CandidateData]] = None,+ cands_list: Optional[Iterable[CandidateData]] = None,force_rebuild: bool = False,initial_timestamp: Optional[int] = None,dense_search_config: Optional[Dict[str, Any]] = None,@@ -1262,7 +1349,9 @@ class PersistentIndex(LocalIndex):name (str): Name identifier for the index (used as subdirectory name)meta: Index metadata containing configurationpath (str): Parent directory path where index data will be stored- cands_list (list): Initial data for creating a new index. Defaults to None.+ cands_list: Initial records for a new native index or dense-search shadow.+ Existing native snapshots consume this iterable only when the configured+ dense-search backend needs to rehydrate its shadow state.force_rebuild (bool): If True, rebuilds the index even if it exists.Defaults to False.initial_timestamp (Optional[int]): Timestamp to use if creating a new index@@ -1282,7 +1371,7 @@ class PersistentIndex(LocalIndex):- Apply any pending delta updates from collection"""if cands_list is None:- cands_list = []+ cands_list = ()validate_name_str(name)self.index_dir = str(safe_join_name(path, name))@@ -1294,6 +1383,11 @@ class PersistentIndex(LocalIndex):# At this point, there is no index, need to create a new oneif not newest_version or force_rebuild:+ # Building a new native index needs both len() and another pass when+ # initializing an optional dense shadow. Recovery of an existing+ # snapshot stays single-pass and does not take this fallback.+ if not isinstance(cands_list, list):+ cands_list = list(cands_list)self._create_new_index(name, meta, cands_list, initial_timestamp)else:self.now_version = str(newest_version)diff --git a/openviking/storage/vectordb/store/local_store.py b/openviking/storage/vectordb/store/local_store.pyindex 9f3728930..7c6b16183 100644--- a/openviking/storage/vectordb/store/local_store.py+++ b/openviking/storage/vectordb/store/local_store.py@@ -1,6 +1,6 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-from typing import List, Tuple, Union+from typing import Iterator, List, Tuple, Unionimport openviking.storage.vectordb.engine as enginefrom openviking.storage.vectordb.store.store import BatchOp, IMutiTableStore, Op, OpType@@ -8,6 +8,8 @@ from openviking.storage.vectordb.utils.stale_lock import clean_stale_rocksdb_loc# Constant for the maximum Unicode character, used for range queries to cover all possible keysMAX_UNICODE_CHAR = "\U0010ffff"+STORE_SCAN_PAGE_SIZE = 1024+STORE_SCAN_PAGE_BYTES = 32 * 1024 * 1024def create_store_engine_proxy(path: str = "") -> "StoreEngineProxy":@@ -107,6 +109,60 @@ class StoreEngineProxy(IMutiTableStore):if data[0].startswith(table_name)]+ def iter_all(+ self,+ table_name: str,+ page_size: int = STORE_SCAN_PAGE_SIZE,+ page_bytes: int = STORE_SCAN_PAGE_BYTES,+ ) -> Iterator[Tuple[str, bytes]]:+ """Scan a table in bounded native pages.++ Pages are capped by row count and ``max(byte budget, one encoded row)``+ so an oversized row cannot cause a false end-of-scan. Each native+ page is internally consistent, but separate page calls do not share a+ snapshot; collection recovery invokes this while the store is quiescent.+ """+ if page_size <= 0:+ raise ValueError("Store scan page size must be positive")+ if page_bytes <= 0:+ raise ValueError("Store scan page byte budget must be positive")++ seek_page = getattr(self.storage_engine, "seek_range_page", None)+ if not callable(seek_page):+ raise RuntimeError(+ "The native VectorDB engine does not support bounded store scans; "+ "rebuild or reinstall the matching OpenViking engine package"+ )++ end_key = table_name + MAX_UNICODE_CHAR+ cursor = table_name+ start_exclusive = False+ try:+ page = seek_page(cursor, end_key, page_size, page_bytes, start_exclusive)+ except NotImplementedError as exc:+ raise RuntimeError(+ "The native VectorDB engine does not support bounded store scans; "+ "rebuild or reinstall the matching OpenViking engine package"+ ) from exc++ while page:+ next_cursor = page[-1][0]+ if next_cursor < cursor or (start_exclusive and next_cursor <= cursor):+ raise RuntimeError(+ "Native bounded store scan did not advance its continuation cursor"+ )+ for full_key, value in page:+ if full_key.startswith(table_name):+ yield full_key[len(table_name) :], value+ # Drop the page before the next native call so two encoded-vector+ # pages are not live at the same time during recovery.+ page = None+ full_key = None+ value = None+ cursor = next_cursor+ start_exclusive = True+ page = seek_page(cursor, end_key, page_size, page_bytes, start_exclusive)+def begin_to_seek(self, end_key: str, table_name: str) -> List[Tuple[str, bytes]]:"""Retrieve all entries from the beginning to a specific key.diff --git a/openviking/storage/vectordb/store/store.py b/openviking/storage/vectordb/store/store.pyindex efbeefc0d..54580b124 100644--- a/openviking/storage/vectordb/store/store.py+++ b/openviking/storage/vectordb/store/store.py@@ -2,7 +2,7 @@# SPDX-License-Identifier: AGPL-3.0from abc import ABC, abstractmethodfrom enum import Enum-from typing import Any, List, Tuple, Union+from typing import Any, Iterator, List, Tuple, Unionclass IKVStore(ABC):@@ -189,6 +189,15 @@ class IMutiTableStore(ABC):"""pass+ def iter_all(self, table_name: str) -> Iterator[Tuple[str, bytes]]:+ """Iterate over a table, with a materializing compatibility fallback.++ Store implementations with a cursor or paged range API should override+ this method. Keeping the default in terms of ``read_all`` preserves+ existing third-party and test-store behavior.+ """+ yield from self.read_all(table_name)+@abstractmethoddef seek_to_end(self, key: str, table_name: str) -> List[Tuple[str, bytes]]:"""Retrieve all entries from a starting key to the end of the table.diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.pyindex 6a14d5f90..10c6b4c3a 100644--- a/openviking/storage/vectordb/store/store_manager.py+++ b/openviking/storage/vectordb/store/store_manager.py@@ -1,7 +1,7 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0import time-from typing import List, Optional, Tuple+from typing import Iterator, List, Optional, Tuplefrom openviking.storage.vectordb.store.data import CandidateData, DeltaRecordfrom openviking.storage.vectordb.store.local_store import create_store_engine_proxy@@ -225,9 +225,17 @@ class StoreManager:Returns:List[CandidateData]: List of all candidate data objects."""- cands_kv_list = self.storage.read_all(StoreManager.CandsTable)- cands_list = [CandidateData.from_bytes(data[1]) for data in cands_kv_list]- return cands_list+ return list(self.iter_all_cands_data())++ def iter_all_cands_data(self) -> Iterator[CandidateData]:+ """Iterate candidates without retaining deserialized full tables.++ The local native store also bounds encoded rows by page. Generic store+ implementations retain the ``IMutiTableStore.iter_all`` compatibility+ fallback and may still materialize their encoded table.+ """+ for _, bytes_data in self.storage.iter_all(StoreManager.CandsTable):+ yield CandidateData.from_bytes(bytes_data)def clear(self):"""Clear all data from the store."""diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 203f6818e..877030eb5 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -9,7 +9,7 @@ import uuidfrom contextlib import asynccontextmanagerfrom typing import Any, AsyncIterator, Dict, List, Optional-from openviking.core.namespace import canonicalize_uri, visible_roots+from openviking.core.namespace import canonicalize_uri, uri_parts, visible_rootsfrom openviking.server.identity import RequestContext, Rolefrom openviking.storage.expr import And, Eq, FilterExpr, In, Or, PathScope, RawDSLfrom openviking.storage.vectordb.collection.collection import Collection@@ -1469,16 +1469,22 @@ class VikingVectorIndexBackend:if context_type:filters.append(Eq("context_type", context_type))+ canonical_targets = [+ canonicalize_uri(target_dir, ctx)+ for target_dir in target_directories or []+ if target_dir+ ]tenant_filter = self._tenant_filter(ctx, context_type=context_type)+ if tenant_filter and self._targets_within_visible_roots(ctx, canonical_targets):+ # The target scopes are already narrower than the tenant-visible+ # roots. Keep account isolation, but avoid recursively evaluating+ # the broader path scopes as an additional filter.+ tenant_filter = Eq("account_id", ctx.account_id)if tenant_filter:filters.append(tenant_filter)- if target_directories:- uri_conds = [- PathScope("uri", canonicalize_uri(target_dir, ctx), depth=-1)- for target_dir in target_directories- if target_dir- ]+ if canonical_targets:+ uri_conds = [PathScope("uri", target_dir, depth=-1) for target_dir in canonical_targets]if uri_conds:filters.append(Or(uri_conds))@@ -1493,6 +1499,20 @@ class VikingVectorIndexBackend:return self._merge_filters(*filters)+ @staticmethod+ def _targets_within_visible_roots(ctx: RequestContext, canonical_targets: List[str]) -> bool:+ if not canonical_targets:+ return False++ root_parts = [tuple(uri_parts(root)) for root in visible_roots(ctx)]+ return all(+ any(+ len(target_parts) >= len(root) and target_parts[: len(root)] == root+ for root in root_parts+ )+ for target_parts in (tuple(uri_parts(target)) for target in canonical_targets)+ )+@staticmethoddef _tenant_filter(ctx: RequestContext, context_type: Optional[str] = Nonediff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.pyindex a15b444d8..1a5bfdee4 100644--- a/openviking/telemetry/operation.py+++ b/openviking/telemetry/operation.py@@ -17,6 +17,7 @@ _CUVS_TIMING_FIELDS = ("total_ms","preflight_ms","queue_ms",+ "gpu_gate_queue_ms","build_ms","filter_prepare_ms","gpu_search_ms",@@ -306,6 +307,12 @@ class TelemetrySummaryBuilder:"routes": cls._counter_breakdown("vector.cuvs.routes", counters),"filter_kinds": cls._counter_breakdown("vector.cuvs.filter_kinds", counters),"filter_cache_hits": cls._i(counters.get("vector.cuvs.filter_cache_hits"), 0),+ "filter_cache_eviction_fallbacks": cls._i(+ counters.get("vector.cuvs.filter_cache_eviction_fallbacks"), 0+ ),+ "packed_filter_queries": cls._i(+ counters.get("vector.cuvs.packed_filter_queries"), 0+ ),"native_filter_reuses": cls._i(counters.get("vector.cuvs.native_filter_reuses"), 0),@@ -499,6 +506,10 @@ class OperationTelemetry:self._counters["vector.cuvs.auto_mode_searches"] += 1if TelemetrySummaryBuilder._bool(metrics.get("filter_cache_hit"), False):self._counters["vector.cuvs.filter_cache_hits"] += 1+ if TelemetrySummaryBuilder._bool(metrics.get("filter_cache_eviction_fallback"), False):+ self._counters["vector.cuvs.filter_cache_eviction_fallbacks"] += 1+ if TelemetrySummaryBuilder._bool(metrics.get("filter_words_packed"), False):+ self._counters["vector.cuvs.packed_filter_queries"] += 1if TelemetrySummaryBuilder._bool(metrics.get("native_filter_reused"), False):self._counters["vector.cuvs.native_filter_reuses"] += 1if TelemetrySummaryBuilder._bool(metrics.get("build_performed"), False):diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cppindex 6ae2ac906..d2a893442 100644--- a/src/abi3_engine_backend.cpp+++ b/src/abi3_engine_backend.cpp@@ -1179,6 +1179,79 @@ PyObject* build_filter_result(const vdb::FilterResult& result) {return payload;}+PyObject* build_packed_filter_result(const vdb::FilterResult& result) {+ PyObject* payload = PyDict_New();+ if (payload == nullptr) {+ return nullptr;+ }++ if (result.bitset_words.size() >+ static_cast<size_t>(PY_SSIZE_T_MAX) / sizeof(uint32_t)) {+ Py_DECREF(payload);+ PyErr_SetString(PyExc_OverflowError, "Packed filter bitmap is too large");+ return nullptr;+ }++ // Encode explicitly instead of exposing the host byte order. This ABI is+ // consumed with NumPy dtype "<u4" before the bitmap is copied to a device.+ const Py_ssize_t packed_size = static_cast<Py_ssize_t>(+ result.bitset_words.size() * sizeof(uint32_t));+ PyObject* bitset_words_le =+ PyBytes_FromStringAndSize(nullptr, packed_size);+ if (bitset_words_le == nullptr) {+ Py_DECREF(payload);+ return nullptr;+ }+ char* packed_words = PyBytes_AsString(bitset_words_le);+ if (packed_words == nullptr) {+ Py_DECREF(bitset_words_le);+ Py_DECREF(payload);+ return nullptr;+ }+ auto pack_words = [&]() {+ for (size_t index = 0; index < result.bitset_words.size(); ++index) {+ const uint32_t word = result.bitset_words[index];+ const size_t offset = index * sizeof(uint32_t);+ packed_words[offset] = static_cast<char>(word & 0xffU);+ packed_words[offset + 1] = static_cast<char>((word >> 8U) & 0xffU);+ packed_words[offset + 2] = static_cast<char>((word >> 16U) & 0xffU);+ packed_words[offset + 3] = static_cast<char>((word >> 24U) & 0xffU);+ }+ };+ if (result.bitset_words.size() >= 256) {+ call_without_gil(pack_words);+ } else {+ pack_words();+ }++ PyObject* eligible_count =+ PyLong_FromUnsignedLongLong(result.eligible_count);+ PyObject* native_filter_token =+ PyLong_FromUnsignedLongLong(result.native_filter_token);+ if (eligible_count == nullptr || native_filter_token == nullptr) {+ Py_XDECREF(eligible_count);+ Py_DECREF(bitset_words_le);+ Py_XDECREF(native_filter_token);+ Py_DECREF(payload);+ return nullptr;+ }++ if (PyDict_SetItemString(payload, "eligible_count", eligible_count) < 0 ||+ PyDict_SetItemString(payload, "bitset_words_le", bitset_words_le) < 0 ||+ PyDict_SetItemString(payload, "native_filter_token",+ native_filter_token) < 0) {+ Py_DECREF(eligible_count);+ Py_DECREF(bitset_words_le);+ Py_DECREF(native_filter_token);+ Py_DECREF(payload);+ return nullptr;+ }+ Py_DECREF(eligible_count);+ Py_DECREF(bitset_words_le);+ Py_DECREF(native_filter_token);+ return payload;+}+PyObject* build_state_result(const vdb::StateResult& result) {PyObject* payload = PyDict_New();if (payload == nullptr) {@@ -1403,6 +1476,28 @@ PyObject* py_index_engine_evaluate_filter(PyObject*, PyObject* args) {}}+PyObject* py_index_engine_evaluate_filter_packed(PyObject*, PyObject* args) {+ PyObject* capsule = nullptr;+ const char* dsl = nullptr;+ if (!PyArg_ParseTuple(args, "Os", &capsule, &dsl)) {+ return nullptr;+ }++ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);+ if (engine == nullptr) {+ return nullptr;+ }++ try {+ const vdb::FilterResult result =+ call_without_gil([&]() { return engine->evaluate_filter(dsl); });+ return build_packed_filter_result(result);+ } catch (const std::exception& exc) {+ raise_runtime_error(exc.what());+ return nullptr;+ }+}+PyObject* py_index_engine_evaluate_filter_cached(PyObject*, PyObject* args) {PyObject* capsule = nullptr;const char* dsl = nullptr;@@ -1429,6 +1524,85 @@ PyObject* py_index_engine_evaluate_filter_cached(PyObject*, PyObject* args) {}}+PyObject* py_index_engine_evaluate_filter_cached_packed(PyObject*,+ PyObject* args) {+ PyObject* capsule = nullptr;+ const char* dsl = nullptr;+ unsigned long long max_cached_candidates = 0;+ if (!PyArg_ParseTuple(args, "OsK", &capsule, &dsl,+ &max_cached_candidates)) {+ return nullptr;+ }++ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);+ if (engine == nullptr) {+ return nullptr;+ }++ try {+ const vdb::FilterResult result = call_without_gil([&]() {+ return engine->evaluate_filter(+ dsl, static_cast<uint64_t>(max_cached_candidates));+ });+ return build_packed_filter_result(result);+ } catch (const std::exception& exc) {+ raise_runtime_error(exc.what());+ return nullptr;+ }+}++PyObject* py_index_engine_evaluate_filter_for_routing(PyObject*,+ PyObject* args) {+ PyObject* capsule = nullptr;+ const char* dsl = nullptr;+ unsigned long long native_threshold = 0;+ if (!PyArg_ParseTuple(args, "OsK", &capsule, &dsl, &native_threshold)) {+ return nullptr;+ }++ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);+ if (engine == nullptr) {+ return nullptr;+ }++ try {+ const vdb::FilterResult result = call_without_gil([&]() {+ return engine->evaluate_filter_for_routing(+ dsl, static_cast<uint64_t>(native_threshold));+ });+ return build_filter_result(result);+ } catch (const std::exception& exc) {+ raise_runtime_error(exc.what());+ return nullptr;+ }+}++PyObject* py_index_engine_evaluate_filter_for_routing_packed(+ PyObject*, PyObject* args) {+ PyObject* capsule = nullptr;+ const char* dsl = nullptr;+ unsigned long long native_threshold = 0;+ if (!PyArg_ParseTuple(args, "OsK", &capsule, &dsl, &native_threshold)) {+ return nullptr;+ }++ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);+ if (engine == nullptr) {+ return nullptr;+ }++ try {+ const vdb::FilterResult result = call_without_gil([&]() {+ return engine->evaluate_filter_for_routing(+ dsl, static_cast<uint64_t>(native_threshold));+ });+ return build_packed_filter_result(result);+ } catch (const std::exception& exc) {+ raise_runtime_error(exc.what());+ return nullptr;+ }+}+PyObject* py_index_engine_dump(PyObject*, PyObject* args) {PyObject* capsule = nullptr;const char* path = nullptr;@@ -1645,6 +1819,33 @@ PyObject* py_store_clear_data(PyObject*, PyObject* args) {}}+PyObject* store_items_to_python(+ const std::vector<std::pair<std::string, std::string>>& items) {+ PyObject* list = PyList_New(static_cast<Py_ssize_t>(items.size()));+ if (list == nullptr) {+ return nullptr;+ }+ for (Py_ssize_t i = 0; i < static_cast<Py_ssize_t>(items.size()); ++i) {+ const auto& item = items[static_cast<size_t>(i)];+ PyObject* tuple = PyTuple_New(2);+ PyObject* key = PyUnicode_FromStringAndSize(+ item.first.data(), static_cast<Py_ssize_t>(item.first.size()));+ PyObject* value = PyBytes_FromStringAndSize(+ item.second.data(), static_cast<Py_ssize_t>(item.second.size()));+ if (tuple == nullptr || key == nullptr || value == nullptr) {+ Py_XDECREF(tuple);+ Py_XDECREF(key);+ Py_XDECREF(value);+ Py_DECREF(list);+ return nullptr;+ }+ PyTuple_SetItem(tuple, 0, key);+ PyTuple_SetItem(tuple, 1, value);+ PyList_SetItem(list, i, tuple);+ }+ return list;+}+PyObject* py_store_seek_range(PyObject*, PyObject* args) {PyObject* capsule = nullptr;const char* start_key = nullptr;@@ -1661,29 +1862,46 @@ PyObject* py_store_seek_range(PyObject*, PyObject* args) {try {const auto items = call_without_gil([&]() { return store->seek_range(start_key, end_key); });- PyObject* list = PyList_New(static_cast<Py_ssize_t>(items.size()));- if (list == nullptr) {- return nullptr;- }- for (Py_ssize_t i = 0; i < static_cast<Py_ssize_t>(items.size()); ++i) {- const auto& item = items[static_cast<size_t>(i)];- PyObject* tuple = PyTuple_New(2);- PyObject* key = PyUnicode_FromStringAndSize(- item.first.data(), static_cast<Py_ssize_t>(item.first.size()));- PyObject* value = PyBytes_FromStringAndSize(- item.second.data(), static_cast<Py_ssize_t>(item.second.size()));- if (tuple == nullptr || key == nullptr || value == nullptr) {- Py_XDECREF(tuple);- Py_XDECREF(key);- Py_XDECREF(value);- Py_DECREF(list);- return nullptr;- }- PyTuple_SetItem(tuple, 0, key);- PyTuple_SetItem(tuple, 1, value);- PyList_SetItem(list, i, tuple);- }- return list;+ return store_items_to_python(items);+ } catch (const std::exception& exc) {+ raise_runtime_error(exc.what());+ return nullptr;+ }+}++PyObject* py_store_seek_range_page(PyObject*, PyObject* args) {+ PyObject* capsule = nullptr;+ const char* start_key = nullptr;+ const char* end_key = nullptr;+ Py_ssize_t limit = 0;+ Py_ssize_t max_bytes = 0;+ int start_exclusive = 0;+ if (!PyArg_ParseTuple(args, "Ossnnp", &capsule, &start_key, &end_key,+ &limit, &max_bytes, &start_exclusive)) {+ return nullptr;+ }+ if (limit <= 0) {+ raise_value_error("seek_range_page limit must be positive");+ return nullptr;+ }+ if (max_bytes <= 0) {+ raise_value_error("seek_range_page max_bytes must be positive");+ return nullptr;+ }++ auto* store = capsule_to_ptr<vdb::KVStore>(capsule, kStoreCapsuleName);+ if (store == nullptr) {+ return nullptr;+ }++ try {+ const auto items = call_without_gil([&]() {+ return store->seek_range_page(start_key, end_key,+ static_cast<size_t>(limit),+ static_cast<size_t>(max_bytes),+ start_exclusive != 0);+ });+ return store_items_to_python(items);} catch (const std::exception& exc) {raise_runtime_error(exc.what());return nullptr;@@ -1722,9 +1940,21 @@ PyMethodDef kModuleMethods[] = {METH_VARARGS, "Register an external label order for scalar filters."},{"_index_engine_evaluate_filter", py_index_engine_evaluate_filter,METH_VARARGS, "Evaluate a scalar filter in an external label order."},+ {"_index_engine_evaluate_filter_packed",+ py_index_engine_evaluate_filter_packed, METH_VARARGS,+ "Evaluate a scalar filter and return little-endian packed words."},{"_index_engine_evaluate_filter_cached",py_index_engine_evaluate_filter_cached, METH_VARARGS,"Evaluate and optionally retain a native scalar filter bitmap."},+ {"_index_engine_evaluate_filter_cached_packed",+ py_index_engine_evaluate_filter_cached_packed, METH_VARARGS,+ "Evaluate and retain a filter while returning packed words."},+ {"_index_engine_evaluate_filter_for_routing",+ py_index_engine_evaluate_filter_for_routing, METH_VARARGS,+ "Evaluate a scalar filter for an adaptive native/GPU route decision."},+ {"_index_engine_evaluate_filter_for_routing_packed",+ py_index_engine_evaluate_filter_for_routing_packed, METH_VARARGS,+ "Route a scalar filter while returning little-endian packed words."},{"_index_engine_dump", py_index_engine_dump, METH_VARARGS,"Dump index state to disk."},{"_index_engine_get_state", py_index_engine_get_state, METH_VARARGS,@@ -1745,6 +1975,8 @@ PyMethodDef kModuleMethods[] = {"Clear the store."},{"_store_seek_range", py_store_seek_range, METH_VARARGS,"Read a key range from the store."},+ {"_store_seek_range_page", py_store_seek_range_page, METH_VARARGS,+ "Read a bounded page from a key range."},{nullptr, nullptr, 0, nullptr},};diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cppindex f00c25981..a693e62d9 100644--- a/src/index/detail/index_manager_impl.cpp+++ b/src/index/detail/index_manager_impl.cpp@@ -1,6 +1,7 @@// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.// SPDX-License-Identifier: AGPL-3.0#include "index/detail/index_manager_impl.h"+#include <algorithm>#include <stdexcept>#include <memory>#include <chrono>@@ -19,6 +20,9 @@ const std::string kMetaFile = "manager_meta.json";const std::string kVectorIndexDir = "vector_index";const std::string kScalarIndexDir = "scalar_index";constexpr size_t kFilterTokenCacheCapacity = 32;+constexpr uint64_t kFilterLayoutInverseMaxSpanFactor = 4;+constexpr uint32_t kMissingFilterLayoutOffset =+ std::numeric_limits<uint32_t>::max();IndexManagerImpl::IndexManagerImpl(const std::string& path_or_json) {int ret = 0;@@ -233,14 +237,51 @@ int IndexManagerImpl::search_with_filter_token(const SearchRequest& req,int IndexManagerImpl::set_filter_layout(const std::vector<uint64_t>& ordered_labels) {std::unique_lock<std::shared_mutex> lock(rw_mutex_);- filter_layout_offsets_.clear();+ clear_filter_layout_();filter_layout_offsets_.reserve(ordered_labels.size());+ uint64_t valid_offset_count = 0;+ uint32_t min_offset = kMissingFilterLayoutOffset;+ uint32_t max_offset = 0;for (const uint64_t label : ordered_labels) {const int offset = vector_index_->get_offset_by_label(label);- filter_layout_offsets_.push_back(- offset >= 0 ? static_cast<uint32_t>(offset)- : std::numeric_limits<uint32_t>::max());+ if (offset < 0) {+ filter_layout_offsets_.push_back(kMissingFilterLayoutOffset);+ continue;+ }+ const auto logical_offset = static_cast<uint32_t>(offset);+ filter_layout_offsets_.push_back(logical_offset);+ ++valid_offset_count;+ min_offset = std::min(min_offset, logical_offset);+ max_offset = std::max(max_offset, logical_offset);+ }++ if (valid_offset_count == 0) {+ return 0;+ }+ const uint64_t inverse_span =+ static_cast<uint64_t>(max_offset) - min_offset + 1;+ if (inverse_span >+ kFilterLayoutInverseMaxSpanFactor * valid_offset_count) {+ return 0;+ }++ filter_layout_rows_by_offset_.assign(+ static_cast<size_t>(inverse_span), kMissingFilterLayoutOffset);+ for (size_t row = 0; row < filter_layout_offsets_.size(); ++row) {+ const uint32_t offset = filter_layout_offsets_[row];+ if (offset == kMissingFilterLayoutOffset) {+ continue;+ }+ const size_t inverse_row = static_cast<size_t>(offset - min_offset);+ if (filter_layout_rows_by_offset_[inverse_row] !=+ kMissingFilterLayoutOffset) {+ filter_layout_rows_by_offset_.clear();+ return 0;+ }+ filter_layout_rows_by_offset_[inverse_row] = static_cast<uint32_t>(row);}+ filter_layout_inverse_base_ = min_offset;+ filter_layout_inverse_ready_ = true;return 0;}@@ -284,6 +325,106 @@ int IndexManagerImpl::evaluate_filter(const std::string& dsl,return 0;}+int IndexManagerImpl::evaluate_filter_for_routing(+ const std::string& dsl, uint64_t native_threshold,+ FilterResult& result) {+ if (native_threshold == 0) {+ return evaluate_filter(dsl, 0, result);+ }++ SearchContext ctx;+ if (int ret = parse_dsl_query(dsl, ctx); ret != 0) {+ SPDLOG_ERROR(+ "IndexManagerImpl::evaluate_filter_for_routing [{}] parse failed",+ dsl);+ return ret;+ }++ std::shared_lock<std::shared_mutex> lock(rw_mutex_);+ BitmapPtr bitmap = nullptr;+ if (ctx.filter_op) {+ bitmap = calculate_filter_bitmap(ctx, dsl);+ if (!bitmap) {+ SPDLOG_DEBUG(+ "IndexManagerImpl::evaluate_filter_for_routing "+ "calculate_filter_bitmap returned null");+ return -1;+ }+ }++ result.eligible_count = 0;+ result.bitset_words.clear();+ result.native_filter_token = 0;++ // A selective scalar bitmap can be projected through the inverse layout in+ // O(bitmap cardinality), without scanning every external dense-index row.+ if (bitmap && filter_layout_inverse_ready_ &&+ bitmap->nbit() <= native_threshold) {+ std::vector<uint32_t> eligible_offsets;+ bitmap->get_set_list(eligible_offsets);+ for (const uint32_t offset : eligible_offsets) {+ if (offset < filter_layout_inverse_base_) {+ continue;+ }+ const uint64_t inverse_row =+ static_cast<uint64_t>(offset) - filter_layout_inverse_base_;+ if (inverse_row >= filter_layout_rows_by_offset_.size() ||+ filter_layout_rows_by_offset_[inverse_row] ==+ kMissingFilterLayoutOffset) {+ continue;+ }+ ++result.eligible_count;+ }+ if (result.eligible_count > 0) {+ result.native_filter_token = cache_filter_bitmap_(bitmap);+ }+ return 0;+ }++ // Keep at most threshold + 1 row ids while counting. If the filter is wide,+ // allocate the full projection only after the route decision is known.+ std::vector<uint32_t> narrow_rows;+ const bool cannot_exceed_threshold =+ native_threshold >= filter_layout_offsets_.size();+ if (!cannot_exceed_threshold) {+ narrow_rows.reserve(static_cast<size_t>(native_threshold + 1));+ }+ for (size_t row = 0; row < filter_layout_offsets_.size(); ++row) {+ const uint32_t offset = filter_layout_offsets_[row];+ const bool eligible = offset != kMissingFilterLayoutOffset &&+ (!bitmap || bitmap->Isset(offset));+ if (!eligible) {+ continue;+ }+ ++result.eligible_count;+ if (cannot_exceed_threshold) {+ continue;+ }+ if (result.bitset_words.empty()) {+ narrow_rows.push_back(static_cast<uint32_t>(row));+ if (result.eligible_count <= native_threshold) {+ continue;+ }+ result.bitset_words.assign((filter_layout_offsets_.size() + 31) / 32,+ 0);+ for (const uint32_t narrow_row : narrow_rows) {+ result.bitset_words[narrow_row / 32] |=+ static_cast<uint32_t>(1U << (narrow_row % 32));+ }+ narrow_rows.clear();+ continue;+ }+ result.bitset_words[row / 32] |=+ static_cast<uint32_t>(1U << (row % 32));+ }++ if (bitmap && result.eligible_count > 0 &&+ result.eligible_count <= native_threshold) {+ result.native_filter_token = cache_filter_bitmap_(bitmap);+ }+ return 0;+}+uint64_t IndexManagerImpl::cache_filter_bitmap_(const BitmapPtr& bitmap) {std::lock_guard<std::mutex> lock(filter_token_mutex_);uint64_t token = next_filter_token_++;@@ -305,6 +446,13 @@ void IndexManagerImpl::clear_filter_token_cache_() {filter_token_order_.clear();}+void IndexManagerImpl::clear_filter_layout_() {+ filter_layout_offsets_.clear();+ filter_layout_rows_by_offset_.clear();+ filter_layout_inverse_base_ = 0;+ filter_layout_inverse_ready_ = false;+}+BitmapPtr IndexManagerImpl::calculate_filter_bitmap(const SearchContext& ctx,const std::string& dsl) {auto bitmap = ctx.filter_op->calc_bitmap(scalar_index_->get_field_sets(),@@ -415,7 +563,7 @@ int IndexManagerImpl::add_data(const std::vector<AddDataRequest>& data_list) {parsed_old_fields_list[i]);}if (has_update) {- filter_layout_offsets_.clear();+ clear_filter_layout_();clear_filter_token_cache_();auto duration = std::chrono::system_clock::now().time_since_epoch();manager_meta_->update_timestamp =@@ -459,7 +607,7 @@ int IndexManagerImpl::delete_data(vector_index_->stream_delete_data(data.label);}if (has_update) {- filter_layout_offsets_.clear();+ clear_filter_layout_();clear_filter_token_cache_();auto duration = std::chrono::system_clock::now().time_since_epoch();manager_meta_->update_timestamp =diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.hindex a510dc486..6180b0246 100644--- a/src/index/detail/index_manager_impl.h+++ b/src/index/detail/index_manager_impl.h@@ -44,6 +44,10 @@ class IndexManagerImpl : public IndexManager {uint64_t max_cached_candidates,FilterResult& result) override;+ int evaluate_filter_for_routing(const std::string& dsl,+ uint64_t native_threshold,+ FilterResult& result) override;+int add_data(const std::vector<AddDataRequest>& data_list) override;int delete_data(const std::vector<DeleteDataRequest>& data_list) override;@@ -73,12 +77,17 @@ class IndexManagerImpl : public IndexManager {void clear_filter_token_cache_();+ void clear_filter_layout_();+private:std::shared_mutex rw_mutex_;std::shared_ptr<ManagerMeta> manager_meta_;std::shared_ptr<ScalarIndex> scalar_index_;std::shared_ptr<VectorIndexAdapter> vector_index_;std::vector<uint32_t> filter_layout_offsets_;+ std::vector<uint32_t> filter_layout_rows_by_offset_;+ uint32_t filter_layout_inverse_base_ = 0;+ bool filter_layout_inverse_ready_ = false;std::mutex filter_token_mutex_;uint64_t next_filter_token_ = 1;std::deque<uint64_t> filter_token_order_;diff --git a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cppindex 1ec660a7e..79dd2b1a5 100644--- a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp+++ b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp@@ -320,6 +320,12 @@ int FieldBitmapGroup::parse_from_stream(std::ifstream& input) {dir_index_ = std::make_shared<DirIndex>();dir_index_->parse_from_stream(input);+ // Runtime leaf-to-bitmap bindings are intentionally not serialized. The+ // on-disk format remains compatible; rebuild the non-owning links after+ // both the bitmap map and trie have been loaded.+ for (const auto& entry : bitmap_group_) {+ dir_index_->add_key(entry.first, entry.second.get());+ }}} catch (std::exception& e) {// SPDLOG_ERROR("FieldBitmapGroup parse_from_stream exception {}",@@ -469,20 +475,8 @@ BitmapPtr FieldBitmapGroupSet::make_path_field_copy(}std::vector<const Bitmap*> bitmaps_to_union;- std::unordered_set<std::string> all_unique_bitmaps;-for (const auto& path_prefix : keys) {- std::unordered_set<std::string> unique_bitmaps;- dip->get_merged_bitmap(path_prefix, depth, unique_bitmaps);- all_unique_bitmaps.insert(unique_bitmaps.begin(), unique_bitmaps.end());- }-- bitmaps_to_union.reserve(all_unique_bitmaps.size());- for (const auto& bitmap_key : all_unique_bitmaps) {- const Bitmap* bm = group->get_bitmap(bitmap_key);- if (bm) {- bitmaps_to_union.push_back(bm);- }+ dip->get_merged_bitmaps(path_prefix, depth, bitmaps_to_union);}auto final_bitmap = std::make_shared<Bitmap>();diff --git a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.hindex d4804ff7c..de43c6f94 100644--- a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h+++ b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h@@ -137,14 +137,12 @@ class FieldBitmapGroup : public BitmapGroupBase {for (auto& key_i : keys) {const std::string norm_key =dir_index_ ? normalize_path_key(key_i) : key_i;- if (!exist_bitmap(norm_key)) {- if (dir_index_) {- dir_index_->add_key(norm_key);- }- }-+ const bool is_new_bitmap = !exist_bitmap(norm_key);Bitmap* temp_p = get_editable_bitmap(norm_key);if (temp_p) {+ if (dir_index_ && is_new_bitmap) {+ dir_index_->add_key(norm_key, temp_p);+ }temp_p->Set(offset);}}@@ -152,13 +150,12 @@ class FieldBitmapGroup : public BitmapGroupBase {} else {const std::string norm_key =dir_index_ ? normalize_path_key(field_str) : field_str;- if (!exist_bitmap(norm_key)) {- if (dir_index_) {- dir_index_->add_key(norm_key);- }- }+ const bool is_new_bitmap = !exist_bitmap(norm_key);Bitmap* temp_p = get_editable_bitmap(norm_key);if (temp_p) {+ if (dir_index_ && is_new_bitmap) {+ dir_index_->add_key(norm_key, temp_p);+ }temp_p->Set(offset);}}diff --git a/src/index/detail/scalar/bitmap_holder/dir_index.cpp b/src/index/detail/scalar/bitmap_holder/dir_index.cppindex 5c6817764..fc2ffc257 100644--- a/src/index/detail/scalar/bitmap_holder/dir_index.cpp+++ b/src/index/detail/scalar/bitmap_holder/dir_index.cpp@@ -1,9 +1,6 @@// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.// SPDX-License-Identifier: AGPL-3.0#include "dir_index.h"-#include <deque>-#include <unordered_set>-#include "index/detail/scalar/bitmap_holder/bitmap_field_group.h"namespace vectordb {@@ -38,38 +35,51 @@ TrieNode* DirIndex::find_node(const std::string& path) const {return node;}-void DirIndex::get_merged_bitmap(- const std::string& path_prefix, int depth,- std::unordered_set<std::string>& unique_bitmaps) const {- TrieNode* start_node = find_node(path_prefix);- if (!start_node) {+void DirIndex::bind_leaf_bitmap(TrieNode* node, const Bitmap* bitmap) {+ if (!node || !bitmap) {+ return;+ }+ if (!node->leaf_bitmap_) {+ node->leaf_bitmap_ = bitmap;+ return;+ }+ if (node->leaf_bitmap_ == bitmap) {return;}- std::string path_buffer = path_prefix.empty() ? "" : path_prefix;- if (!path_buffer.empty() && path_buffer[0] != '/') {- path_buffer.insert(path_buffer.begin(), '/');+ auto& collisions = leaf_bitmap_collisions_[node];+ if (std::find(collisions.begin(), collisions.end(), bitmap) ==+ collisions.end()) {+ collisions.push_back(bitmap);}- while (path_buffer.size() > 1 && path_buffer.back() == '/') {- path_buffer.pop_back();+}++void DirIndex::get_merged_bitmaps(+ const std::string& path_prefix, int depth,+ std::vector<const Bitmap*>& bitmaps) const {+ TrieNode* start_node = find_node(path_prefix);+ if (!start_node) {+ return;}- collect_bitmaps_recursive_optimized(start_node, 0, depth, unique_bitmaps,- path_buffer);+ collect_bitmaps_recursive(start_node, 0, depth, bitmaps);}-void DirIndex::collect_bitmaps_recursive_optimized(+void DirIndex::collect_bitmaps_recursive(const TrieNode* node, int current_depth, int max_depth,- std::unordered_set<std::string>& bitmaps, std::string& path_buffer) const {+ std::vector<const Bitmap*>& bitmaps) const {if (!node) {return;}if (node->is_leaf_) {- if (path_buffer.empty() || path_buffer == "/") {- bitmaps.insert("/");- } else {- bitmaps.insert(path_buffer);+ if (node->leaf_bitmap_) {+ bitmaps.push_back(node->leaf_bitmap_);+ }+ const auto collision_it = leaf_bitmap_collisions_.find(node);+ if (collision_it != leaf_bitmap_collisions_.end()) {+ bitmaps.insert(bitmaps.end(), collision_it->second.begin(),+ collision_it->second.end());}}@@ -78,19 +88,9 @@ void DirIndex::collect_bitmaps_recursive_optimized(}for (const auto& child_pair : node->children_) {- const auto& segment = child_pair.first;const auto& child_node = child_pair.second;- size_t original_size = path_buffer.length();-- if (path_buffer.empty() || path_buffer == "/") {- path_buffer = "/" + segment;- } else {- path_buffer += "/" + segment;- }-- collect_bitmaps_recursive_optimized(child_node.get(), current_depth + 1,- max_depth, bitmaps, path_buffer);- path_buffer.resize(original_size);+ collect_bitmaps_recursive(child_node.get(), current_depth + 1, max_depth,+ bitmaps);}}@@ -129,7 +129,8 @@ std::unique_ptr<TrieNode> DirIndex::parse_recursive(std::ifstream& input,return node;}void DirIndex::parse_from_stream(std::ifstream& input) {+ leaf_bitmap_collisions_.clear();root_ = parse_recursive(input, nullptr);}-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/index/detail/scalar/bitmap_holder/dir_index.h b/src/index/detail/scalar/bitmap_holder/dir_index.hindex 1a7ef696a..acec3bc48 100644--- a/src/index/detail/scalar/bitmap_holder/dir_index.h+++ b/src/index/detail/scalar/bitmap_holder/dir_index.h@@ -2,13 +2,11 @@// SPDX-License-Identifier: AGPL-3.0#pragma once#include <algorithm>-#include <string>-#include <vector>-#include <set>#include <memory>-#include <unordered_map>-#include <unordered_set>#include <sstream>+#include <string>+#include <unordered_map>+#include <vector>#include "common/io_utils.h"#include "index/detail/scalar/bitmap_holder/bitmap.h"@@ -18,6 +16,10 @@ struct TrieNode {std::string path_segment_;TrieNode* parent_ = nullptr;std::unordered_map<std::string, std::unique_ptr<TrieNode>> children_;+ // Non-owning pointer into BitmapGroupBase::bitmap_group_. Bitmap objects are+ // updated in place and remain stable during every live directory-index+ // access under IndexManager's reader/writer lock.+ const Bitmap* leaf_bitmap_ = nullptr;bool is_leaf_ = false;TrieNode() = default;@@ -31,7 +33,7 @@ class DirIndex {DirIndex() = default;virtual ~DirIndex() = default;- void add_key(const std::string& key) {+ void add_key(const std::string& key, const Bitmap* bitmap) {TrieNode* node = root_.get();for (const auto& segment : split_path(key)) {auto it = node->children_.find(segment);@@ -45,29 +47,35 @@ class DirIndex {}}node->is_leaf_ = true;+ bind_leaf_bitmap(node, bitmap);}- void get_merged_bitmap(const std::string& path_prefix, int depth,- std::unordered_set<std::string>& unique_bitmaps) const;+ void get_merged_bitmaps(const std::string& path_prefix, int depth,+ std::vector<const Bitmap*>& bitmaps) const;virtual void serialize_to_stream(std::ofstream& output);virtual void parse_from_stream(std::ifstream& input);private:std::unique_ptr<TrieNode> root_ = std::make_unique<TrieNode>("", nullptr);+ // Canonically equivalent spellings can resolve to the same trie leaf. Keep+ // their additional bitmaps out of TrieNode so the normal one-bitmap case+ // does not pay per-node vector overhead.+ std::unordered_map<const TrieNode*, std::vector<const Bitmap*>>+ leaf_bitmap_collisions_;TrieNode* find_node(const std::string& path) const;+ void bind_leaf_bitmap(TrieNode* node, const Bitmap* bitmap);std::vector<std::string> split_path(const std::string& path) const;void serialize_recursive(const TrieNode* node, std::ofstream& output) const;std::unique_ptr<TrieNode> parse_recursive(std::ifstream& input,TrieNode* parent);- void collect_bitmaps_recursive_optimized(+ void collect_bitmaps_recursive(const TrieNode* node, int current_depth, int max_depth,- std::unordered_set<std::string>& unique_bitmaps,- std::string& path_buffer) const;+ std::vector<const Bitmap*>& bitmaps) const;};using DirIndexPtr = std::shared_ptr<DirIndex>;-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/index/index_engine.cpp b/src/index/index_engine.cppindex da9924566..0ebd15e1f 100644--- a/src/index/index_engine.cpp+++ b/src/index/index_engine.cpp@@ -50,6 +50,18 @@ FilterResult IndexEngine::evaluate_filter(return result;}+FilterResult IndexEngine::evaluate_filter_for_routing(+ const std::string& dsl, uint64_t native_threshold) {+ FilterResult result;+ const int ret =+ impl_->evaluate_filter_for_routing(dsl, native_threshold, result);+ if (ret != 0) {+ throw std::runtime_error(+ "Failed to evaluate native scalar filter for routing");+ }+ return result;+}+int IndexEngine::add_data(const std::vector<AddDataRequest>& data_list) {return impl_->add_data(data_list);}diff --git a/src/index/index_engine.h b/src/index/index_engine.hindex cea8004fd..1c82a9244 100644--- a/src/index/index_engine.h+++ b/src/index/index_engine.h@@ -33,6 +33,9 @@ class IndexEngine {FilterResult evaluate_filter(const std::string& dsl,uint64_t max_cached_candidates = 0);+ FilterResult evaluate_filter_for_routing(const std::string& dsl,+ uint64_t native_threshold);+int64_t dump(const std::string& dir);StateResult get_state();diff --git a/src/index/index_manager.h b/src/index/index_manager.hindex 420112bd9..9b6e1f92f 100644--- a/src/index/index_manager.h+++ b/src/index/index_manager.h@@ -26,6 +26,12 @@ class IndexManager {uint64_t max_cached_candidates,FilterResult& result) = 0;+ // A narrow result may omit bitset_words because adaptive mode will route it+ // to native search. A zero threshold retains evaluate_filter semantics.+ virtual int evaluate_filter_for_routing(const std::string& dsl,+ uint64_t native_threshold,+ FilterResult& result) = 0;+virtual int add_data(const std::vector<AddDataRequest>& data_list) = 0;virtual int delete_data(const std::vector<DeleteDataRequest>& data_list) = 0;diff --git a/src/store/kv_store.h b/src/store/kv_store.hindex 7ba551a43..0cb2761f8 100644--- a/src/store/kv_store.h+++ b/src/store/kv_store.h@@ -1,6 +1,7 @@// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.// SPDX-License-Identifier: AGPL-3.0#pragma once+#include <cstddef>#include <string>#include <vector>#include "store/common_structs.h"@@ -25,6 +26,10 @@ class KVStore {virtual std::vector<std::pair<std::string, std::string>> seek_range(const std::string& start_key, const std::string& end_key) = 0;++ virtual std::vector<std::pair<std::string, std::string>> seek_range_page(+ const std::string& start_key, const std::string& end_key, size_t limit,+ size_t max_bytes, bool start_exclusive) = 0;};-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/store/persist_store.cpp b/src/store/persist_store.cppindex dcdf44373..6bf28abf5 100644--- a/src/store/persist_store.cpp+++ b/src/store/persist_store.cpp@@ -4,6 +4,7 @@#include "spdlog/spdlog.h"#include <stdexcept>#include <filesystem>+#include <memory>#include "leveldb/write_batch.h"namespace vectordb {@@ -112,6 +113,43 @@ std::vector<std::pair<std::string, std::string>> PersistStore::seek_range(return key_values;}+std::vector<std::pair<std::string, std::string>>+PersistStore::seek_range_page(const std::string& start_key,+ const std::string& end_key, size_t limit,+ size_t max_bytes, bool start_exclusive) {+ std::vector<std::pair<std::string, std::string>> key_values;+ if (limit == 0 || max_bytes == 0) {+ return key_values;+ }+ size_t page_bytes = 0;+ std::unique_ptr<leveldb::Iterator> it(+ db_->NewIterator(leveldb::ReadOptions()));+ it->Seek(start_key);+ if (start_exclusive && it->Valid() && it->key().ToString() == start_key) {+ it->Next();+ }+ for (; it->Valid() && it->key().ToString() < end_key &&+ key_values.size() < limit;+ it->Next()) {+ const auto key = it->key();+ const auto value = it->value();+ const size_t item_bytes = key.size() + value.size();+ // Always admit one oversized row so pagination can make progress. Every+ // additional row must fit within the encoded-byte budget.+ if (!key_values.empty() &&+ (page_bytes >= max_bytes || item_bytes > max_bytes - page_bytes)) {+ break;+ }+ key_values.push_back({key.ToString(), value.ToString()});+ page_bytes += item_bytes;+ }+ if (!it->status().ok()) {+ throw std::runtime_error("PersistStore::seek_range_page iterate error: " ++ it->status().ToString());+ }+ return key_values;+}+int PersistStore::exec_op(const std::vector<StorageOp>& ops) {leveldb::WriteBatch batch;for (const auto& op : ops) {@@ -134,4 +172,4 @@ int PersistStore::exec_op(const std::vector<StorageOp>& ops) {return 0;}-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/store/persist_store.h b/src/store/persist_store.hindex cc878d751..23e7bb99e 100644--- a/src/store/persist_store.h+++ b/src/store/persist_store.h@@ -30,8 +30,12 @@ class PersistStore : public KVStore {std::vector<std::pair<std::string, std::string>> seek_range(const std::string& start_key, const std::string& end_key) override;+ std::vector<std::pair<std::string, std::string>> seek_range_page(+ const std::string& start_key, const std::string& end_key, size_t limit,+ size_t max_bytes, bool start_exclusive) override;+private:leveldb::DB* db_ = nullptr;};-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/store/volatile_store.cpp b/src/store/volatile_store.cppindex f3ee9e3ed..9b8a6b66b 100644--- a/src/store/volatile_store.cpp+++ b/src/store/volatile_store.cpp@@ -56,6 +56,34 @@ std::vector<std::pair<std::string, std::string>> VolatileStore::seek_range(return key_values;}+std::vector<std::pair<std::string, std::string>>+VolatileStore::seek_range_page(const std::string& start_key,+ const std::string& end_key, size_t limit,+ size_t max_bytes, bool start_exclusive) {+ std::shared_lock<std::shared_mutex> lock(mutex_);+ std::vector<std::pair<std::string, std::string>> key_values;+ if (limit == 0 || max_bytes == 0) {+ return key_values;+ }+ size_t page_bytes = 0;+ auto iter = start_exclusive ? data_.upper_bound(start_key)+ : data_.lower_bound(start_key);+ for (; iter != data_.end() && iter->first < end_key &&+ key_values.size() < limit;+ ++iter) {+ const size_t item_bytes = iter->first.size() + iter->second.size();+ // Always admit one oversized row so pagination can make progress. Every+ // additional row must fit within the encoded-byte budget.+ if (!key_values.empty() &&+ (page_bytes >= max_bytes || item_bytes > max_bytes - page_bytes)) {+ break;+ }+ key_values.push_back({iter->first, iter->second});+ page_bytes += item_bytes;+ }+ return key_values;+}+int VolatileStore::exec_op(const std::vector<StorageOp>& ops) {std::unique_lock<std::shared_mutex> lock(mutex_);for (const auto& op : ops) {@@ -71,4 +99,4 @@ int VolatileStore::exec_op(const std::vector<StorageOp>& ops) {return 0;}-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/store/volatile_store.h b/src/store/volatile_store.hindex 07d76e347..afbf64440 100644--- a/src/store/volatile_store.h+++ b/src/store/volatile_store.h@@ -31,9 +31,13 @@ class VolatileStore : public KVStore {std::vector<std::pair<std::string, std::string>> seek_range(const std::string& start_key, const std::string& end_key) override;+ std::vector<std::pair<std::string, std::string>> seek_range_page(+ const std::string& start_key, const std::string& end_key, size_t limit,+ size_t max_bytes, bool start_exclusive) override;+private:std::map<std::string, std::string> data_;mutable std::shared_mutex mutex_;};-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/tests/engine/test_index_engine.cpp b/tests/engine/test_index_engine.cppindex 330b74723..9bd4186dd 100644--- a/tests/engine/test_index_engine.cpp+++ b/tests/engine/test_index_engine.cpp@@ -1,6 +1,8 @@// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.// SPDX-License-Identifier: AGPL-3.0#include "index/index_engine.h"+#include "store/persist_store.h"+#include "store/volatile_store.h"#include <iostream>#include <vector>#include <cassert>@@ -16,6 +18,23 @@ bool is_close(float a, float b, float epsilon = 1e-5) {return std::fabs(a - b) < epsilon;}+void expect_filter_projection(IndexEngine& engine, const std::string& dsl,+ uint64_t expected_count,+ uint32_t expected_first_word) {+ FilterResult result = engine.evaluate_filter(dsl);+ const uint32_t first_word =+ result.bitset_words.empty() ? 0U : result.bitset_words[0];+ if (result.eligible_count != expected_count ||+ result.bitset_words.size() != 1 || first_word != expected_first_word) {+ SPDLOG_ERROR(+ "Unexpected filter projection: dsl={}, count={} (expected {}), "+ "words={}, first_word={} (expected {})",+ dsl, result.eligible_count, expected_count, result.bitset_words.size(),+ first_word, expected_first_word);+ exit(1);+ }+}+void test_basic_workflow() {SPDLOG_INFO("[Running] test_basic_workflow...");@@ -108,6 +127,39 @@ void test_basic_workflow() {exit(1);}+ FilterResult routed_filter_res =+ engine.evaluate_filter_for_routing(uri_filter, 10);+ if (routed_filter_res.eligible_count != 1 ||+ !routed_filter_res.bitset_words.empty() ||+ routed_filter_res.native_filter_token == 0) {+ SPDLOG_ERROR("Sparse routed filter projection failed");+ exit(1);+ }++ FilterResult routed_threshold_zero =+ engine.evaluate_filter_for_routing(uri_filter, 0);+ if (routed_threshold_zero.eligible_count != filter_res.eligible_count ||+ routed_threshold_zero.bitset_words != filter_res.bitset_words ||+ routed_threshold_zero.native_filter_token != 0) {+ SPDLOG_ERROR("Threshold-zero routed filter projection changed semantics");+ exit(1);+ }++ const std::string wide_uri_filter =+ R"({"op":"must","field":"uri","conds":["/docs","/other"],"para":"-d=-1"})";+ FilterResult generic_wide_filter = engine.evaluate_filter(wide_uri_filter);+ FilterResult routed_wide_filter =+ engine.evaluate_filter_for_routing(wide_uri_filter, 1);+ if (generic_wide_filter.eligible_count != 2 ||+ generic_wide_filter.bitset_words.size() != 1 ||+ generic_wide_filter.bitset_words[0] != 0b101U ||+ routed_wide_filter.eligible_count != generic_wide_filter.eligible_count ||+ routed_wide_filter.bitset_words != generic_wide_filter.bitset_words ||+ routed_wide_filter.native_filter_token != 0) {+ SPDLOG_ERROR("Wide routed filter projection changed generic semantics");+ exit(1);+ }+FilterResult cached_filter_res = engine.evaluate_filter(uri_filter, 10);if (cached_filter_res.native_filter_token == 0) {SPDLOG_ERROR("Native filter token was not retained");@@ -159,11 +211,383 @@ void test_basic_workflow() {exit(1);}+ std::filesystem::remove_all(db_path);SPDLOG_INFO("[Passed] test_basic_workflow");}+void test_routed_filter_projection_edge_cases() {+ SPDLOG_INFO("[Running] test_routed_filter_projection_edge_cases...");++ const std::string config = R"({+ "CollectionName": "routed_filter_projection",+ "IndexName": "default",+ "VectorIndex": {+ "IndexType": "flat",+ "ElementCount": 0,+ "MaxElementCount": 32,+ "Dimension": 2,+ "Distance": "l2",+ "Quant": "float"+ },+ "ScalarIndex": [+ {"FieldName": "uri", "FieldType": "path"}+ ]+ })";++ IndexEngine engine(config);+ AddDataRequest early;+ early.label = 3001;+ early.vector = {1.0, 0.0};+ early.fields_str = R"({"uri":"/keep/a"})";+ if (engine.add_data({early}) != 0) {+ SPDLOG_ERROR("Failed to add the early routed-filter record");+ exit(1);+ }++ std::vector<AddDataRequest> fillers;+ std::vector<DeleteDataRequest> filler_deletes;+ for (uint64_t i = 0; i < 16; ++i) {+ AddDataRequest filler;+ filler.label = 3100 + i;+ filler.vector = {0.0, 1.0};+ filler.fields_str =+ "{\"uri\":\"/drop/" + std::to_string(i) + "\"}";+ fillers.push_back(filler);++ DeleteDataRequest deleted;+ deleted.label = filler.label;+ deleted.old_fields_str = filler.fields_str;+ filler_deletes.push_back(deleted);+ }+ if (engine.add_data(fillers) != 0 ||+ engine.delete_data(filler_deletes) != 0) {+ SPDLOG_ERROR("Failed to create sparse logical offsets");+ exit(1);+ }++ AddDataRequest late;+ late.label = 3002;+ late.vector = {0.9, 0.1};+ late.fields_str = R"({"uri":"/keep/b"})";+ if (engine.add_data({late}) != 0) {+ SPDLOG_ERROR("Failed to add the late routed-filter record");+ exit(1);+ }++ const std::string keep_recursive =+ R"({"op":"must","field":"uri","conds":["/keep"],"para":"-d=-1"})";+ const std::string keep_a =+ R"({"op":"must","field":"uri","conds":["/keep/a"],"para":"-d=0"})";++ // The surviving logical offsets are deliberately far apart. The inverse+ // layout must decline an oversized span and preserve the fallback result.+ engine.set_filter_layout({3001, 9999, 3002});+ FilterResult generic = engine.evaluate_filter(keep_recursive);+ FilterResult routed = engine.evaluate_filter_for_routing(keep_recursive, 2);+ if (generic.eligible_count != 2 || generic.bitset_words.size() != 1 ||+ generic.bitset_words[0] != 0b101U || routed.eligible_count != 2 ||+ !routed.bitset_words.empty() || routed.native_filter_token == 0) {+ SPDLOG_ERROR("Large-span routed filter projection was incorrect");+ exit(1);+ }++ FilterResult threshold_zero =+ engine.evaluate_filter_for_routing(keep_recursive, 0);+ if (threshold_zero.eligible_count != generic.eligible_count ||+ threshold_zero.bitset_words != generic.bitset_words ||+ threshold_zero.native_filter_token != 0) {+ SPDLOG_ERROR("Threshold-zero routing changed generic projection semantics");+ exit(1);+ }++ // Duplicate logical offsets cannot use the one-to-one inverse. The fallback+ // must count and project every external row exactly as the generic API does.+ engine.set_filter_layout({3001, 3001, 3002, 9999});+ generic = engine.evaluate_filter(keep_a);+ routed = engine.evaluate_filter_for_routing(keep_a, 1);+ if (generic.eligible_count != 2 || generic.bitset_words.size() != 1 ||+ generic.bitset_words[0] != 0b0011U ||+ routed.eligible_count != generic.eligible_count ||+ routed.bitset_words != generic.bitset_words ||+ routed.native_filter_token != 0) {+ SPDLOG_ERROR("Duplicate-offset routed filter projection was incorrect");+ exit(1);+ }++ FilterResult narrow_duplicate =+ engine.evaluate_filter_for_routing(keep_a, 2);+ if (narrow_duplicate.eligible_count != 2 ||+ !narrow_duplicate.bitset_words.empty() ||+ narrow_duplicate.native_filter_token == 0) {+ SPDLOG_ERROR("Narrow duplicate-offset filter did not retain native state");+ exit(1);+ }++ AddDataRequest update = early;+ update.fields_str = R"({"uri":"/other/a"})";+ update.old_fields_str = early.fields_str;+ if (engine.add_data({update}) != 0) {+ SPDLOG_ERROR("Failed to mutate routed-filter data");+ exit(1);+ }+ SearchRequest query;+ query.query = {1.0, 0.0};+ query.topk = 2;+ if (engine.search_with_filter_token(+ query, narrow_duplicate.native_filter_token)) {+ SPDLOG_ERROR("A routed filter token survived mutation");+ exit(1);+ }+ routed = engine.evaluate_filter_for_routing(keep_recursive, 2);+ if (routed.eligible_count != 0 || !routed.bitset_words.empty() ||+ routed.native_filter_token != 0) {+ SPDLOG_ERROR("Mutation did not invalidate the registered filter layout");+ exit(1);+ }++ engine.set_filter_layout({3001, 3002});+ routed = engine.evaluate_filter_for_routing(keep_recursive, 2);+ if (routed.eligible_count != 1 || !routed.bitset_words.empty() ||+ routed.native_filter_token == 0) {+ SPDLOG_ERROR("Routed filter was incorrect after layout registration");+ exit(1);+ }++ DeleteDataRequest delete_late;+ delete_late.label = late.label;+ delete_late.old_fields_str = late.fields_str;+ const uint64_t pre_delete_token = routed.native_filter_token;+ if (engine.delete_data({delete_late}) != 0) {+ SPDLOG_ERROR("Failed to delete routed-filter data");+ exit(1);+ }+ if (engine.search_with_filter_token(query, pre_delete_token)) {+ SPDLOG_ERROR("A routed filter token survived deletion");+ exit(1);+ }+ engine.set_filter_layout({3001, 3002});+ routed = engine.evaluate_filter_for_routing(keep_recursive, 2);+ if (routed.eligible_count != 0 || !routed.bitset_words.empty() ||+ routed.native_filter_token != 0) {+ SPDLOG_ERROR("Missing-label routed filter projection was incorrect");+ exit(1);+ }++ SPDLOG_INFO("[Passed] test_routed_filter_projection_edge_cases");+}++void test_path_bitmap_lifecycle_and_reload() {+ SPDLOG_INFO("[Running] test_path_bitmap_lifecycle_and_reload...");++ const std::string db_path = "test_data_cpp/path_bitmap_lifecycle";+ if (std::filesystem::exists(db_path)) {+ std::filesystem::remove_all(db_path);+ }+ std::filesystem::create_directories(db_path);++ const std::string config = R"({+ "CollectionName": "path_bitmap_lifecycle",+ "IndexName": "default",+ "VectorIndex": {+ "IndexType": "flat",+ "ElementCount": 0,+ "MaxElementCount": 6,+ "Dimension": 4,+ "Distance": "l2",+ "Quant": "float"+ },+ "ScalarIndex": [+ {"FieldName": "uri", "FieldType": "path"}+ ]+ })";++ IndexEngine engine(config);+ if (!engine.is_valid()) {+ SPDLOG_ERROR("Path bitmap lifecycle engine initialization failed");+ exit(1);+ }++ std::vector<AddDataRequest> initial(5);+ initial[0].label = 2001;+ initial[0].vector = {1.0, 0.0, 0.0, 0.0};+ initial[0].fields_str = R"({"uri":"/docs/a"})";+ initial[1].label = 2002;+ initial[1].vector = {0.9, 0.1, 0.0, 0.0};+ initial[1].fields_str = R"({"uri":"/docs/deep/b"})";+ initial[2].label = 2003;+ initial[2].vector = {0.0, 1.0, 0.0, 0.0};+ initial[2].fields_str = R"({"uri":"/other/c"})";+ // These spellings resolve to one trie leaf. Both bitmap bindings must be+ // retained without adding a vector to every ordinary TrieNode.+ initial[3].label = 2005;+ initial[3].vector = {0.0, 0.0, 1.0, 0.0};+ initial[3].fields_str = R"({"uri":"/aliases/a"})";+ initial[4].label = 2006;+ initial[4].vector = {0.0, 0.0, 0.9, 0.1};+ initial[4].fields_str = R"({"uri":"/aliases//a"})";+ if (engine.add_data(initial) != 0) {+ SPDLOG_ERROR("Initial path bitmap data add failed");+ exit(1);+ }++ const std::string docs_recursive =+ R"({"op":"must","field":"uri","conds":["/docs"],"para":"-d=-1"})";+ const std::string docs_depth_one =+ R"({"op":"must","field":"uri","conds":["/docs"],"para":"-d=1"})";+ const std::string docs_overlapping =+ R"({"op":"must","field":"uri","conds":["/docs","/docs/deep"],"para":"-d=-1"})";+ const std::string other_recursive =+ R"({"op":"must","field":"uri","conds":["/other"],"para":"-d=-1"})";+ const std::string aliases_recursive =+ R"({"op":"must","field":"uri","conds":["/aliases"],"para":"-d=-1"})";++ engine.set_filter_layout({2001, 2002, 2003});+ expect_filter_projection(engine, docs_recursive, 2, 0b011U);+ expect_filter_projection(engine, docs_depth_one, 1, 0b001U);+ expect_filter_projection(engine, docs_overlapping, 2, 0b011U);++ engine.set_filter_layout({2005, 2006});+ expect_filter_projection(engine, aliases_recursive, 2, 0b11U);++ engine.set_filter_layout({2001, 2002, 2003});+ FilterResult cached_docs = engine.evaluate_filter(docs_recursive, 10);+ if (cached_docs.native_filter_token == 0) {+ SPDLOG_ERROR("Path filter token was not retained before mutation");+ exit(1);+ }++ AddDataRequest update;+ update.label = 2001;+ update.vector = {1.0, 0.0, 0.0, 0.0};+ update.fields_str = R"({"uri":"/other/a"})";+ update.old_fields_str = R"({"uri":"/docs/a"})";+ if (engine.add_data({update}) != 0) {+ SPDLOG_ERROR("Path bitmap update failed");+ exit(1);+ }+ SearchRequest token_query;+ token_query.query = {1.0, 0.0, 0.0, 0.0};+ token_query.topk = 10;+ if (engine.search_with_filter_token(token_query,+ cached_docs.native_filter_token)) {+ SPDLOG_ERROR("Path filter token survived an upsert mutation");+ exit(1);+ }++ engine.set_filter_layout({2001, 2002, 2003});+ expect_filter_projection(engine, docs_recursive, 1, 0b010U);+ expect_filter_projection(engine, other_recursive, 2, 0b101U);++ AddDataRequest added;+ added.label = 2004;+ added.vector = {0.8, 0.2, 0.0, 0.0};+ added.fields_str = R"({"uri":"/docs/new"})";+ if (engine.add_data({added}) != 0) {+ SPDLOG_ERROR("New descendant add failed");+ exit(1);+ }+ engine.set_filter_layout({2001, 2002, 2003, 2004});+ expect_filter_projection(engine, docs_recursive, 2, 0b1010U);++ DeleteDataRequest deleted;+ deleted.label = 2002;+ deleted.old_fields_str = R"({"uri":"/docs/deep/b"})";+ if (engine.delete_data({deleted}) != 0) {+ SPDLOG_ERROR("Path descendant delete failed");+ exit(1);+ }+ engine.set_filter_layout({2001, 2003, 2004});+ expect_filter_projection(engine, docs_recursive, 1, 0b100U);+ expect_filter_projection(engine, other_recursive, 2, 0b011U);++ if (engine.dump(db_path) <= 0) {+ SPDLOG_ERROR("Path bitmap lifecycle dump failed");+ exit(1);+ }++ {+ IndexEngine reloaded(db_path);+ if (!reloaded.is_valid()) {+ SPDLOG_ERROR("Reloaded path bitmap engine is invalid");+ exit(1);+ }+ reloaded.set_filter_layout({2001, 2003, 2004});+ expect_filter_projection(reloaded, docs_recursive, 1, 0b100U);+ expect_filter_projection(reloaded, other_recursive, 2, 0b011U);+ reloaded.set_filter_layout({2005, 2006});+ expect_filter_projection(reloaded, aliases_recursive, 2, 0b11U);+ }++ std::filesystem::remove_all(db_path);+ SPDLOG_INFO("[Passed] test_path_bitmap_lifecycle_and_reload");+}++void expect_paged_store_scan(KVStore& store) {+ const std::vector<std::string> keys = {+ "candidate:1", "candidate:10", "candidate:2", "other:1"};+ const std::vector<std::string> values = {"one", "ten", "two", "other"};+ if (store.put_data(keys, values) != 0) {+ SPDLOG_ERROR("Paged store test data add failed");+ exit(1);+ }++ const auto first =+ store.seek_range_page("candidate:", "candidate;", 2, 1024, false);+ if (first.size() != 2 || first[0].first != "candidate:1" ||+ first[1].first != "candidate:10") {+ SPDLOG_ERROR("Paged store first page was incorrect");+ exit(1);+ }+ const auto second =+ store.seek_range_page(first.back().first, "candidate;", 2, 1024, true);+ if (second.size() != 1 || second[0].first != "candidate:2") {+ SPDLOG_ERROR("Paged store exclusive continuation was incorrect");+ exit(1);+ }+ const auto byte_limited =+ store.seek_range_page("candidate:", "candidate;", 10, 14, false);+ if (byte_limited.size() != 1 ||+ byte_limited[0].first != "candidate:1") {+ SPDLOG_ERROR("Paged store byte budget was not enforced");+ exit(1);+ }+ const auto oversized_first =+ store.seek_range_page("candidate:", "candidate;", 10, 1, false);+ if (oversized_first.size() != 1 ||+ oversized_first[0].first != "candidate:1") {+ SPDLOG_ERROR("Paged store did not admit one oversized row for progress");+ exit(1);+ }+ const auto full = store.seek_range("candidate:", "candidate;");+ if (full.size() != 3) {+ SPDLOG_ERROR("Existing full store scan behavior changed");+ exit(1);+ }+}++void test_paged_store_scan() {+ SPDLOG_INFO("[Running] test_paged_store_scan...");+ VolatileStore volatile_store;+ expect_paged_store_scan(volatile_store);++ const std::string db_path = "test_data_cpp/paged_persist_store";+ if (std::filesystem::exists(db_path)) {+ std::filesystem::remove_all(db_path);+ }+ std::filesystem::create_directories(db_path);+ {+ PersistStore persist_store(db_path);+ expect_paged_store_scan(persist_store);+ }+ std::filesystem::remove_all(db_path);+ SPDLOG_INFO("[Passed] test_paged_store_scan");+}+int main() {init_logging("INFO", "stdout", "[%Y-%m-%d %H:%M:%S.%e] [%l] %v");test_basic_workflow();+ test_routed_filter_projection_edge_cases();+ test_path_bitmap_lifecycle_and_reload();+ test_paged_store_scan();return 0;}diff --git a/tests/storage/test_viking_vector_scope_filter.py b/tests/storage/test_viking_vector_scope_filter.pynew file mode 100644index 000000000..a250068e5--- /dev/null+++ b/tests/storage/test_viking_vector_scope_filter.py@@ -0,0 +1,257 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++import pytest++from openviking.core.namespace import uri_parts+from openviking.server.identity import RequestContext, Role+from openviking.storage.expr import And, Eq, In, Or, PathScope+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+from openviking_cli.session.user_id import UserIdentifier+++def _ctx(*, role: Role = Role.USER, actor_peer_id: str | None = None) -> RequestContext:+ return RequestContext(+ user=UserIdentifier("acct", "alice"),+ role=role,+ actor_peer_id=actor_peer_id,+ )+++def _build(+ ctx: RequestContext,+ targets: list[str] | None,+ *,+ context_type: str | None = "resource",+ extra_filter=None,+ level: list[int] | None = None,+):+ backend = object.__new__(VikingVectorIndexBackend)+ return backend._build_scope_filter(+ ctx=ctx,+ context_type=context_type,+ target_directories=targets,+ extra_filter=extra_filter,+ level=level,+ )+++def _tenant_filter(ctx: RequestContext):+ return VikingVectorIndexBackend._tenant_filter(ctx, context_type="resource")+++def test_descendant_target_elides_only_visible_root_path_filter():+ ctx = _ctx()+ target = "viking://resources/wiki/physics"++ result = _build(+ ctx,+ [target],+ extra_filter=Eq("status", "ready"),+ level=[2],+ )++ assert result == And(+ [+ Eq("context_type", "resource"),+ Eq("account_id", "acct"),+ Or([PathScope("uri", target, depth=-1)]),+ Eq("status", "ready"),+ In("level", [2]),+ ]+ )+++def test_equal_visible_root_elides_only_visible_root_path_filter():+ ctx = _ctx()++ result = _build(ctx, ["viking://resources"])++ assert result == And(+ [+ Eq("context_type", "resource"),+ Eq("account_id", "acct"),+ Or([PathScope("uri", "viking://resources", depth=-1)]),+ ]+ )+++def test_all_targets_may_be_under_different_visible_roots():+ ctx = _ctx()+ targets = [+ "viking://resources/wiki/physics",+ "viking://user/resources/private-notes",+ "viking://agent/skills/research",+ ]++ result = _build(ctx, targets)++ assert result == And(+ [+ Eq("context_type", "resource"),+ Eq("account_id", "acct"),+ Or(+ [+ PathScope("uri", "viking://resources/wiki/physics", depth=-1),+ PathScope("uri", "viking://user/alice/resources/private-notes", depth=-1),+ PathScope("uri", "viking://agent/skills/research", depth=-1),+ ]+ ),+ ]+ )+++def test_mixed_visible_and_outside_targets_keep_original_tenant_filter():+ ctx = _ctx()+ targets = ["viking://resources/wiki", "viking://upload/staged"]++ result = _build(ctx, targets)++ assert result == And(+ [+ Eq("context_type", "resource"),+ _tenant_filter(ctx),+ Or([PathScope("uri", target, depth=-1) for target in targets]),+ ]+ )+++async def test_cross_user_targets_cannot_bypass_visible_roots_in_tenant_search():+ ctx = _ctx()+ own_uri = "viking://user/alice/resources/notes"+ cross_user_uri = "viking://user/bob/resources/notes"+ records = [+ {+ "id": "own",+ "uri": own_uri,+ "account_id": "acct",+ "context_type": "resource",+ },+ {+ "id": "cross-user",+ "uri": cross_user_uri,+ "account_id": "acct",+ "context_type": "resource",+ },+ ]+ observed_filters = []++ def matches(expr, record):+ if isinstance(expr, And):+ return all(matches(cond, record) for cond in expr.conds)+ if isinstance(expr, Or):+ return any(matches(cond, record) for cond in expr.conds)+ if isinstance(expr, Eq):+ return record.get(expr.field) == expr.value+ if isinstance(expr, PathScope):+ root = uri_parts(expr.path)+ path = uri_parts(str(record.get(expr.field, "")))+ if path[: len(root)] != root:+ return False+ return expr.depth == -1 or len(path) - len(root) <= expr.depth+ raise AssertionError(f"Unexpected filter expression in test: {expr!r}")++ async def fake_search(*, filter, **_kwargs):+ observed_filters.append(filter)+ return [record for record in records if matches(filter, record)]++ backend = object.__new__(VikingVectorIndexBackend)+ backend.search = fake_search++ cross_user_only = await backend.search_in_tenant(+ ctx=ctx,+ query_vector=[1.0],+ context_type="resource",+ target_directories=[cross_user_uri],+ )+ mixed = await backend.search_in_tenant(+ ctx=ctx,+ query_vector=[1.0],+ context_type="resource",+ target_directories=[own_uri, cross_user_uri],+ )++ assert cross_user_only == []+ assert [record["id"] for record in mixed] == ["own"]+ assert observed_filters == [+ And(+ [+ Eq("context_type", "resource"),+ _tenant_filter(ctx),+ Or([PathScope("uri", cross_user_uri, depth=-1)]),+ ]+ ),+ And(+ [+ Eq("context_type", "resource"),+ _tenant_filter(ctx),+ Or(+ [+ PathScope("uri", own_uri, depth=-1),+ PathScope("uri", cross_user_uri, depth=-1),+ ]+ ),+ ]+ ),+ ]+++def test_segment_prefix_and_visible_root_ancestor_do_not_elide_tenant_filter():+ ctx = _ctx()++ segment_prefix = _build(ctx, ["viking://resources-other/wiki"])+ ancestor = _build(ctx, ["viking://agent"])++ assert segment_prefix == And(+ [+ Eq("context_type", "resource"),+ _tenant_filter(ctx),+ Or([PathScope("uri", "viking://resources-other/wiki", depth=-1)]),+ ]+ )+ assert ancestor == And(+ [+ Eq("context_type", "resource"),+ _tenant_filter(ctx),+ Or([PathScope("uri", "viking://agent", depth=-1)]),+ ]+ )+++def test_no_target_keeps_original_tenant_filter():+ ctx = _ctx()++ assert _build(ctx, None) == And(+ [+ Eq("context_type", "resource"),+ _tenant_filter(ctx),+ ]+ )+++def test_root_role_keeps_existing_target_only_behavior():+ ctx = _ctx(role=Role.ROOT)+ target = "viking://resources/wiki"++ assert _build(ctx, [target]) == And(+ [+ Eq("context_type", "resource"),+ Or([PathScope("uri", target, depth=-1)]),+ ]+ )+++def test_actor_peer_target_retains_account_and_exact_target_scope():+ ctx = _ctx(actor_peer_id="visitor-a")+ target = "viking://user/alice/peers/visitor-a/resources/cases"++ result = _build(ctx, [target])++ assert result == And(+ [+ Eq("context_type", "resource"),+ Eq("account_id", "acct"),+ Or([PathScope("uri", target, depth=-1)]),+ ]+ )diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.pyindex b4ad6ad85..16d487983 100644--- a/tests/test_telemetry_runtime.py+++ b/tests/test_telemetry_runtime.py@@ -300,6 +300,8 @@ def test_cuvs_telemetry_aggregation_is_completion_order_independent():"auto_mode": False,"route_reason": "cuvs","filter_kind": "none",+ "filter_cache_eviction_fallback": True,+ "filter_words_packed": True,"build_performed": True,"records_generation": 2,"index_size": 100,@@ -308,6 +310,7 @@ def test_cuvs_telemetry_aggregation_is_completion_order_independent():"memory_usable_bytes": 7000,"total_ms": 12,"queue_ms": 2,+ "gpu_gate_queue_ms": 1.5,"build_ms": 5,"gpu_search_ms": 5,},@@ -343,8 +346,11 @@ def test_cuvs_telemetry_aggregation_is_completion_order_independent():assert forward == reverseassert forward["routes"] == {"cuvs": 1, "native_filter_threshold": 1}assert forward["builds"] == 1+ assert forward["filter_cache_eviction_fallbacks"] == 1+ assert forward["packed_filter_queries"] == 1assert forward["memory"]["free_bytes_min"] == 6000assert forward["timings_ms"]["total"] == {"sum": 15.0, "max": 12.0}+ assert forward["timings_ms"]["gpu_gate_queue"] == {"sum": 1.5, "max": 1.5}def test_cuvs_telemetry_timing_sum_is_strictly_order_independent():diff --git a/tests/vectordb/test_cuvs_collection.py b/tests/vectordb/test_cuvs_collection.pyindex 9b6536fd4..7edfb4cba 100644--- a/tests/vectordb/test_cuvs_collection.py+++ b/tests/vectordb/test_cuvs_collection.py@@ -1,14 +1,22 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+import gcimport threadingimport traceback+import weakreffrom concurrent.futures import ThreadPoolExecutor+from pathlib import Path++import pytestfrom openviking.storage.vectordb.collection.local_collection import (get_or_create_local_collection,)from openviking.storage.vectordb.index import cuvs_index+from openviking.storage.vectordb.store.data import CandidateData+from openviking.storage.vectordb.store.local_store import StoreEngineProxy+from openviking.storage.vectordb.store.store_manager import StoreManagerfrom openviking.telemetry.backends.memory import MemoryOperationTelemetryfrom openviking.telemetry.context import bind_telemetry@@ -356,6 +364,243 @@ def test_persistent_collection_rehydrates_cuvs_from_local_store(monkeypatch, tmpreopened.close()+def test_persistent_cuvs_recovery_deserializes_candidates_lazily(monkeypatch, tmp_path):+ patch_cuvs_runtime(monkeypatch)+ path = str(tmp_path / "cuvs-streaming-recovery")+ config = {"dense_search": {"backend": "cuvs", "algorithm": "brute_force"}}+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "cuvs_streaming_recovery",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ path=path,+ config=config,+ )+ for index_name in ("first", "second"):+ collection.create_index(+ index_name,+ {+ "IndexName": index_name,+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ record_count = 64+ collection.upsert_data(+ [+ {+ "id": f"row-{offset}",+ "vector": [float(offset + 1), 1.0, 0.0, 0.0],+ }+ for offset in range(record_count)+ ]+ )+ collection.close()++ original_from_bytes = CandidateData.from_bytes+ original_pack_vector = cuvs_index.CuVSDenseIndex._pack_vector+ candidate_refs = []+ events = []+ peak_live_candidates = 0++ def tracked_from_bytes(data):+ nonlocal peak_live_candidates+ candidate = original_from_bytes(data)+ candidate_refs.append(weakref.ref(candidate))+ events.append("decode")+ peak_live_candidates = max(+ peak_live_candidates,+ sum(candidate_ref() is not None for candidate_ref in candidate_refs),+ )+ return candidate++ def tracked_pack_vector(self, vector):+ events.append("pack")+ return original_pack_vector(self, vector)++ monkeypatch.setattr(CandidateData, "from_bytes", staticmethod(tracked_from_bytes))+ monkeypatch.setattr(cuvs_index.CuVSDenseIndex, "_pack_vector", tracked_pack_vector)++ reopened = get_or_create_local_collection(path=path, config=config)+ try:+ for index_name in ("first", "second"):+ index = reopened.get_index(index_name)+ assert index is not None+ assert index.dense_search is not None+ assert index.dense_search.size == record_count++ assert events.count("decode") == record_count * 2+ assert events.count("pack") == record_count * 2+ assert events.index("pack") < len(events) - 1 - events[::-1].index("decode")+ assert peak_live_candidates <= 3+ gc.collect()+ assert not any(candidate_ref() is not None for candidate_ref in candidate_refs)+ finally:+ reopened.close()+++def test_persistent_native_recovery_does_not_scan_candidates(monkeypatch, tmp_path):+ path = str(tmp_path / "native-recovery-no-candidate-scan")+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "native_recovery_no_candidate_scan",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ path=path,+ )+ collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ collection.upsert_data([{"id": "persisted", "vector": [1.0, 0.0, 0.0, 0.0]}])+ collection.close()++ original_read_all = StoreEngineProxy.read_all++ def reject_candidate_scan(self, table_name):+ if table_name == StoreManager.CandsTable:+ raise AssertionError("native snapshot recovery must not scan candidate rows")+ return original_read_all(self, table_name)++ monkeypatch.setattr(StoreEngineProxy, "read_all", reject_candidate_scan)+ reopened = get_or_create_local_collection(path=path)+ try:+ result = reopened.search_by_vector("default", dense_vector=[1.0, 0.0, 0.0, 0.0], limit=1)+ assert [item.id for item in result.data] == ["persisted"]+ finally:+ reopened.close()+++def test_persistent_cuvs_recovery_materializes_when_native_snapshot_is_missing(+ monkeypatch, tmp_path+):+ patch_cuvs_runtime(monkeypatch)+ path = str(tmp_path / "cuvs-missing-native-snapshot")+ config = {"dense_search": {"backend": "cuvs", "algorithm": "brute_force"}}+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "cuvs_missing_native_snapshot",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ path=path,+ config=config,+ )+ collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ collection.upsert_data([{"id": "persisted", "vector": [1.0, 0.0, 0.0, 0.0]}])+ index = collection.get_index("default")+ assert index is not None+ version_dir = Path(index.version_dir)+ collection.close()++ markers = list(version_dir.glob("*.write_done"))+ assert markers+ for marker in markers:+ marker.unlink()++ reopened = get_or_create_local_collection(path=path, config=config)+ try:+ result = reopened.search_by_vector("default", dense_vector=[1.0, 0.0, 0.0, 0.0], limit=1)+ assert [item.id for item in result.data] == ["persisted"]+ recovered_index = reopened.get_index("default")+ assert recovered_index is not None+ assert recovered_index.dense_search is not None+ assert recovered_index.dense_search.size == 1+ finally:+ reopened.close()+++def test_failed_cuvs_recovery_closes_candidate_iterator_and_partial_shadow(monkeypatch, tmp_path):+ patch_cuvs_runtime(monkeypatch)+ path = str(tmp_path / "cuvs-failed-streaming-recovery")+ config = {"dense_search": {"backend": "cuvs", "algorithm": "brute_force"}}+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "cuvs_failed_streaming_recovery",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ path=path,+ config=config,+ )+ collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ collection.upsert_data([{"id": "persisted", "vector": [1.0, 0.0, 0.0, 0.0]}])+ collection.close()++ iterator_closed = False+ original_iter_all_cands_data = StoreManager.iter_all_cands_data++ def broken_candidates(_self):+ nonlocal iterator_closed+ try:+ yield CandidateData(label=1, vector=[1.0, 0.0, 0.0, 0.0])+ yield CandidateData(label=2, vector=[1.0, 0.0])+ finally:+ iterator_closed = True++ class TrackingRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__("inner_product")+ self.closed = False++ def close(self):+ self.closed = True++ runtime = TrackingRuntime()+ monkeypatch.setattr(StoreManager, "iter_all_cands_data", broken_candidates)+ monkeypatch.setattr(+ cuvs_index,+ "_CuVSRuntime",+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,+ )++ with pytest.raises(ValueError, match="cuVS vector dimension mismatch"):+ get_or_create_local_collection(path=path, config=config)++ assert iterator_closed+ assert runtime.closed++ # Release must be complete when the constructor raises: restoring a valid+ # stream and runtime should allow this same persistent collection to reopen+ # immediately, without waiting for traceback GC to drop native locks.+ monkeypatch.setattr(StoreManager, "iter_all_cands_data", original_iter_all_cands_data)+ patch_cuvs_runtime(monkeypatch)+ reopened = get_or_create_local_collection(path=path, config=config)+ try:+ result = reopened.search_by_vector(+ "default",+ dense_vector=[1.0, 0.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["persisted"]+ finally:+ reopened.close()++def test_auto_cuvs_falls_back_then_retries_when_memory_is_available(monkeypatch):runtimes = []diff --git a/tests/vectordb/test_cuvs_filter_packed.py b/tests/vectordb/test_cuvs_filter_packed.pynew file mode 100644index 000000000..5e8119fac--- /dev/null+++ b/tests/vectordb/test_cuvs_filter_packed.py@@ -0,0 +1,244 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import gc+import json++import numpy as np+import pytest++from openviking.storage.vectordb.index.cuvs_index import (+ CuVSDenseIndex,+ CuVSSearchTelemetry,+ _CuVSRuntime,+)+from openviking.storage.vectordb.store.data import CandidateData+++class _HostOnlyRuntime:+ pass+++def _candidate(label: int) -> CandidateData:+ return CandidateData(label=label, vector=[float(label)], fields=json.dumps({"group": "a"}))+++def _index(row_count: int) -> CuVSDenseIndex:+ index = CuVSDenseIndex(+ dimension=1,+ distance="ip",+ normalize_vectors=False,+ field_types={"group": "string"},+ config={},+ runtime=_HostOnlyRuntime(),+ )+ index.add_candidates(_candidate(label) for label in range(1, row_count + 1))+ with index._lock:+ index._filter_layout_generation = index._records_generation+ return index+++def test_packed_filter_rejects_malformed_byte_length():+ index = _index(1)++ with pytest.raises(ValueError, match="multiple of 4 bytes"):+ index._resolve_native_filter({}, lambda _filters: (b"\x01", 1))+++def test_packed_filter_completeness_uses_word_count_not_byte_count():+ index = _index(33)++ with pytest.raises(RuntimeError, match=r"got 1 words.*expected at least 2"):+ index._resolve_native_filter({}, lambda _filters: (b"\x01\x00\x00\x00", 1))+++def test_packed_filter_preserves_legacy_host_mask_fallback_and_telemetry():+ index = _index(2)+ filters = {"op": "must", "field": "group", "conds": ["a"]}+ telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)+ packed = b"\x02\x00\x00\x00"++ host_filter = index._prepare_host_filter(+ filters,+ lambda _filters: (packed, 1),+ lambda _labels: None,+ telemetry,+ )+ assert host_filter is not None+ resolved = host_filter.resolved_native_filter+ assert resolved is not None+ prepared = index._prepare_filter(+ filters,+ [1, 2],+ lambda _filters: (packed, 1),+ resolved,+ )++ assert resolved.bitset_words is packed+ assert prepared.prepared == (False, True)+ assert prepared.eligible_count == 1+ assert telemetry.as_dict()["filter_words_packed"] is True++ cache_hit_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)+ cached_host_filter = index._prepare_host_filter(+ filters,+ lambda _filters: (_ for _ in ()).throw(AssertionError("resolver should not run")),+ lambda _labels: None,+ cache_hit_telemetry,+ )+ assert cached_host_filter is not None+ assert cached_host_filter.cached_metadata is not None+ assert cache_hit_telemetry.filter_cache_hit is True+ assert cache_hit_telemetry.filter_words_packed is True+++def test_legacy_filter_words_keep_packed_origin_false_across_cache_hits():+ index = _index(2)+ filters = {"op": "must", "field": "group", "conds": ["a"]}+ first_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)++ first_host_filter = index._prepare_host_filter(+ filters,+ lambda _filters: ([0b01], 1),+ lambda _labels: None,+ first_telemetry,+ )+ assert first_host_filter is not None+ resolved = first_host_filter.resolved_native_filter+ assert resolved is not None+ index._prepare_filter(filters, [1, 2], lambda _filters: ([0b01], 1), resolved)++ cache_hit_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)+ index._prepare_host_filter(+ filters,+ lambda _filters: (_ for _ in ()).throw(AssertionError("resolver should not run")),+ lambda _labels: None,+ cache_hit_telemetry,+ )++ assert first_telemetry.filter_words_packed is False+ assert cache_hit_telemetry.filter_cache_hit is True+ assert cache_hit_telemetry.filter_words_packed is False+++def test_in_gate_resolver_fallback_retains_packed_origin_and_native_token():+ index = _index(2)+ filters = {"op": "must", "field": "group", "conds": ["a"]}++ prepared = index._prepare_filter(+ filters,+ [1, 2],+ lambda _filters: (b"\x01\x00\x00\x00", 1, 23),+ )++ assert prepared.filter_words_packed is True+ assert prepared.native_filter_token == 23+ assert prepared.prepared == (True, False)+++def test_runtime_uses_numpy_frombuffer_for_packed_filter_words():+ class _Device:+ def __enter__(self):+ return self++ def __exit__(self, _exc_type, _exc, _traceback):+ return False++ class _Cuda:+ @staticmethod+ def Device(_device_id):+ return _Device()++ class _CuPy:+ uint32 = object()+ cuda = _Cuda()++ def __init__(self):+ self.host_words = None+ self.dtype = None++ def empty(self, shape, dtype):+ assert shape == (2,)+ self.dtype = dtype+ owner = self++ class _DeviceWords:+ def set(self, values):+ owner.host_words = values++ return _DeviceWords()++ runtime = _CuVSRuntime.__new__(_CuVSRuntime)+ runtime.cp = _CuPy()+ runtime.device_id = 0+ packed = b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa"++ result = runtime.prepare_filter_words(packed)++ assert result is not None+ assert isinstance(runtime.cp.host_words, np.ndarray)+ assert runtime.cp.host_words.dtype.str == "<u4"+ assert runtime.cp.host_words.tolist() == [0x01020304, 0xAABBCCDD]+ assert runtime.cp.host_words.base is packed+ assert runtime.cp.dtype is runtime.cp.uint32+++def test_runtime_rejects_malformed_packed_words_before_cupy_conversion():+ runtime = _CuVSRuntime.__new__(_CuVSRuntime)+ runtime.cp = object()+ runtime.device_id = 0++ with pytest.raises(ValueError, match="multiple of 4 bytes"):+ # Validation happens before touching CUDA, so malformed native output+ # cannot enter the device-copy path.+ runtime.prepare_filter_words(b"\x00")++[email protected]("failure_type", [RuntimeError, KeyboardInterrupt])+def test_failed_packed_copy_releases_device_words_in_captured_scope(failure_type):+ class _State:+ current_device = 9+ releases = []++ class _Device:+ def __init__(self, device_id):+ self.device_id = device_id++ def __enter__(self):+ self.previous = _State.current_device+ _State.current_device = self.device_id+ return self++ def __exit__(self, _exc_type, _exc, _traceback):+ _State.current_device = self.previous+ return False++ class _DeviceWords:+ def set(self, _values):+ raise failure_type("injected packed copy failure")++ def __del__(self):+ _State.releases.append(_State.current_device)++ class _Cuda:+ Device = _Device++ class _CuPy:+ uint32 = object()+ cuda = _Cuda()++ @staticmethod+ def empty(_shape, dtype):+ assert dtype is _CuPy.uint32+ return _DeviceWords()++ runtime = _CuVSRuntime.__new__(_CuVSRuntime)+ runtime.cp = _CuPy()+ runtime.device_id = 3++ with pytest.raises(failure_type, match="injected packed copy failure"):+ runtime.prepare_filter_words(b"\x01\x00\x00\x00")+ gc.collect()++ assert _State.releases == [3]+ assert _State.current_device == 9diff --git a/tests/vectordb/test_cuvs_index.py b/tests/vectordb/test_cuvs_index.pyindex a67930fa1..5ded0d3e9 100644--- a/tests/vectordb/test_cuvs_index.py+++ b/tests/vectordb/test_cuvs_index.py@@ -3,10 +3,13 @@import threadingimport time+from array import arrayfrom concurrent.futures import ThreadPoolExecutor+from contextlib import nullcontextimport pytest+from openviking.storage.vectordb.index import cuvs_index as cuvs_index_modulefrom openviking.storage.vectordb.index.cuvs_index import (CuVSDenseIndex,CuVSMemoryBudgetError,@@ -15,6 +18,7 @@ from openviking.storage.vectordb.index.cuvs_index import (CuVSUnavailableError,UnsupportedCuVSFilterError,_CuVSRuntime,+ _PackedFP32Rows,estimate_cuvs_memory,matches_filter,)@@ -33,6 +37,8 @@ class FakeCuVSRuntime:self.free_memory_bytes = 1 << 60self.total_memory_bytes = 1 << 60self.release_count = 0+ self.close_count = 0+ self.close_error = Nonedef build(self, dataset):self.dataset = [list(vector) for vector in dataset]@@ -73,6 +79,11 @@ class FakeCuVSRuntime:return Falsedef close(self):+ self.close_count += 1+ if self.close_error is not None:+ error = self.close_error+ self.close_error = None+ raise errorself.closed = True@@ -92,6 +103,7 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():self.resource_count = 0self.fail_build = Falseself.fail_search = False+ self.fail_host_copy = Falsedef set_current(self, device_id):self.local.current = device_id@@ -181,6 +193,8 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():@staticmethoddef asnumpy(values):spy.require_captured_device("asnumpy")+ if spy.fail_host_copy:+ raise RuntimeError("injected host copy failure")return values@staticmethod@@ -247,8 +261,10 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():runtime._owned_resources = []runtime._resource_limit = 1runtime._resources_closed = False- runtime._use_explicit_resources = False- runtime.set_max_concurrent_searches(2)+ # The default serialized setting must still use an explicit synchronized+ # resource. This is the production shape used when benchmark callers+ # churn worker threads while GPU admission remains at one search.+ runtime.set_max_concurrent_searches(1)constructing_thread = threading.get_ident()def use_runtime_from_worker():@@ -293,6 +309,11 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():assert spy.current() == 9assert spy.releases == [("resources-1", 3)]assert runtime.search(runtime_index, [1.0, 0.0], 1, [True]) == ([0], [0.25])+ spy.fail_host_copy = True+ with pytest.raises(RuntimeError, match="injected host copy failure"):+ runtime.search(runtime_index, [1.0, 0.0], 1, [True])+ spy.fail_host_copy = False+ assert spy.releases == [("resources-1", 3), ("resources-2", 3)]runtime.release_index()assert spy.current() == 9return runtime_index@@ -317,18 +338,23 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():assert not thread.is_alive()assert churn_errors == []- assert spy.resource_count == 2- assert len(runtime._owned_resources) <= 2+ assert spy.resource_count == 3+ assert len(runtime._owned_resources) <= 1assert len(runtime._available_resources) == 1- # A failed search discarded the first resource under device 3. The bounded- # pool reused the replacement across short-lived threads and retains it- # until close can also destroy it on the captured device.- assert spy.releases == [("resources-1", 3)]+ # Search and post-sync host-copy failures discarded the first two resources+ # under device 3. The bounded pool reuses the replacement across+ # short-lived threads and retains it until close can also destroy it on the+ # captured device.+ assert spy.releases == [("resources-1", 3), ("resources-2", 3)]spy.set_current(9)runtime.close()assert spy.current() == 9- assert spy.releases == [("resources-1", 3), ("resources-2", 3)]+ assert spy.releases == [+ ("resources-1", 3),+ ("resources-2", 3),+ ("resources-3", 3),+ ]assert {"memory_info",@@ -356,6 +382,170 @@ def delta(label, vector, **fields):return DeltaRecord(label=label, vector=vector, fields=json.dumps(fields))+def wait_for_preflight_participants(index, expected, timeout=5.0):+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ with index._lock:+ if any(flight.participants >= expected for flight in index._preflight_flights.values()):+ return+ time.sleep(0.001)+ raise AssertionError(f"Timed out waiting for {expected} preflight participants")+++class _PreflightAbort(BaseException):+ """Model an asynchronous owner abort that ordinary Exception misses."""+++def test_cuvs_host_shadow_is_immutable_compact_fp32():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={},+ config={"dtype": "float16"},+ runtime=runtime,+ )+ source = [0.1, -3.25]+ index.add_candidates([candidate(10, source), candidate(20, [2.0, 4.0])])++ assert index.host_shadow_nbytes == 2 * 2 * 4+ stored = index._records[10].vector+ assert isinstance(stored, bytes)+ assert len(stored) == 2 * 4+ assert list(memoryview(stored).cast("f")) == pytest.approx([0.1, -3.25])++ # The host snapshot owns FP32 values instead of references to caller-owned+ # Python floats, even when the configured device dataset uses FP16.+ source[:] = [9.0, 9.0]+ rebuild = index.prepare_rebuild()+ assert rebuild is not None+ assert rebuild.labels == (10, 20)+ assert runtime.dataset[0] == pytest.approx([0.1, -3.25])+ assert index.commit_rebuild(rebuild)++ # Updating an existing label keeps dict/row order; delete plus reinsert+ # retains the previous append-at-end behavior.+ index.upsert([delta(10, [5.0, 6.0])])+ replacement = index.prepare_rebuild()+ assert replacement is not None+ assert replacement.labels == (10, 20)+ assert runtime.dataset[0] == pytest.approx([5.0, 6.0])+ assert index.commit_rebuild(replacement)+ index.delete([DeltaRecord(label=10)])+ index.upsert([delta(10, [7.0, 8.0])])+ reinserted = index.prepare_rebuild()+ assert reinserted is not None+ assert reinserted.labels == (20, 10)+ assert runtime.dataset[1] == pytest.approx([7.0, 8.0])++ index.close()+ assert index.size == 0+ assert index.host_shadow_nbytes == 0+ assert runtime.closed+++ ("dtype", "expected_upload_dtype"),+ [("float32", "float32"), ("float16", "float16")],+)+def test_cuvs_runtime_uploads_packed_rows_in_bounded_batches(+ monkeypatch, dtype, expected_upload_dtype+):+ class DeviceDataset:+ def __init__(self, shape, device_dtype):+ self.rows = [[None] * shape[1] for _ in range(shape[0])]+ self.dtype = device_dtype+ self.upload_dtypes = []++ def __getitem__(self, row_slice):+ owner = self++ class DeviceSlice:+ def set(self, host_rows):+ owner.upload_dtypes.append(str(host_rows.dtype))+ owner.rows[row_slice] = host_rows.tolist()++ return DeviceSlice()++ class FakeCuPy:+ float16 = "float16"+ float32 = "float32"++ @staticmethod+ def empty(shape, dtype):+ return DeviceDataset(shape, dtype)++ class FakeBruteForce:+ @staticmethod+ def build(dataset, metric):+ assert metric == "inner_product"+ return dataset++ runtime = _CuVSRuntime.__new__(_CuVSRuntime)+ runtime.cp = FakeCuPy()+ runtime.device_scope = lambda: nullcontext()+ runtime.device_dtype = FakeCuPy.float16 if dtype == "float16" else FakeCuPy.float32+ runtime.dtype = dtype+ runtime.algorithm = "brute_force"+ runtime.metric = "inner_product"+ runtime.brute_force = FakeBruteForce()+ runtime.cagra = object()+ runtime.build_params = {}++ rows = _PackedFP32Rows(+ tuple(array("f", values).tobytes() for values in ([1, 2], [3, 4], [5, 6])),+ dimension=2,+ )+ monkeypatch.setattr(cuvs_index_module, "_FP32_UPLOAD_BATCH_BYTES", 8)+ built = runtime.build(rows)++ assert built.dataset.rows == [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]+ assert built.dataset.upload_dtypes == [expected_upload_dtype] * 3+ assert rows.nbytes == 3 * 2 * 4+ assert [end - start for start, end, _payload in rows.iter_packed_batches(8)] == [1, 1, 1]+++def test_mutation_during_packed_rebuild_keeps_candidate_snapshot_immutable():+ class BlockingRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__()+ self.first_build_started = threading.Event()+ self.resume_first_build = threading.Event()++ def build(self, dataset):+ if self.build_count == 0:+ self.first_build_started.set()+ assert self.resume_first_build.wait(timeout=5)+ return super().build(dataset)++ runtime = BlockingRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={},+ config={},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0])])++ with ThreadPoolExecutor(max_workers=1) as executor:+ pending = executor.submit(index.prepare_rebuild)+ assert runtime.first_build_started.wait(timeout=5)+ index.upsert([delta(1, [0.0, 1.0])])+ runtime.resume_first_build.set()+ stale = pending.result(timeout=5)++ assert stale is not None+ assert runtime.dataset == [[1.0, 0.0]]+ assert index.commit_rebuild(stale) is False+ replacement = index.prepare_rebuild()+ assert replacement is not None+ assert runtime.dataset == [[0.0, 1.0]]+ assert index.commit_rebuild(replacement)++def test_cuvs_rebuild_objects_are_released_on_the_captured_device():class DeviceOwned:def __init__(self, runtime, name):@@ -432,6 +622,84 @@ def test_cuvs_rebuild_objects_are_released_on_the_captured_device():assert runtime.current_device == 9+def test_prepared_filter_lru_and_close_release_on_captured_device():+ class DeviceOwnedFilter:+ def __init__(self, runtime, name, values):+ self.runtime = runtime+ self.name = name+ self.values = values++ def __getitem__(self, index):+ return self.values[index]++ def __del__(self):+ self.runtime.released.append((self.name, self.runtime.current_device))++ class LifecycleRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__()+ self.current_device = 9+ self.prepared_count = 0+ self.released = []++ def device_scope(self):+ runtime = self++ class DeviceContext:+ def __enter__(self):+ self.previous = runtime.current_device+ runtime.current_device = 3++ def __exit__(self, _exc_type, _exc, _traceback):+ runtime.current_device = self.previous++ return DeviceContext()++ def prepare_filter_words(self, words):+ with self.device_scope():+ self.prepared_count += 1+ values = tuple(bool(words[0] & (1 << row)) for row in range(2))+ return DeviceOwnedFilter(self, f"filter-{self.prepared_count}", values)++ def close(self):+ assert self.current_device == 3+ super().close()++ runtime = LifecycleRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"filter_cache_size": 1},+ runtime=runtime,+ )+ index.add_candidates(+ [+ candidate(1, [1.0, 0.0], account_id="a"),+ candidate(2, [0.0, 1.0], account_id="b"),+ ]+ )++ def resolve(filters):+ return ([0b01], 1) if filters["conds"][0] == "a" else ([0b10], 1)++ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ filter_b = {"op": "must", "field": "account_id", "conds": ["b"]}++ def registrar(_labels):+ return None++ assert index.search([1.0, 0.0], 1, filter_a, resolve, registrar)[0] == [1]+ assert runtime.released == []+ assert index.search([0.0, 1.0], 1, filter_b, resolve, registrar)[0] == [2]+ assert runtime.released == [("filter-1", 3)]++ index.close()+ assert runtime.released == [("filter-1", 3), ("filter-2", 3)]+ assert runtime.current_device == 9++def test_inflight_snapshot_is_released_on_captured_device_after_replacement():class DeviceOwned:def __init__(self, runtime, name):@@ -736,28 +1004,385 @@ def test_cuvs_gpu_search_gate_serializes_kernels_by_default():with self.active_lock:self.active -= 1- runtime = SerialRuntime()+ runtime = SerialRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={},+ config={},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0])])+ assert index.search([1.0, 0.0], 1, None)[0] == [1]++ telemetry = [CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False) for _ in range(4)]+ with ThreadPoolExecutor(max_workers=4) as executor:+ futures = [+ executor.submit(index.search, [1.0, 0.0], 1, None, None, None, item)+ for item in telemetry+ ]+ assert [future.result(timeout=5)[0] for future in futures] == [[1]] * 4++ assert runtime.peak_active == 1+ assert max(item.queue_ms for item in telemetry) > 0+ assert max(item.gpu_gate_queue_ms for item in telemetry) > 0+ assert all(item.queue_ms >= item.gpu_gate_queue_ms for item in telemetry)+++def test_native_filter_resolution_runs_before_gpu_search_admission():+ class BlockingRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__()+ self.block_next = False+ self.search_started = threading.Event()+ self.resume_search = threading.Event()++ def search(self, index, query, limit, mask):+ if self.block_next:+ self.block_next = False+ self.search_started.set()+ assert self.resume_search.wait(timeout=5)+ return super().search(index, query, limit, mask)++ runtime = BlockingRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])+ assert index.search([1.0, 0.0], 1, None)[0] == [1]++ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ resolver_ran = threading.Event()+ telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)++ def resolve(_filters):+ resolver_ran.set()+ return [0b1], 1++ runtime.block_next = True+ with ThreadPoolExecutor(max_workers=2) as executor:+ occupying = executor.submit(index.search, [1.0, 0.0], 1, None)+ assert runtime.search_started.wait(timeout=5)+ filtered = executor.submit(+ index.search,+ [1.0, 0.0],+ 1,+ filter_a,+ resolve,+ lambda _labels: None,+ telemetry,+ )+ # The resolver is host work and must complete while the sole GPU+ # admission permit is still occupied by the first search.+ assert resolver_ran.wait(timeout=5)+ runtime.resume_search.set()+ assert occupying.result(timeout=5)[0] == [1]+ assert filtered.result(timeout=5)[0] == [1]++ assert telemetry.gpu_gate_queue_ms > 0+ assert telemetry.queue_ms >= telemetry.gpu_gate_queue_ms+++def test_explicit_same_filter_host_resolution_is_singleflight():+ class WordPreparingRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__()+ self.prepare_filter_words_count = 0++ def prepare_filter_words(self, words):+ self.prepare_filter_words_count += 1+ return (bool(words[0] & 1),)++ runtime = WordPreparingRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"filter_cache_size": 2},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ start = threading.Barrier(2)+ resolver_started = threading.Event()+ release_resolver = threading.Event()+ resolver_calls = 0++ def resolve(_filters):+ nonlocal resolver_calls+ resolver_calls += 1+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ return [0b1], 1++ def invoke():+ start.wait(timeout=5)+ return index.search(+ [1.0, 0.0],+ 1,+ filter_a,+ resolve,+ lambda _labels: None,+ )++ with ThreadPoolExecutor(max_workers=2) as executor:+ futures = [executor.submit(invoke) for _ in range(2)]+ assert resolver_started.wait(timeout=5)+ wait_for_preflight_participants(index, 2)+ release_resolver.set()+ assert [future.result(timeout=5)[0] for future in futures] == [[1], [1]]++ assert resolver_calls == 1+ assert runtime.prepare_filter_words_count == 1+ assert not index._preflight_flights+++def test_mutation_discards_host_filter_resolved_for_old_generation():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"filter_cache_size": 2},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="old")])+ filter_new = {"op": "must", "field": "account_id", "conds": ["new"]}+ resolver_started = threading.Event()+ release_resolver = threading.Event()+ resolver_calls = 0+ registered_layouts = []++ def resolve(_filters):+ nonlocal resolver_calls+ resolver_calls += 1+ if resolver_calls == 1:+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ return [0b0], 0+ return [0b10], 1++ with ThreadPoolExecutor(max_workers=1) as executor:+ future = executor.submit(+ index.search,+ [0.0, 1.0],+ 1,+ filter_new,+ resolve,+ lambda labels: registered_layouts.append(list(labels)),+ )+ assert resolver_started.wait(timeout=5)+ index.add_candidates([candidate(2, [0.0, 1.0], account_id="new")])+ release_resolver.set()+ assert future.result(timeout=5)[0] == [2]++ assert resolver_calls == 2+ assert registered_layouts == [[1], [1, 2]]+ assert runtime.build_count == 1+++def test_close_during_host_filter_resolution_rejects_search_before_gpu_admission():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ resolver_started = threading.Event()+ release_resolver = threading.Event()++ def resolve(_filters):+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ return [0b1], 1++ with ThreadPoolExecutor(max_workers=1) as executor:+ future = executor.submit(+ index.search,+ [1.0, 0.0],+ 1,+ filter_a,+ resolve,+ lambda _labels: None,+ )+ assert resolver_started.wait(timeout=5)+ index.close()+ release_resolver.set()+ with pytest.raises(RuntimeError, match="dense index is closed"):+ future.result(timeout=5)++ assert runtime.build_count == 0+ assert runtime.closed+++def test_close_while_warmed_filter_waits_for_gpu_gate_does_not_borrow_device_cache():+ class BlockingWordRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__()+ self.block_next = False+ self.search_started = threading.Event()+ self.resume_search = threading.Event()++ def prepare_filter_words(self, words):+ return (bool(words[0] & 1),)++ def search(self, index, query, limit, mask):+ if self.block_next:+ self.block_next = False+ self.search_started.set()+ assert self.resume_search.wait(timeout=5)+ return super().search(index, query, limit, mask)++ runtime = BlockingWordRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={},+ runtime=runtime,+ )+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}++ def resolver(_filters):+ return [0b1], 1++ def registrar(_labels):+ return None++ assert index.search([1.0, 0.0], 1, filter_a, resolver, registrar)[0] == [1]++ host_prepared = threading.Event()+ original_prepare = index._prepare_host_filter++ def observe_prepare(*args, **kwargs):+ prepared = original_prepare(*args, **kwargs)+ host_prepared.set()+ return prepared++ index._prepare_host_filter = observe_prepare+ runtime.block_next = True+ with ThreadPoolExecutor(max_workers=3) as executor:+ occupying = executor.submit(index.search, [1.0, 0.0], 1, None)+ assert runtime.search_started.wait(timeout=5)+ waiting = executor.submit(+ index.search,+ [1.0, 0.0],+ 1,+ filter_a,+ resolver,+ registrar,+ )+ assert host_prepared.wait(timeout=5)++ closing = executor.submit(index.close)+ deadline = time.monotonic() + 5+ while not index._closed and time.monotonic() < deadline:+ time.sleep(0.001)+ assert index._closed++ runtime.resume_search.set()+ assert occupying.result(timeout=5)[0] == [1]+ with pytest.raises(RuntimeError, match="dense index is closed"):+ waiting.result(timeout=5)+ closing.result(timeout=5)++ assert runtime.closed+++def test_device_cache_eviction_before_admission_uses_one_in_gate_fallback():+ class WordPreparingRuntime(FakeCuVSRuntime):+ def __init__(self):+ super().__init__()+ self.prepare_filter_words_count = 0++ def prepare_filter_words(self, words):+ self.prepare_filter_words_count += 1+ return tuple(bool(words[0] & (1 << row)) for row in range(2))++ runtime = WordPreparingRuntime()index = CuVSDenseIndex(dimension=2,distance="ip",normalize_vectors=False,- field_types={},- config={},+ field_types={"account_id": "string"},+ config={"filter_cache_size": 1, "max_concurrent_gpu_searches": 2},runtime=runtime,)- index.add_candidates([candidate(1, [1.0, 0.0])])- assert index.search([1.0, 0.0], 1, None)[0] == [1]-- telemetry = [CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False) for _ in range(4)]- with ThreadPoolExecutor(max_workers=4) as executor:- futures = [- executor.submit(index.search, [1.0, 0.0], 1, None, None, None, item)- for item in telemetry+ index.add_candidates(+ [+ candidate(1, [1.0, 0.0], account_id="a"),+ candidate(2, [0.0, 1.0], account_id="b"),]- assert [future.result(timeout=5)[0] for future in futures] == [[1]] * 4+ )+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ filter_b = {"op": "must", "field": "account_id", "conds": ["b"]}+ resolver_calls = {"a": 0, "b": 0}- assert runtime.peak_active == 1- assert max(item.queue_ms for item in telemetry) > 0+ def resolve(filters):+ account_id = filters["conds"][0]+ resolver_calls[account_id] += 1+ return ([0b01], 1) if account_id == "a" else ([0b10], 1)++ def registrar(_labels):+ return None++ # Warm A so the delayed search below initially observes a device-cache hit.+ assert index.search([1.0, 0.0], 1, filter_a, resolve, registrar)[0] == [1]+ assert resolver_calls == {"a": 1, "b": 0}++ a_host_prepared = threading.Event()+ release_a = threading.Event()+ original_prepare = index._prepare_host_filter++ def pause_a_after_host_prepare(filters, *args, **kwargs):+ prepared = original_prepare(filters, *args, **kwargs)+ if filters == filter_a:+ a_host_prepared.set()+ assert release_a.wait(timeout=5)+ return prepared++ index._prepare_host_filter = pause_a_after_host_prepare+ eviction_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)+ with ThreadPoolExecutor(max_workers=1) as executor:+ delayed_a = executor.submit(+ index.search,+ [1.0, 0.0],+ 1,+ filter_a,+ resolve,+ registrar,+ eviction_telemetry,+ )+ assert a_host_prepared.wait(timeout=5)++ # Materializing B evicts A from the size-one device LRU after A's host+ # context was captured but before A consumes an admission permit.+ assert index.search([0.0, 1.0], 1, filter_b, resolve, registrar)[0] == [2]+ release_a.set()+ assert delayed_a.result(timeout=5)[0] == [1]++ # A resolves once to warm and exactly once in the admitted eviction+ # fallback; it does not loop under alternating hot-key churn.+ assert resolver_calls == {"a": 2, "b": 1}+ assert runtime.prepare_filter_words_count == 3+ assert eviction_telemetry.filter_cache_hit is False+ assert eviction_telemetry.filter_cache_eviction_fallback is True+ assert eviction_telemetry.as_dict()["filter_cache_eviction_fallback"] is Truedef test_inflight_search_keeps_immutable_snapshot_during_rebuild():@@ -1133,6 +1758,245 @@ def test_auto_mode_caches_native_route_for_selective_filter():assert runtime.prepare_filter_count == 0+ ("evaluation", "expected_route"),+ [(([], 1), 1), (([0b11], 2), None)],+ ids=["narrow-native", "broad-gpu"],+)+def test_auto_mode_same_key_preflight_is_singleflight(evaluation, expected_route):+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates(+ [+ candidate(10, [1.0, 0.0], account_id="a"),+ candidate(20, [0.0, 1.0], account_id="a"),+ ]+ )+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ start = threading.Barrier(4)+ resolver_started = threading.Event()+ release_resolver = threading.Event()+ resolver_calls = 0++ def resolve(_filters):+ nonlocal resolver_calls+ resolver_calls += 1+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ return evaluation++ def invoke():+ start.wait(timeout=5)+ return index.preflight_native_count(+ filter_a,+ resolve,+ lambda _labels: None,+ )++ with ThreadPoolExecutor(max_workers=4) as executor:+ futures = [executor.submit(invoke) for _ in range(4)]+ assert resolver_started.wait(timeout=5)+ wait_for_preflight_participants(index, 4)+ release_resolver.set()+ routes = [future.result(timeout=5) for future in futures]++ assert routes == [expected_route] * 4+ assert resolver_calls == 1+ assert not index._preflight_flights+++ "error_type",+ [RuntimeError, _PreflightAbort],+ ids=["exception", "base-exception"],+)+def test_auto_mode_same_key_preflight_broadcasts_error_and_allows_retry(error_type):+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(10, [1.0, 0.0], account_id="a")])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ start = threading.Barrier(4)+ resolver_started = threading.Event()+ release_resolver = threading.Event()+ resolver_calls = 0++ def failing_resolve(_filters):+ nonlocal resolver_calls+ resolver_calls += 1+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ raise error_type("preflight failed")++ def invoke():+ start.wait(timeout=5)+ return index.preflight_native_count(+ filter_a,+ failing_resolve,+ lambda _labels: None,+ )++ with ThreadPoolExecutor(max_workers=4) as executor:+ futures = [executor.submit(invoke) for _ in range(4)]+ assert resolver_started.wait(timeout=5)+ wait_for_preflight_participants(index, 4)+ release_resolver.set()+ for future in futures:+ with pytest.raises(error_type, match="preflight failed"):+ future.result(timeout=5)++ assert resolver_calls == 1+ assert not index._preflight_flights+ assert (+ index.preflight_native_count(+ filter_a,+ lambda _filters: ([], 1),+ lambda _labels: None,+ )+ == 1+ )+++def test_auto_mode_mutation_wakes_same_key_preflight_waiters_and_drops_old_result():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(10, [1.0, 0.0], account_id="a")])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ start = threading.Barrier(2)+ resolver_started = threading.Event()+ release_resolver = threading.Event()+ resolver_calls = 0++ def resolve(_filters):+ nonlocal resolver_calls+ resolver_calls += 1+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ return [], 1++ def invoke():+ start.wait(timeout=5)+ return index.preflight_native_count(+ filter_a,+ resolve,+ lambda _labels: None,+ )++ with ThreadPoolExecutor(max_workers=2) as executor:+ futures = [executor.submit(invoke) for _ in range(2)]+ assert resolver_started.wait(timeout=5)+ wait_for_preflight_participants(index, 2)+ index.add_candidates([candidate(20, [0.0, 1.0], account_id="a")])++ deadline = time.monotonic() + 5+ while not any(future.done() for future in futures) and time.monotonic() < deadline:+ time.sleep(0.001)+ assert any(future.done() for future in futures)++ release_resolver.set()+ assert [future.result(timeout=5) for future in futures] == [None, None]++ assert resolver_calls == 1+ assert not index._preflight_flights+ assert (+ index.preflight_native_count(+ filter_a,+ lambda _filters: ([0b11], 2),+ lambda _labels: None,+ )+ is None+ )+++def test_auto_mode_close_wakes_same_key_preflight_waiters():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(10, [1.0, 0.0], account_id="a")])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}+ start = threading.Barrier(2)+ resolver_started = threading.Event()+ release_resolver = threading.Event()++ def resolve(_filters):+ resolver_started.set()+ assert release_resolver.wait(timeout=5)+ return [], 1++ def invoke():+ start.wait(timeout=5)+ return index.preflight_native_count(+ filter_a,+ resolve,+ lambda _labels: None,+ )++ with ThreadPoolExecutor(max_workers=2) as executor:+ futures = [executor.submit(invoke) for _ in range(2)]+ assert resolver_started.wait(timeout=5)+ wait_for_preflight_participants(index, 2)+ index.close()+ assert runtime.closed+ assert not index._preflight_flights+ release_resolver.set()+ assert [future.result(timeout=5) for future in futures] == [None, None]+++def test_close_retries_runtime_cleanup_and_rejects_new_searches():+ runtime = FakeCuVSRuntime()+ runtime.close_error = RuntimeError("close failed")+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={},+ config={},+ runtime=runtime,+ )++ with pytest.raises(RuntimeError, match="close failed"):+ index.close()+ assert not index._close_complete+ with pytest.raises(RuntimeError, match="dense index is closed"):+ index.search([1.0, 0.0], 1, None)++ index.close()+ index.close()+ assert index._close_complete+ assert runtime.closed+ assert runtime.close_count == 2++def test_auto_mode_preflights_different_filters_concurrently():runtime = FakeCuVSRuntime()index = CuVSDenseIndex(@@ -1366,6 +2230,105 @@ def test_auto_mode_wide_filter_reuses_preflight_bitmap_after_build():assert runtime.build_count == 1[email protected]("words", [[], [0xFFFFFFFF]], ids=["empty", "short"])+def test_auto_mode_rejects_incomplete_gpu_filter_bitset(words):+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(33)])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}++ with pytest.raises(RuntimeError, match="incomplete bitset for GPU routing"):+ index.preflight_native_count(+ filter_a,+ lambda _filters: (words, 33),+ lambda _labels: None,+ )+++def test_explicit_mode_rejects_short_gpu_bitset_below_auto_threshold():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 100},+ runtime=runtime,+ auto_memory=False,+ )+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(33)])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}++ with pytest.raises(RuntimeError, match="incomplete bitset for GPU routing"):+ index.search(+ [1.0, 0.0],+ 1,+ filter_a,+ lambda _filters: ([0b1], 1),+ lambda _labels: None,+ )+++def test_auto_mode_accepts_omitted_bitset_for_native_route():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(33)])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}++ assert (+ index.preflight_native_count(+ filter_a,+ lambda _filters: ([], 1),+ lambda _labels: None,+ )+ == 1+ )+++def test_auto_mode_does_not_validate_stale_filter_projection_after_mutation():+ runtime = FakeCuVSRuntime()+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={"account_id": "string"},+ config={"auto_filter_native_threshold": 1},+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(32)])+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}++ def resolve_and_mutate(_filters):+ index.add_candidates([candidate(1000, [0.0, 1.0], account_id="a")])+ return [0xFFFFFFFF], 32++ assert (+ index.preflight_native_count(+ filter_a,+ resolve_and_mutate,+ lambda _labels: None,+ )+ is None+ )++def test_auto_mode_retains_native_filter_token_for_selective_route():runtime = FakeCuVSRuntime()index = CuVSDenseIndex(diff --git a/tests/vectordb/test_engine_filter_packed_native_abi.py b/tests/vectordb/test_engine_filter_packed_native_abi.pynew file mode 100644index 000000000..426b957b2--- /dev/null+++ b/tests/vectordb/test_engine_filter_packed_native_abi.py@@ -0,0 +1,77 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import json++import pytest++import openviking.storage.vectordb.engine as engine+++def _native_packed_abi_available() -> bool:+ backend = getattr(engine, "_BACKEND", None)+ return backend is not None and hasattr(backend, "_index_engine_evaluate_filter_packed")+++ not _native_packed_abi_available(),+ reason="the packaged native extension predates the additive packed filter ABI",+)+def test_native_packed_filter_abi_has_exact_little_endian_layout_and_shapes():+ config = json.dumps(+ {+ "CollectionName": "packed_filter_abi_test",+ "IndexName": "default",+ "VectorIndex": {+ "IndexType": "flat",+ "ElementCount": 0,+ "MaxElementCount": 96,+ "Dimension": 1,+ "Distance": "l2",+ "Quant": "float",+ },+ "ScalarIndex": [{"FieldName": "uri", "FieldType": "path"}],+ }+ )+ index = engine.IndexEngine(config)+ selected_rows = {0, 8, 16, 24, 31, 32, 63, 64}+ labels = list(range(1000, 1065))+ requests = []+ for row, label in enumerate(labels):+ request = engine.AddDataRequest()+ request.label = label+ request.vector = [float(row)]+ request.fields_str = json.dumps(+ {"uri": f"/keep/item-{row}" if row in selected_rows else f"/drop/item-{row}"}+ )+ requests.append(request)+ assert index.add_data(requests) == 0+ assert index.set_filter_layout(labels) == 0++ keep_filter = json.dumps({"op": "must", "field": "uri", "conds": ["/keep"], "para": "-d=-1"})+ missing_filter = json.dumps(+ {"op": "must", "field": "uri", "conds": ["/missing"], "para": "-d=-1"}+ )+ expected_words = [0x81010101, 0x80000001, 0x00000001]+ expected_bytes = b"\x01\x01\x01\x81\x01\x00\x00\x80\x01\x00\x00\x00"++ legacy = index.evaluate_filter(keep_filter)+ generic = index.evaluate_filter_packed(keep_filter)+ cached = index.evaluate_filter_packed(keep_filter, max_cached_candidates=10)+ routed_narrow = index.evaluate_filter_for_routing_packed(keep_filter, native_threshold=10)+ routed_wide = index.evaluate_filter_for_routing_packed(keep_filter, native_threshold=1)+ empty = index.evaluate_filter_packed(missing_filter)++ assert legacy.bitset_words == expected_words+ assert generic.bitset_words == expected_bytes+ assert generic.eligible_count == 8+ assert generic.native_filter_token == 0+ assert cached.bitset_words == expected_bytes+ assert cached.native_filter_token > 0+ assert routed_narrow.bitset_words == b""+ assert routed_narrow.eligible_count == 8+ assert routed_narrow.native_filter_token > 0+ assert routed_wide.bitset_words == expected_bytes+ assert routed_wide.native_filter_token == 0+ assert empty.bitset_words == b"\x00" * 12+ assert empty.eligible_count == 0diff --git a/tests/vectordb/test_engine_filter_routing_abi.py b/tests/vectordb/test_engine_filter_routing_abi.pynew file mode 100644index 000000000..b07d7ea66--- /dev/null+++ b/tests/vectordb/test_engine_filter_routing_abi.py@@ -0,0 +1,190 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import pytest++from openviking.storage.vectordb.engine._python_api import build_abi3_exports+++class _CurrentBackend:+ def __init__(self):+ self.calls = []++ def _new_index_engine(self, path_or_json):+ self.calls.append(("new", path_or_json))+ return "engine-handle"++ def _index_engine_evaluate_filter(self, handle, dsl):+ self.calls.append(("generic", handle, dsl))+ return {"eligible_count": 1, "bitset_words": [1], "native_filter_token": 0}++ def _index_engine_evaluate_filter_cached(self, handle, dsl, threshold):+ self.calls.append(("cached", handle, dsl, threshold))+ return {"eligible_count": 1, "bitset_words": [1], "native_filter_token": 7}++ def _index_engine_evaluate_filter_for_routing(self, handle, dsl, threshold):+ self.calls.append(("routed", handle, dsl, threshold))+ if threshold == 0:+ return {"eligible_count": 1, "bitset_words": [1], "native_filter_token": 0}+ return {"eligible_count": 1, "bitset_words": [], "native_filter_token": 11}+++class _LegacyBackend:+ def __init__(self):+ self.calls = []++ def _new_index_engine(self, path_or_json):+ return "legacy-handle"++ def _index_engine_evaluate_filter(self, handle, dsl):+ self.calls.append(("generic", handle, dsl))+ return {"eligible_count": 1, "bitset_words": [2], "native_filter_token": 0}++ def _index_engine_evaluate_filter_cached(self, handle, dsl, threshold):+ self.calls.append(("cached", handle, dsl, threshold))+ return {"eligible_count": 1, "bitset_words": [2], "native_filter_token": 13}+++class _PackedBackend(_CurrentBackend):+ def _index_engine_evaluate_filter_packed(self, handle, dsl):+ self.calls.append(("generic-packed", handle, dsl))+ return {+ "eligible_count": 2,+ "bitset_words_le": b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa",+ "native_filter_token": 0,+ }++ def _index_engine_evaluate_filter_cached_packed(self, handle, dsl, threshold):+ self.calls.append(("cached-packed", handle, dsl, threshold))+ return {+ "eligible_count": 2,+ "bitset_words_le": b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa",+ "native_filter_token": 17,+ }++ def _index_engine_evaluate_filter_for_routing_packed(self, handle, dsl, threshold):+ self.calls.append(("routed-packed", handle, dsl, threshold))+ if threshold >= 2:+ return {+ "eligible_count": 2,+ "bitset_words_le": b"",+ "native_filter_token": 19,+ }+ return {+ "eligible_count": 2,+ "bitset_words_le": b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa",+ "native_filter_token": 0,+ }+++def test_routed_filter_uses_additive_abi_without_changing_generic_calls():+ backend = _CurrentBackend()+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ routed = index_engine.evaluate_filter_for_routing("dsl", native_threshold=5)+ generic = index_engine.evaluate_filter("dsl", max_cached_candidates=5)++ assert routed.eligible_count == 1+ assert routed.bitset_words == []+ assert routed.native_filter_token == 11+ assert generic.bitset_words == [1]+ assert generic.native_filter_token == 7+ assert ("routed", "engine-handle", "dsl", 5) in backend.calls+ assert ("cached", "engine-handle", "dsl", 5) in backend.calls+++def test_routed_filter_threshold_zero_reaches_new_abi():+ backend = _CurrentBackend()+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ result = index_engine.evaluate_filter_for_routing("dsl", native_threshold=0)++ assert result.eligible_count == 1+ assert result.bitset_words == [1]+ assert result.native_filter_token == 0+ assert ("routed", "engine-handle", "dsl", 0) in backend.calls+++def test_routed_filter_falls_back_with_a_legacy_extension():+ backend = _LegacyBackend()+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ result = index_engine.evaluate_filter_for_routing("dsl", native_threshold=5)+ threshold_zero = index_engine.evaluate_filter_for_routing("dsl", native_threshold=0)++ assert result.eligible_count == 1+ assert result.bitset_words == [2]+ assert result.native_filter_token == 13+ assert threshold_zero.bitset_words == [2]+ assert threshold_zero.native_filter_token == 0+ assert backend.calls == [+ ("cached", "legacy-handle", "dsl", 5),+ ("generic", "legacy-handle", "dsl"),+ ]+++def test_packed_filter_abi_preserves_little_endian_bytes_and_route_shapes():+ backend = _PackedBackend()+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ generic = index_engine.evaluate_filter_packed("wide")+ cached = index_engine.evaluate_filter_packed("wide", max_cached_candidates=4)+ routed = index_engine.evaluate_filter_for_routing_packed("narrow", native_threshold=2)++ expected = b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa"+ assert generic.bitset_words == expected+ assert cached.bitset_words == expected+ assert cached.native_filter_token == 17+ assert routed.bitset_words == b""+ assert routed.eligible_count == 2+ assert routed.native_filter_token == 19+ assert backend.calls[-3:] == [+ ("generic-packed", "engine-handle", "wide"),+ ("cached-packed", "engine-handle", "wide", 4),+ ("routed-packed", "engine-handle", "narrow", 2),+ ]+++def test_packed_filter_methods_fall_back_to_legacy_backend_results():+ backend = _LegacyBackend()+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ generic = index_engine.evaluate_filter_packed("wide")+ cached = index_engine.evaluate_filter_packed("wide", max_cached_candidates=5)+ routed = index_engine.evaluate_filter_for_routing_packed("narrow", native_threshold=5)++ assert generic.bitset_words == [2]+ assert cached.bitset_words == [2]+ assert cached.native_filter_token == 13+ assert routed.bitset_words == [2]+ assert routed.native_filter_token == 13+ assert backend.calls == [+ ("generic", "legacy-handle", "wide"),+ ("cached", "legacy-handle", "wide", 5),+ ("cached", "legacy-handle", "narrow", 5),+ ]+++def test_packed_filter_abi_rejects_a_non_bytes_payload():+ backend = _PackedBackend()+ backend._index_engine_evaluate_filter_packed = lambda _handle, _dsl: {+ "eligible_count": 1,+ "bitset_words_le": [1],+ "native_filter_token": 0,+ }+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ with pytest.raises(TypeError, match="bitset_words_le as bytes"):+ index_engine.evaluate_filter_packed("dsl")+++def test_packed_filter_abi_rejects_a_missing_payload():+ backend = _PackedBackend()+ backend._index_engine_evaluate_filter_packed = lambda _handle, _dsl: {+ "eligible_count": 0,+ "native_filter_token": 0,+ }+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")++ with pytest.raises(KeyError, match="missing bitset_words_le"):+ index_engine.evaluate_filter_packed("dsl")diff --git a/tests/vectordb/test_store_streaming.py b/tests/vectordb/test_store_streaming.pynew file mode 100644index 000000000..7add1d426--- /dev/null+++ b/tests/vectordb/test_store_streaming.py@@ -0,0 +1,221 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import gc+import weakref++import pytest++import openviking.storage.vectordb.engine as engine+from openviking.storage.vectordb.store.data import CandidateData+from openviking.storage.vectordb.store.local_store import (+ STORE_SCAN_PAGE_SIZE,+ StoreEngineProxy,+)+from openviking.storage.vectordb.store.store_manager import StoreManager+++class _TrackedPage(list):+ pass+++class _PagedStorageEngine:+ def __init__(self, rows):+ self.rows = sorted(rows)+ self.page_refs = []+ self.page_sizes = []+ self.peak_live_pages = 0+ self.full_scan_calls = 0++ def seek_range(self, _start_key, _end_key):+ self.full_scan_calls += 1+ raise AssertionError("paged candidate recovery must not use a full range scan")++ def seek_range_page(+ self,+ start_key,+ end_key,+ limit,+ max_bytes,+ start_exclusive=False,+ ):+ gc.collect()+ selected = []+ selected_bytes = 0+ for row in self.rows:+ after_start = row[0] > start_key if start_exclusive else row[0] >= start_key+ if after_start and row[0] < end_key:+ row_bytes = len(row[0].encode("utf-8")) + len(row[1])+ if selected and (+ selected_bytes >= max_bytes or row_bytes > max_bytes - selected_bytes+ ):+ break+ selected.append(row)+ selected_bytes += row_bytes+ if len(selected) == limit:+ break+ page = _TrackedPage(selected)+ self.page_refs.append(weakref.ref(page))+ self.page_sizes.append(len(page))+ self.peak_live_pages = max(+ self.peak_live_pages,+ sum(page_ref() is not None for page_ref in self.page_refs),+ )+ return page+++class _LegacyStorageEngine:+ def __init__(self, rows):+ self.rows = sorted(rows)+ self.full_scan_calls = 0++ def seek_range(self, start_key, end_key):+ self.full_scan_calls += 1+ return [row for row in self.rows if start_key <= row[0] < end_key]+++def _decode_candidate_payloads(monkeypatch):+ candidate_refs = []+ peak_live_candidates = 0++ def decode(payload):+ nonlocal peak_live_candidates+ candidate = CandidateData(label=int(payload.decode("ascii")), vector=[1.0, 0.0])+ candidate_refs.append(weakref.ref(candidate))+ peak_live_candidates = max(+ peak_live_candidates,+ sum(candidate_ref() is not None for candidate_ref in candidate_refs),+ )+ return candidate++ monkeypatch.setattr(CandidateData, "from_bytes", staticmethod(decode))+ return candidate_refs, lambda: peak_live_candidates+++def test_candidate_recovery_bounds_encoded_pages_and_deserialized_rows(monkeypatch):+ prefix = StoreManager.CandsTable+ row_count = STORE_SCAN_PAGE_SIZE + 7+ keys = [str(index) for index in range(row_count)]+ rows = [(prefix + key, key.encode("ascii")) for key in keys]+ storage_engine = _PagedStorageEngine(rows)+ store = StoreEngineProxy(storage_engine)+ manager = StoreManager(store)+ candidate_refs, peak_live_candidates = _decode_candidate_payloads(monkeypatch)++ labels = []+ for candidate in manager.iter_all_cands_data():+ labels.append(candidate.label)++ assert labels == [int(key) for key in sorted(keys)]+ assert storage_engine.full_scan_calls == 0+ assert storage_engine.page_sizes == [STORE_SCAN_PAGE_SIZE, 7, 0]+ assert storage_engine.peak_live_pages == 1+ assert peak_live_candidates() <= 2+ del candidate+ gc.collect()+ assert not any(candidate_ref() is not None for candidate_ref in candidate_refs)+++def test_store_proxy_pages_without_retaining_the_previous_encoded_page():+ rows = [+ ("candidate:" + key, ("value-" + key).encode("ascii"))+ for key in ("1", "10", "2", "20", "3", "30", "4")+ ]+ storage_engine = _PagedStorageEngine(rows)+ store = StoreEngineProxy(storage_engine)++ keys = []+ for key, value in store.iter_all("candidate:", page_size=100, page_bytes=40):+ keys.append(key)+ assert value == ("value-" + key).encode("ascii")++ assert keys == ["1", "10", "2", "20", "3", "30", "4"]+ assert storage_engine.page_sizes == [2, 2, 2, 1, 0]+ assert storage_engine.peak_live_pages == 1+ assert storage_engine.full_scan_calls == 0++ oversized_engine = _PagedStorageEngine(rows)+ oversized_items = list(+ StoreEngineProxy(oversized_engine).iter_all(+ "candidate:",+ page_size=100,+ page_bytes=1,+ )+ )+ assert [key for key, _value in oversized_items] == keys+ assert oversized_engine.page_sizes == [1] * len(rows) + [0]+++def test_store_proxy_rejects_older_engines_without_bounded_scan():+ prefix = StoreManager.CandsTable+ rows = [(prefix + "1", b"1"), (prefix + "2", b"2")]+ storage_engine = _LegacyStorageEngine(rows)+ manager = StoreManager(StoreEngineProxy(storage_engine))++ with pytest.raises(RuntimeError, match="does not support bounded store scans"):+ manager.get_all_cands_data()++ assert storage_engine.full_scan_calls == 0+++def test_store_proxy_rejects_a_nonadvancing_native_cursor():+ class StalledStorageEngine:+ def seek_range_page(+ self,+ _start_key,+ _end_key,+ _limit,+ _max_bytes,+ _start_exclusive=False,+ ):+ return [("candidate:1", b"value")]++ store = StoreEngineProxy(StalledStorageEngine())++ with pytest.raises(RuntimeError, match="did not advance its continuation cursor"):+ list(store.iter_all("candidate:", page_size=1))+++def test_store_proxy_accepts_the_table_prefix_as_the_first_inclusive_key():+ storage_engine = _PagedStorageEngine([("candidate:", b"root"), ("candidate:1", b"child")])++ items = list(StoreEngineProxy(storage_engine).iter_all("candidate:", page_size=1))++ assert items == [("", b"root"), ("1", b"child")]+ assert storage_engine.page_sizes == [1, 1, 0]+++def test_closing_store_stream_releases_the_current_encoded_page():+ rows = [(f"candidate:{index}", b"value") for index in range(4)]+ storage_engine = _PagedStorageEngine(rows)+ stream = StoreEngineProxy(storage_engine).iter_all("candidate:", page_size=2)++ assert next(stream)[0] == "0"+ stream.close()+ gc.collect()++ assert not any(page_ref() is not None for page_ref in storage_engine.page_refs)+++def test_native_volatile_store_paged_scan_abi():+ if engine.ENGINE_VARIANT == "unavailable":+ pytest.skip("No native VectorDB engine is packaged in this test environment")++ store = engine.VolatileStore()+ assert (+ store.put_data(+ ["candidate:1", "candidate:10", "candidate:2"],+ [b"one", b"ten", b"two"],+ )+ == 0+ )++ first = store.seek_range_page("candidate:", "candidate;", 2, 1024, False)+ second = store.seek_range_page(first[-1][0], "candidate;", 2, 1024, True)+ byte_limited = store.seek_range_page("candidate:", "candidate;", 10, 14, False)+ oversized_first = store.seek_range_page("candidate:", "candidate;", 10, 1, False)++ assert first == [("candidate:1", b"one"), ("candidate:10", b"ten")]+ assert second == [("candidate:2", b"two")]+ assert byte_limited == [("candidate:1", b"one")]+ assert oversized_first == [("candidate:1", b"one")]