Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(cuvs): improve recovery scalability and filtered-search efficiency (#3311)

Yuanqing ZHAO2d84a6f13cfe
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 3f5e5fcb2..7bf80d019 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -1295,9 +1295,6 @@ class PersistCollection(LocalCollection):
pass
if index_count > 0:
logger.info("Recovering %d index(es) from %s", index_count, self.index_dir)
- recovery_candidates = (
- self.store_mgr.get_all_cands_data() if self.store_mgr is not None else []
- )
for folder in os.listdir(self.index_dir):
try:
index_dir = safe_join(self.index_dir, folder)
@@ -1329,6 +1326,12 @@ class PersistCollection(LocalCollection):
# all data from the delta log (CandidateData) regardless of when that data was created.
# If we used the default (current time), the index would ignore older data in the log.
# Keep the dense worker stopped until the native snapshot has replayed those deltas.
+ # Create a fresh lazy stream for each index. In particular, cuVS can pack
+ # one candidate at a time instead of retaining every deserialized FP32
+ # vector alongside its compact host shadow during collection recovery.
+ recovery_candidates = (
+ self.store_mgr.iter_all_cands_data() if self.store_mgr is not None else iter(())
+ )
index = PersistentIndex(
name=index_name,
path=self.index_dir,
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.py
index 92a951368..9ae9f1d3e 100644
--- a/openviking/storage/vectordb/engine/_python_api.py
+++ b/openviking/storage/vectordb/engine/_python_api.py
@@ -126,6 +126,7 @@ class Schema:
def get_field_order(self) -> list[FieldMeta]:
return self.field_orders
+
def _get_row_value(row_data: Any, field_name: str, default_value: Any) -> Any:
if isinstance(row_data, dict):
return row_data.get(field_name, default_value)
@@ -350,11 +351,11 @@ class FilterResult:
self,
*,
eligible_count: int = 0,
- bitset_words: list[int] | None = None,
+ bitset_words: list[int] | bytes | None = None,
native_filter_token: int = 0,
):
self.eligible_count = eligible_count
- self.bitset_words = bitset_words or []
+ self.bitset_words = [] if bitset_words is None else bitset_words
self.native_filter_token = native_filter_token
@classmethod
@@ -365,6 +366,19 @@ class FilterResult:
native_filter_token=int(payload.get("native_filter_token", 0)),
)
+ @classmethod
+ def from_packed_backend(cls, payload: dict[str, Any]) -> "FilterResult":
+ if "bitset_words_le" not in payload:
+ raise KeyError("Packed filter ABI response is missing bitset_words_le")
+ bitset_words = payload["bitset_words_le"]
+ if not isinstance(bitset_words, bytes):
+ raise TypeError("Packed filter ABI must return bitset_words_le as bytes")
+ return cls(
+ eligible_count=int(payload.get("eligible_count", 0)),
+ bitset_words=bitset_words,
+ native_filter_token=int(payload.get("native_filter_token", 0)),
+ )
+
class StateResult:
__slots__ = ("update_timestamp", "element_count")
@@ -481,6 +495,66 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:
payload = self._backend._index_engine_evaluate_filter(self._handle, dsl)
return FilterResult.from_backend(payload)
+ def evaluate_filter_packed(self, dsl: str, max_cached_candidates: int = 0) -> FilterResult:
+ symbol_name = (
+ "_index_engine_evaluate_filter_cached_packed"
+ if max_cached_candidates > 0
+ else "_index_engine_evaluate_filter_packed"
+ )
+ evaluate_packed = getattr(self._backend, symbol_name, None)
+ if evaluate_packed is None:
+ return self.evaluate_filter(
+ dsl,
+ max_cached_candidates=max_cached_candidates,
+ )
+ if max_cached_candidates > 0:
+ payload = evaluate_packed(
+ self._handle,
+ dsl,
+ max_cached_candidates,
+ )
+ else:
+ payload = evaluate_packed(self._handle, dsl)
+ return FilterResult.from_packed_backend(payload)
+
+ def evaluate_filter_for_routing(self, dsl: str, native_threshold: int) -> FilterResult:
+ evaluate_for_routing = getattr(
+ self._backend,
+ "_index_engine_evaluate_filter_for_routing",
+ None,
+ )
+ if evaluate_for_routing is None:
+ return self.evaluate_filter(
+ dsl,
+ max_cached_candidates=native_threshold,
+ )
+ payload = evaluate_for_routing(
+ self._handle,
+ dsl,
+ native_threshold,
+ )
+ return FilterResult.from_backend(payload)
+
+ def evaluate_filter_for_routing_packed(
+ self, dsl: str, native_threshold: int
+ ) -> FilterResult:
+ evaluate_packed = getattr(
+ self._backend,
+ "_index_engine_evaluate_filter_for_routing_packed",
+ None,
+ )
+ if evaluate_packed is None:
+ return self.evaluate_filter_for_routing(
+ dsl,
+ native_threshold=native_threshold,
+ )
+ payload = evaluate_packed(
+ self._handle,
+ dsl,
+ native_threshold,
+ )
+ return FilterResult.from_packed_backend(payload)
+
def dump(self, path: str) -> int:
return int(self._backend._index_engine_dump(self._handle, path))
@@ -510,6 +584,28 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:
def seek_range(self, start_key: str, end_key: str) -> list[tuple[str, bytes]]:
return list(self._backend._store_seek_range(self._handle, start_key, end_key))
+ def seek_range_page(
+ self,
+ start_key: str,
+ end_key: str,
+ limit: int,
+ max_bytes: int,
+ start_exclusive: bool = False,
+ ) -> list[tuple[str, bytes]]:
+ seek_page = getattr(self._backend, "_store_seek_range_page", None)
+ if seek_page is None:
+ raise NotImplementedError("This native engine does not support paged store scans")
+ return list(
+ seek_page(
+ self._handle,
+ start_key,
+ end_key,
+ limit,
+ max_bytes,
+ start_exclusive,
+ )
+ )
+
class PersistStore(_StoreBase):
def __init__(self, path: str):
super().__init__(backend._new_persist_store(path))
diff --git a/openviking/storage/vectordb/index/cuvs_index.py b/openviking/storage/vectordb/index/cuvs_index.py
index c6c173078..303b1a15c 100644
--- a/openviking/storage/vectordb/index/cuvs_index.py
+++ b/openviking/storage/vectordb/index/cuvs_index.py
@@ -20,21 +20,42 @@ import json
import logging
import math
import re
+import struct
import threading
import time
import traceback
+from array import array
from collections import OrderedDict
from contextlib import nullcontext
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
+from dataclasses import dataclass, field
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ Iterator,
+ List,
+ Mapping,
+ Optional,
+ Sequence,
+ Tuple,
+ Union,
+ overload,
+)
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
logger = logging.getLogger(__name__)
+_FP32_BYTES = 4
+_U32_BYTES = 4
+_FP32_UPLOAD_BATCH_BYTES = 64 * 1024 * 1024
+
+NativeFilterWords = Union[Sequence[int], bytes]
+StoredNativeFilterWords = Union[Tuple[int, ...], bytes]
NativeFilterEvaluation = Union[
- Tuple[Sequence[int], int],
- Tuple[Sequence[int], int, int],
+ Tuple[NativeFilterWords, int],
+ Tuple[NativeFilterWords, int, int],
]
NativeFilterResolver = Callable[[Mapping[str, Any]], NativeFilterEvaluation]
@@ -55,6 +76,10 @@ class UnsupportedCuVSFilterError(ValueError):
"""Raised when a filter cannot be translated to a cuVS prefilter."""
+class _StalePreparedFilter(RuntimeError):
+ """Internal retry signal for a host filter invalidated before admission."""
+
+
@dataclass(frozen=True)
class CuVSMemoryEstimate:
"""Conservative GPU-memory estimate used only for auto-admission."""
@@ -77,6 +102,8 @@ class CuVSSearchTelemetry:
route_reason: str = "pending"
filter_kind: str = "none"
filter_cache_hit: bool = False
+ filter_cache_eviction_fallback: bool = False
+ filter_words_packed: bool = False
native_filter_reused: bool = False
build_performed: bool = False
eligible_count: Optional[int] = None
@@ -88,6 +115,7 @@ class CuVSSearchTelemetry:
total_ms: float = 0.0
preflight_ms: float = 0.0
queue_ms: float = 0.0
+ gpu_gate_queue_ms: float = 0.0
build_ms: float = 0.0
filter_prepare_ms: float = 0.0
gpu_search_ms: float = 0.0
@@ -102,6 +130,8 @@ class CuVSSearchTelemetry:
"route_reason": self.route_reason,
"filter_kind": self.filter_kind,
"filter_cache_hit": self.filter_cache_hit,
+ "filter_cache_eviction_fallback": self.filter_cache_eviction_fallback,
+ "filter_words_packed": self.filter_words_packed,
"native_filter_reused": self.native_filter_reused,
"build_performed": self.build_performed,
"eligible_count": self.eligible_count,
@@ -113,6 +143,7 @@ class CuVSSearchTelemetry:
"total_ms": round(self.total_ms, 3),
"preflight_ms": round(self.preflight_ms, 3),
"queue_ms": round(self.queue_ms, 3),
+ "gpu_gate_queue_ms": round(self.gpu_gate_queue_ms, 3),
"build_ms": round(self.build_ms, 3),
"filter_prepare_ms": round(self.filter_prepare_ms, 3),
"gpu_search_ms": round(self.gpu_search_ms, 3),
@@ -355,6 +386,60 @@ def matches_filter(
raise UnsupportedCuVSFilterError(f"Unsupported cuVS filter operation: {op!r}")
+@dataclass(frozen=True)
+class _PackedFP32Rows(Sequence[Sequence[float]]):
+ """Immutable FP32 rows captured for one cuVS rebuild.
+
+ The row blobs keep mutation snapshots cheap and safe: an upsert replaces a
+ blob instead of modifying storage which a background rebuild may still be
+ reading. Runtime uploads concatenate only a bounded number of rows at a
+ time, so a rebuild does not require another full host-side dataset copy.
+ """
+
+ rows: Tuple[bytes, ...]
+ dimension: int
+
+ def __len__(self) -> int:
+ return len(self.rows)
+
+ @overload
+ def __getitem__(self, index: int) -> Sequence[float]: ...
+
+ @overload
+ def __getitem__(self, index: slice) -> Sequence[Sequence[float]]: ...
+
+ def __getitem__(
+ self, index: Union[int, slice]
+ ) -> Union[Sequence[float], Sequence[Sequence[float]]]:
+ if isinstance(index, slice):
+ return tuple(memoryview(row).cast("f") for row in self.rows[index])
+ return memoryview(self.rows[index]).cast("f")
+
+ def __iter__(self) -> Iterator[Sequence[float]]:
+ for row in self.rows:
+ yield memoryview(row).cast("f")
+
+ @property
+ def nbytes(self) -> int:
+ return len(self.rows) * self.dimension * _FP32_BYTES
+
+ def iter_packed_batches(
+ self,
+ max_bytes: int,
+ ) -> Iterator[Tuple[int, int, bytes]]:
+ """Yield row-aligned host buffers bounded by ``max_bytes`` when possible."""
+
+ if max_bytes <= 0:
+ raise ValueError("cuVS FP32 upload batch size must be positive")
+ row_bytes = self.dimension * _FP32_BYTES
+ if row_bytes <= 0:
+ return
+ rows_per_batch = max(1, max_bytes // row_bytes)
+ for start in range(0, len(self.rows), rows_per_batch):
+ end = min(start + rows_per_batch, len(self.rows))
+ yield start, end, b"".join(self.rows[start:end])
+
+
class _CuVSRuntime:
"""Small adapter around the public cuVS Python API."""
@@ -401,13 +486,11 @@ class _CuVSRuntime:
self._owned_resources: List[Any] = []
self._resource_limit = 1
self._resources_closed = False
- self._use_explicit_resources = False
def set_max_concurrent_searches(self, value: int) -> None:
limit = max(1, int(value))
with self._resource_condition:
self._resource_limit = limit
- self._use_explicit_resources = limit > 1
self._resource_condition.notify_all()
def device_scope(self):
@@ -442,9 +525,36 @@ class _CuVSRuntime:
def build(self, dataset: Sequence[Sequence[float]]):
with self.device_scope():
device_dataset = None
+ packed_batch = None
+ host_batch = None
index = None
try:
- device_dataset = self.cp.asarray(dataset, dtype=self.device_dtype)
+ if isinstance(dataset, _PackedFP32Rows):
+ # CuPy already depends on NumPy, but keep the import on the
+ # GPU-only path so native OpenViking users do not gain a
+ # new import-time dependency through this module.
+ import numpy as np
+
+ device_dataset = self.cp.empty(
+ (len(dataset), dataset.dimension),
+ dtype=self.device_dtype,
+ )
+ for start, end, packed_batch in dataset.iter_packed_batches(
+ _FP32_UPLOAD_BATCH_BYTES
+ ):
+ host_batch = np.frombuffer(packed_batch, dtype=np.float32).reshape(
+ end - start, dataset.dimension
+ )
+ if self.dtype == "float16":
+ host_batch = host_batch.astype(np.float16)
+ # ndarray.set() without an explicit stream performs a
+ # synchronous host-to-device copy. The bounded host
+ # buffer can therefore be released before the next batch.
+ device_dataset[start:end].set(host_batch)
+ host_batch = None
+ packed_batch = None
+ else:
+ device_dataset = self.cp.asarray(dataset, dtype=self.device_dtype)
if self.algorithm == "brute_force":
index = self.brute_force.build(device_dataset, metric=self.metric)
return _CuVSRuntimeIndex(index=index, dataset=device_dataset)
@@ -456,6 +566,8 @@ class _CuVSRuntime:
# Drop them before Device.__exit__ restores the caller's device.
index = None
device_dataset = None
+ host_batch = None
+ packed_batch = None
# Python/Cython exception frames can retain CUDA arguments even
# after the locals above are cleared. Preserve stack locations
# while releasing frame locals under the captured device.
@@ -504,10 +616,33 @@ class _CuVSRuntime:
words[index // 32] |= 1 << (index % 32)
return self.cp.asarray(words, dtype=self.cp.uint32)
- def prepare_filter_words(self, words: Sequence[int]):
+ def prepare_filter_words(self, words: NativeFilterWords):
"""Copy an already packed native filter bitmap to the device."""
+ if isinstance(words, bytes) and len(words) % _U32_BYTES != 0:
+ raise ValueError("Packed native filter bitmap length must be a multiple of 4 bytes")
with self.device_scope():
+ if isinstance(words, bytes):
+ # Keep NumPy on the GPU-only path. The explicit little-endian
+ # dtype matches the additive native ABI without constructing a
+ # Python int for every bitmap word.
+ import numpy as np
+
+ host_words = np.frombuffer(words, dtype="<u4")
+ device_words = None
+ try:
+ device_words = self.cp.empty(host_words.shape, dtype=self.cp.uint32)
+ # ndarray.set() without an explicit stream completes the host
+ # copy before the immutable bytes owner can leave the cache.
+ device_words.set(host_words)
+ return device_words
+ except BaseException as exc:
+ # Drop the last device reference before restoring the
+ # caller's CUDA device; exception frames can otherwise
+ # retain it beyond Device.__exit__.
+ device_words = None
+ traceback.clear_frames(exc.__traceback__)
+ raise
return self.cp.asarray(words, dtype=self.cp.uint32)
def search(
@@ -527,8 +662,14 @@ class _CuVSRuntime:
resource_reusable = False
try:
index = runtime_index.index
- resources = self._acquire_resources() if self._use_explicit_resources else None
- resource_kwargs = {"resources": resources} if resources is not None else {}
+ # Always provide an explicit resource, including the serialized
+ # max_concurrent_gpu_searches=1 case. cuVS synchronizes an
+ # implicit resource before returning, but an owned reusable
+ # resource also pins worker-thread churn to a known stream and
+ # gives filter/result lifetimes an explicit synchronization
+ # boundary inside this admitted search.
+ resources = self._acquire_resources()
+ resource_kwargs = {"resources": resources}
queries = self.cp.asarray([query], dtype=self.device_dtype)
if mask is None:
prefilter = None
@@ -558,14 +699,18 @@ class _CuVSRuntime:
filter=prefilter,
**resource_kwargs,
)
- if resources is not None:
- resources.sync()
- resource_reusable = True
+ resources.sync()
host_neighbors = self.cp.asnumpy(neighbors)[0].tolist()
host_distances = self.cp.asnumpy(distances)[0].tolist()
- return [int(item) for item in host_neighbors], [
- float(item) for item in host_distances
- ]
+ result = (
+ [int(item) for item in host_neighbors],
+ [float(item) for item in host_distances],
+ )
+ # Reuse only after the complete call, including host result
+ # materialization, succeeds. Any exception discards a
+ # potentially poisoned resource from the pool.
+ resource_reusable = True
+ return result
except Exception as exc:
traceback.clear_frames(exc.__traceback__)
raise
@@ -597,7 +742,7 @@ class _CuVSRuntime:
@dataclass(frozen=True)
class _Record:
- vector: Tuple[float, ...]
+ vector: bytes
fields: Mapping[str, Any]
@@ -629,15 +774,57 @@ class _CachedFilter:
route_native: bool = False
native_threshold: int = 0
native_filter_token: int = 0
+ filter_words_packed: bool = False
@dataclass(frozen=True)
class _ResolvedNativeFilter:
- bitset_words: Tuple[int, ...]
+ bitset_words: StoredNativeFilterWords
eligible_count: int
route_native: bool
native_threshold: int
native_filter_token: int = 0
+ filter_words_packed: bool = False
+
+
+@dataclass(frozen=True)
+class _CachedFilterMetadata:
+ """Host-only copy of route metadata from a potentially device-backed cache entry."""
+
+ eligible_count: int
+ route_native: bool
+ native_threshold: int
+ native_filter_token: int
+ filter_words_packed: bool
+
+
+@dataclass(frozen=True)
+class _PreparedHostFilter:
+ """Generation-bound host work completed before GPU admission.
+
+ This context deliberately never owns a device allocation. A cache hit is
+ represented only by copied route metadata; the device entry is borrowed
+ again after admission. Otherwise ``resolved_native_filter`` contains host
+ words whose conversion to a device bitset remains inside the GPU gate.
+ """
+
+ generation: int
+ cache_key: Optional[str]
+ cached_metadata: Optional[_CachedFilterMetadata] = None
+ resolved_native_filter: Optional[_ResolvedNativeFilter] = None
+
+
+@dataclass
+class _NativeFilterPreflightFlight:
+ generation: int
+ done: threading.Event = field(default_factory=threading.Event)
+ route_count: Optional[int] = None
+ eligible_count: int = 0
+ cached_metadata: Optional[_CachedFilterMetadata] = None
+ resolved_native_filter: Optional[_ResolvedNativeFilter] = None
+ error: Optional[BaseException] = None
+ stale: bool = False
+ participants: int = 1
class CuVSDenseIndex:
@@ -724,6 +911,7 @@ class CuVSDenseIndex:
self._dirty = True
self._filter_cache: OrderedDict[str, _CachedFilter] = OrderedDict()
self._preflight_filter_cache: OrderedDict[str, _ResolvedNativeFilter] = OrderedDict()
+ self._preflight_flights: Dict[Tuple[int, str], _NativeFilterPreflightFlight] = {}
self._lock = threading.RLock()
self._idle_condition = threading.Condition(self._lock)
self._active_searches = 0
@@ -731,6 +919,8 @@ class CuVSDenseIndex:
self._filter_layout_lock = threading.Lock()
self._records_generation = 0
self._filter_layout_generation = -1
+ self._closed = False
+ self._close_complete = False
logger.info(
"Initialized cuVS dense index: algorithm=%s metric=%s dimension=%d",
self.algorithm,
@@ -747,18 +937,33 @@ class CuVSDenseIndex:
with self._lock:
return len(self._records)
+ @property
+ def host_shadow_nbytes(self) -> int:
+ """Return the compact FP32 vector payload size, excluding Python metadata."""
+
+ with self._lock:
+ return len(self._records) * self.dimension * _FP32_BYTES
+
@property
def needs_rebuild(self) -> bool:
with self._lock:
return self._dirty
- def _prepare_vector(self, vector: Sequence[float]) -> Tuple[float, ...]:
+ def _prepare_vector_values(self, vector: Sequence[float]) -> List[float]:
if len(vector) != self.dimension:
raise ValueError(
f"cuVS vector dimension mismatch: expected {self.dimension}, got {len(vector)}"
)
- prepared = _normalize(vector) if self.normalize_vectors else [float(v) for v in vector]
- return tuple(prepared)
+ return _normalize(vector) if self.normalize_vectors else [float(v) for v in vector]
+
+ def _prepare_vector(self, vector: Sequence[float]) -> Tuple[float, ...]:
+ return tuple(self._prepare_vector_values(vector))
+
+ def _pack_vector(self, vector: Sequence[float]) -> bytes:
+ packed = array("f", self._prepare_vector_values(vector))
+ if packed.itemsize != _FP32_BYTES:
+ raise RuntimeError("cuVS host vector storage requires 4-byte IEEE FP32 values")
+ return packed.tobytes()
@staticmethod
def _parse_fields(value: str) -> Mapping[str, Any]:
@@ -776,7 +981,7 @@ class CuVSDenseIndex:
if not candidate.vector:
continue
self._records[int(candidate.label)] = _Record(
- vector=self._prepare_vector(candidate.vector),
+ vector=self._pack_vector(candidate.vector),
fields=self._parse_fields(candidate.fields),
)
self._invalidate()
@@ -788,7 +993,7 @@ class CuVSDenseIndex:
if not record.vector:
continue
self._records[int(record.label)] = _Record(
- vector=self._prepare_vector(record.vector),
+ vector=self._pack_vector(record.vector),
fields=self._parse_fields(record.fields),
)
changed = True
@@ -807,10 +1012,20 @@ class CuVSDenseIndex:
def _invalidate(self) -> None:
self._records_generation += 1
self._dirty = True
+ self._cancel_preflight_flights_()
with self._runtime_device_scope():
self._filter_cache.clear()
self._preflight_filter_cache.clear()
+ def _cancel_preflight_flights_(self) -> None:
+ """Mark in-flight projections stale and wake waiters under ``_lock``."""
+
+ flights = tuple(self._preflight_flights.values())
+ self._preflight_flights.clear()
+ for flight in flights:
+ flight.stale = True
+ flight.done.set()
+
@staticmethod
def _filter_cache_key(filters: Mapping[str, Any]) -> Optional[str]:
try:
@@ -826,6 +1041,16 @@ class CuVSDenseIndex:
self._filter_cache[cache_key] = cached
return cached
+ @staticmethod
+ def _cached_filter_metadata(cached: _CachedFilter) -> _CachedFilterMetadata:
+ return _CachedFilterMetadata(
+ eligible_count=cached.eligible_count,
+ route_native=cached.route_native,
+ native_threshold=cached.native_threshold,
+ native_filter_token=cached.native_filter_token,
+ filter_words_packed=cached.filter_words_packed,
+ )
+
def _cache_filter(self, cache_key: Optional[str], cached: _CachedFilter) -> None:
if cache_key is None or self.filter_cache_size <= 0:
return
@@ -855,6 +1080,51 @@ class CuVSDenseIndex:
while len(self._preflight_filter_cache) > capacity:
self._preflight_filter_cache.popitem(last=False)
+ def _lookup_preflight_route_(
+ self,
+ cache_key: Optional[str],
+ ) -> Optional[Tuple[Optional[int], int]]:
+ cached = self._get_cached_filter(cache_key)
+ if cached is not None:
+ return (
+ cached.eligible_count if cached.route_native else None,
+ cached.eligible_count,
+ )
+ resolved = self._get_preflight_filter(cache_key)
+ if resolved is None:
+ return None
+ return (
+ resolved.eligible_count if resolved.route_native else None,
+ resolved.eligible_count,
+ )
+
+ def _store_preflight_route_(
+ self,
+ cache_key: Optional[str],
+ resolved: _ResolvedNativeFilter,
+ ) -> Tuple[Optional[int], int, bool]:
+ existing = self._lookup_preflight_route_(cache_key)
+ if existing is not None:
+ route_count, eligible_count = existing
+ return route_count, eligible_count, True
+ if not resolved.route_native and resolved.eligible_count != 0:
+ self._cache_preflight_filter(cache_key, resolved)
+ return None, resolved.eligible_count, False
+ cached = _CachedFilter(
+ prepared=None,
+ eligible_count=resolved.eligible_count,
+ route_native=resolved.route_native,
+ native_threshold=resolved.native_threshold,
+ native_filter_token=resolved.native_filter_token,
+ filter_words_packed=resolved.filter_words_packed,
+ )
+ self._cache_filter(cache_key, cached)
+ return (
+ cached.eligible_count if cached.route_native else None,
+ cached.eligible_count,
+ False,
+ )
+
def native_filter_threshold(self, filters: Mapping[str, Any]) -> int:
return (
self.auto_path_filter_native_threshold
@@ -877,6 +1147,8 @@ class CuVSDenseIndex:
with self._filter_layout_lock:
with self._lock:
generation = self._records_generation
+ if self._closed:
+ return None
if self._filter_layout_generation == generation:
return generation
ordered_labels = list(self._records)
@@ -884,7 +1156,7 @@ class CuVSDenseIndex:
native_filter_layout_registrar(ordered_labels)
with self._lock:
- if self._records_generation != generation:
+ if self._closed or self._records_generation != generation:
return None
self._filter_layout_generation = generation
return generation
@@ -902,8 +1174,9 @@ class CuVSDenseIndex:
try:
if not self.auto_memory or not filters:
return None
- cache_key = self._filter_cache_key(filters)
with self._lock:
+ if self._closed:
+ return None
if telemetry is not None:
telemetry.filter_kind = (
"path"
@@ -912,77 +1185,253 @@ class CuVSDenseIndex:
)
telemetry.records_generation = self._records_generation
telemetry.index_size = len(self._records)
- cached = self._get_cached_filter(cache_key)
- if cached is not None:
- if telemetry is not None:
- telemetry.filter_cache_hit = True
- telemetry.eligible_count = cached.eligible_count
- return cached.eligible_count if cached.route_native else None
native_threshold = self.native_filter_threshold(filters)
if native_threshold <= 0:
return None
- preflight_cached = self._get_preflight_filter(cache_key)
- if preflight_cached is not None:
- if telemetry is not None:
- telemetry.filter_cache_hit = True
- telemetry.eligible_count = preflight_cached.eligible_count
- return (
- preflight_cached.eligible_count if preflight_cached.route_native else None
- )
+ prepared = self._prepare_host_filter(
+ filters,
+ native_filter_resolver,
+ native_filter_layout_registrar,
+ telemetry,
+ )
+ if prepared is None:
+ return None
+ cached = prepared.cached_metadata
+ resolved = prepared.resolved_native_filter
+ if cached is None and resolved is None:
+ raise RuntimeError("Native filter preflight completed without a result")
+ eligible_count = (
+ cached.eligible_count if cached is not None else resolved.eligible_count
+ )
+ route_native = cached.route_native if cached is not None else resolved.route_native
+ return eligible_count if route_native else None
+ finally:
+ if telemetry is not None:
+ telemetry.preflight_ms += (time.perf_counter() - started) * 1000.0
+
+ def _prepare_host_filter(
+ self,
+ filters: Mapping[str, Any],
+ native_filter_resolver: NativeFilterResolver,
+ native_filter_layout_registrar: Callable[[Sequence[int]], None],
+ telemetry: Optional[CuVSSearchTelemetry] = None,
+ ) -> Optional[_PreparedHostFilter]:
+ """Resolve and cache a generation-bound native bitmap without GPU admission."""
- generation = self._ensure_native_filter_layout(native_filter_layout_registrar)
- if generation is None:
+ cache_key = self._filter_cache_key(filters)
+ with self._lock:
+ if self._closed:
return None
- resolved = self._resolve_native_filter(filters, native_filter_resolver)
+ generation = self._records_generation
if telemetry is not None:
- telemetry.eligible_count = resolved.eligible_count
+ telemetry.filter_kind = (
+ "path"
+ if _filter_uses_field_type(filters, self.field_types, "path")
+ else "scalar"
+ )
+ telemetry.records_generation = generation
+ telemetry.index_size = len(self._records)
+ cached = self._get_cached_filter(cache_key)
+ if cached is not None:
+ if telemetry is not None:
+ telemetry.filter_cache_hit = True
+ telemetry.eligible_count = cached.eligible_count
+ telemetry.filter_words_packed = cached.filter_words_packed
+ return _PreparedHostFilter(
+ generation,
+ cache_key,
+ cached_metadata=self._cached_filter_metadata(cached),
+ )
+ resolved = self._get_preflight_filter(cache_key)
+ if resolved is not None:
+ if telemetry is not None:
+ telemetry.filter_cache_hit = True
+ telemetry.eligible_count = resolved.eligible_count
+ telemetry.filter_words_packed = resolved.filter_words_packed
+ return _PreparedHostFilter(
+ generation,
+ cache_key,
+ resolved_native_filter=resolved,
+ )
+
+ # Keep the established _filter_layout_lock -> _lock order. The native
+ # resolver can then run unlocked against this registered row layout.
+ generation = self._ensure_native_filter_layout(native_filter_layout_registrar)
+ if generation is None:
+ return None
+ flight_key: Optional[Tuple[int, str]] = None
+ flight: Optional[_NativeFilterPreflightFlight] = None
+ is_owner = True
+ try:
with self._lock:
- if self._records_generation != generation:
+ if self._closed or self._records_generation != generation:
return None
cached = self._get_cached_filter(cache_key)
if cached is not None:
if telemetry is not None:
telemetry.filter_cache_hit = True
telemetry.eligible_count = cached.eligible_count
- return cached.eligible_count if cached.route_native else None
- if not resolved.route_native and resolved.eligible_count != 0:
- self._cache_preflight_filter(cache_key, resolved)
+ telemetry.filter_words_packed = cached.filter_words_packed
+ return _PreparedHostFilter(
+ generation,
+ cache_key,
+ cached_metadata=self._cached_filter_metadata(cached),
+ )
+ resolved = self._get_preflight_filter(cache_key)
+ if resolved is not None:
+ if telemetry is not None:
+ telemetry.filter_cache_hit = True
+ telemetry.eligible_count = resolved.eligible_count
+ telemetry.filter_words_packed = resolved.filter_words_packed
+ return _PreparedHostFilter(
+ generation,
+ cache_key,
+ resolved_native_filter=resolved,
+ )
+ if cache_key is not None:
+ flight_key = (generation, cache_key)
+ flight = self._preflight_flights.get(flight_key)
+ if flight is None:
+ flight = _NativeFilterPreflightFlight(generation=generation)
+ self._preflight_flights[flight_key] = flight
+ else:
+ flight.participants += 1
+ is_owner = False
+
+ if not is_owner:
+ # Mutation and close wake waiters through _cancel_preflight_flights_.
+ # Never wait while holding _lock.
+ flight.done.wait()
+ with self._lock:
+ stale = flight.stale or self._closed or self._records_generation != generation
+ error = flight.error
+ cached = flight.cached_metadata
+ resolved = flight.resolved_native_filter
+ if stale:
return None
- cached = _CachedFilter(
- prepared=None,
- eligible_count=resolved.eligible_count,
- route_native=resolved.route_native,
- native_threshold=resolved.native_threshold,
- native_filter_token=resolved.native_filter_token,
+ if error is not None:
+ raise error
+ if telemetry is not None:
+ telemetry.filter_cache_hit = True
+ telemetry.eligible_count = flight.eligible_count
+ telemetry.filter_words_packed = bool(
+ cached.filter_words_packed
+ if cached is not None
+ else resolved is not None and resolved.filter_words_packed
+ )
+ return _PreparedHostFilter(
+ generation,
+ cache_key,
+ cached_metadata=cached,
+ resolved_native_filter=resolved,
)
- self._cache_filter(cache_key, cached)
- return cached.eligible_count if cached.route_native else None
- finally:
- if telemetry is not None:
- telemetry.preflight_ms += (time.perf_counter() - started) * 1000.0
+
+ resolved = self._resolve_native_filter(filters, native_filter_resolver)
+ with self._lock:
+ stale = (
+ self._closed
+ or self._records_generation != generation
+ or (flight is not None and flight.stale)
+ )
+ if flight is not None and self._preflight_flights.get(flight_key) is flight:
+ self._preflight_flights.pop(flight_key, None)
+ if stale:
+ if flight is not None:
+ flight.stale = True
+ flight.done.set()
+ return None
+ route_count, eligible_count, cache_hit = self._store_preflight_route_(
+ cache_key,
+ resolved,
+ )
+ cached = self._get_cached_filter(cache_key)
+ cached_metadata = (
+ self._cached_filter_metadata(cached) if cached is not None else None
+ )
+ if flight is not None:
+ flight.route_count = route_count
+ flight.eligible_count = eligible_count
+ flight.cached_metadata = cached_metadata
+ flight.resolved_native_filter = None if cached is not None else resolved
+ flight.done.set()
+ except BaseException as exc:
+ if is_owner and flight is not None:
+ with self._lock:
+ if self._preflight_flights.get(flight_key) is flight:
+ self._preflight_flights.pop(flight_key, None)
+ if not flight.stale:
+ flight.error = exc
+ flight.done.set()
+ raise
+
+ if telemetry is not None:
+ telemetry.filter_cache_hit = cache_hit
+ telemetry.eligible_count = eligible_count
+ telemetry.filter_words_packed = (
+ cached_metadata.filter_words_packed
+ if cached_metadata is not None
+ else resolved.filter_words_packed
+ )
+ return _PreparedHostFilter(
+ generation,
+ cache_key,
+ cached_metadata=cached_metadata,
+ resolved_native_filter=None if cached is not None else resolved,
+ )
def _resolve_native_filter(
self,
filters: Mapping[str, Any],
native_filter_resolver: NativeFilterResolver,
) -> _ResolvedNativeFilter:
+ with self._lock:
+ projection_generation = self._records_generation
+ projection_layout_generation = self._filter_layout_generation
+ projection_row_count = len(self._records)
evaluation = native_filter_resolver(filters)
if len(evaluation) == 2:
words, eligible_count = evaluation
native_filter_token = 0
else:
words, eligible_count, native_filter_token = evaluation
+ if isinstance(words, bytes):
+ if len(words) % _U32_BYTES != 0:
+ raise ValueError("Packed native filter bitmap length must be a multiple of 4 bytes")
+ bitset_words: StoredNativeFilterWords = words
+ bitset_word_count = len(words) // _U32_BYTES
+ else:
+ bitset_words = tuple(int(word) for word in words)
+ bitset_word_count = len(bitset_words)
+ eligible_count = int(eligible_count)
native_threshold = self.native_filter_threshold(filters)
route_native = (
self.auto_memory and native_threshold > 0 and eligible_count <= native_threshold
)
+ if not route_native and eligible_count > 0:
+ # Resolver work intentionally runs outside the records lock. Only
+ # enforce the ABI against the same registered layout snapshot;
+ # callers discard the result when a concurrent mutation wins.
+ with self._lock:
+ projection_is_stable = (
+ projection_layout_generation == projection_generation
+ and self._records_generation == projection_generation
+ and self._filter_layout_generation == projection_generation
+ )
+ required_words = (projection_row_count + 31) // 32
+ if projection_is_stable and bitset_word_count < required_words:
+ raise RuntimeError(
+ "Native filter resolver returned an incomplete bitset for GPU routing: "
+ f"got {bitset_word_count} words for {projection_row_count} rows "
+ f"(expected at least {required_words})"
+ )
return _ResolvedNativeFilter(
- bitset_words=tuple(int(word) for word in words),
- eligible_count=int(eligible_count),
+ bitset_words=bitset_words,
+ eligible_count=eligible_count,
route_native=route_native,
native_threshold=native_threshold,
native_filter_token=int(native_filter_token),
+ filter_words_packed=isinstance(bitset_words, bytes),
)
def _prepare_filter(
@@ -997,6 +1446,7 @@ class CuVSDenseIndex:
if cached is not None:
return cached
+ resolved: Optional[_ResolvedNativeFilter] = None
if native_filter_resolver is not None:
resolved = resolved_native_filter or self._resolve_native_filter(
filters, native_filter_resolver
@@ -1012,7 +1462,18 @@ class CuVSDenseIndex:
prepared = prepare_filter_words(resolved.bitset_words)
else:
prepared = tuple(
- bool(resolved.bitset_words[row // 32] & (1 << (row % 32)))
+ bool(
+ (
+ struct.unpack_from(
+ "<I",
+ resolved.bitset_words,
+ (row // 32) * _U32_BYTES,
+ )[0]
+ if isinstance(resolved.bitset_words, bytes)
+ else resolved.bitset_words[row // 32]
+ )
+ & (1 << (row % 32))
+ )
for row in range(len(labels))
)
else:
@@ -1035,11 +1496,8 @@ class CuVSDenseIndex:
eligible_count=eligible_count,
route_native=route_native,
native_threshold=native_threshold,
- native_filter_token=(
- resolved_native_filter.native_filter_token
- if resolved_native_filter is not None
- else 0
- ),
+ native_filter_token=(resolved.native_filter_token if resolved is not None else 0),
+ filter_words_packed=(resolved.filter_words_packed if resolved is not None else False),
)
self._cache_filter(cache_key, cached)
return cached
@@ -1119,7 +1577,10 @@ class CuVSDenseIndex:
generation=generation,
)
labels = tuple(self._records)
- dataset = [self._records[label].vector for label in labels]
+ dataset = _PackedFP32Rows(
+ tuple(self._records[label].vector for label in labels),
+ self.dimension,
+ )
with _CUVS_MEMORY_COORDINATOR.build_lock(self._runtime):
# Admission is checked again while the per-device build lock is
@@ -1228,35 +1689,95 @@ class CuVSDenseIndex:
) -> Tuple[List[int], List[float]]:
if limit <= 0:
return [], []
- queue_started = time.perf_counter()
- self._gpu_search_gate.acquire()
- if telemetry is not None:
- telemetry.queue_ms += (time.perf_counter() - queue_started) * 1000.0
- try:
- return self._search_admitted(
- query_vector,
- limit,
- filters,
- native_filter_resolver,
- native_filter_layout_registrar,
- telemetry,
- )
- finally:
- self._gpu_search_gate.release()
+ query = self._prepare_vector(query_vector)
+ while True:
+ prepared_filter: Optional[_PreparedHostFilter] = None
+ if (
+ filters
+ and native_filter_resolver is not None
+ and native_filter_layout_registrar is not None
+ ):
+ filter_started = time.perf_counter()
+ try:
+ prepared_filter = self._prepare_host_filter(
+ filters,
+ native_filter_resolver,
+ native_filter_layout_registrar,
+ telemetry,
+ )
+ finally:
+ if telemetry is not None:
+ telemetry.filter_prepare_ms += (
+ time.perf_counter() - filter_started
+ ) * 1000.0
+ if prepared_filter is None:
+ with self._lock:
+ if self._closed:
+ raise RuntimeError("cuVS dense index is closed")
+ # A mutation invalidated the registered native layout while
+ # the resolver ran. Retry before consuming GPU admission.
+ continue
+ cached = prepared_filter.cached_metadata
+ resolved = prepared_filter.resolved_native_filter
+ if cached is None and resolved is None:
+ raise RuntimeError("Native filter preparation completed without a result")
+ eligible_count = (
+ cached.eligible_count if cached is not None else resolved.eligible_count
+ )
+ route_native = cached.route_native if cached is not None else resolved.route_native
+ native_threshold = (
+ cached.native_threshold if cached is not None else resolved.native_threshold
+ )
+ if eligible_count == 0:
+ return [], []
+ if route_native:
+ raise CuVSNativeRouteError(
+ "cuVS auto mode routed a selective filter to native search "
+ f"({eligible_count} candidates <= {native_threshold})"
+ )
+
+ queue_started = time.perf_counter()
+ self._gpu_search_gate.acquire()
+ waited_ms = (time.perf_counter() - queue_started) * 1000.0
+ if telemetry is not None:
+ # queue_ms remains the aggregate of all lock/admission waits;
+ # gpu_gate_queue_ms isolates only this device-search permit.
+ telemetry.queue_ms += waited_ms
+ telemetry.gpu_gate_queue_ms += waited_ms
+ try:
+ try:
+ return self._search_admitted(
+ query,
+ limit,
+ filters,
+ native_filter_resolver,
+ native_filter_layout_registrar,
+ prepared_filter,
+ telemetry,
+ )
+ except _StalePreparedFilter:
+ continue
+ finally:
+ self._gpu_search_gate.release()
def _search_admitted(
self,
- query_vector: Sequence[float],
+ query: Sequence[float],
limit: int,
filters: Optional[Mapping[str, Any]],
native_filter_resolver: Optional[NativeFilterResolver] = None,
native_filter_layout_registrar: Optional[Callable[[Sequence[int]], None]] = None,
+ prepared_filter: Optional[_PreparedHostFilter] = None,
telemetry: Optional[CuVSSearchTelemetry] = None,
) -> Tuple[List[int], List[float]]:
- query = self._prepare_vector(query_vector)
- if self.auto_memory and native_filter_layout_registrar is not None:
- self._ensure_native_filter_layout(native_filter_layout_registrar)
with self._lock:
+ if self._closed:
+ raise RuntimeError("cuVS dense index is closed")
+ if (
+ prepared_filter is not None
+ and prepared_filter.generation != self._records_generation
+ ):
+ raise _StalePreparedFilter
if telemetry is not None:
telemetry.filter_kind = (
"path"
@@ -1268,24 +1789,45 @@ class CuVSDenseIndex:
telemetry.records_generation = self._records_generation
telemetry.index_size = len(self._records)
cached_filter: Optional[_CachedFilter] = None
- resolved_native_filter: Optional[_ResolvedNativeFilter] = None
+ resolved_native_filter = (
+ prepared_filter.resolved_native_filter if prepared_filter is not None else None
+ )
+ if prepared_filter is not None and prepared_filter.cached_metadata is not None:
+ cached_filter = self._get_cached_filter(prepared_filter.cache_key)
+ if cached_filter is None:
+ # The device LRU entry was evicted after host preparation.
+ # This is the rare guaranteed-progress fallback: leave the
+ # cached/resolved values empty so _prepare_filter resolves
+ # and materializes this filter once inside the gate. The
+ # common cache-miss path still resolves before admission.
+ resolved_native_filter = None
+ if telemetry is not None:
+ telemetry.filter_cache_hit = False
+ telemetry.filter_cache_eviction_fallback = True
filter_layout_is_current = self._filter_layout_generation == self._records_generation
# Auto mode decides whether a selective filter should remain native
# before paying GPU admission or rebuild costs. A dirty native
# layout is refreshed against the pending cuVS row order only; the
# live GPU row mapping is not changed until a build actually runs.
- if filters and self.auto_memory and native_filter_resolver is not None:
+ if (
+ filters
+ and self.auto_memory
+ and native_filter_resolver is not None
+ and prepared_filter is None
+ ):
cache_key = self._filter_cache_key(filters)
cached_filter = self._get_cached_filter(cache_key)
if cached_filter is not None and telemetry is not None:
telemetry.filter_cache_hit = True
telemetry.eligible_count = cached_filter.eligible_count
+ telemetry.filter_words_packed = cached_filter.filter_words_packed
if cached_filter is None:
resolved_native_filter = self._get_preflight_filter(cache_key)
if resolved_native_filter is not None and telemetry is not None:
telemetry.filter_cache_hit = True
telemetry.eligible_count = resolved_native_filter.eligible_count
+ telemetry.filter_words_packed = resolved_native_filter.filter_words_packed
if resolved_native_filter is None:
if (
self._dirty
@@ -1298,6 +1840,10 @@ class CuVSDenseIndex:
resolved_native_filter = self._resolve_native_filter(
filters, native_filter_resolver
)
+ if telemetry is not None:
+ telemetry.filter_words_packed = (
+ resolved_native_filter.filter_words_packed
+ )
if (
resolved_native_filter.route_native
or resolved_native_filter.eligible_count == 0
@@ -1308,6 +1854,7 @@ class CuVSDenseIndex:
route_native=resolved_native_filter.route_native,
native_threshold=resolved_native_filter.native_threshold,
native_filter_token=resolved_native_filter.native_filter_token,
+ filter_words_packed=resolved_native_filter.filter_words_packed,
)
self._cache_filter(cache_key, cached_filter)
@@ -1336,6 +1883,7 @@ class CuVSDenseIndex:
cached_filter = self._get_cached_filter(self._filter_cache_key(filters))
if cached_filter is not None and telemetry is not None:
telemetry.filter_cache_hit = True
+ telemetry.filter_words_packed = cached_filter.filter_words_packed
cached_filter = cached_filter or self._prepare_filter(
filters,
snapshot.labels,
@@ -1345,6 +1893,7 @@ class CuVSDenseIndex:
if telemetry is not None:
telemetry.filter_prepare_ms += (time.perf_counter() - filter_started) * 1000.0
telemetry.eligible_count = cached_filter.eligible_count
+ telemetry.filter_words_packed = cached_filter.filter_words_packed
mask = cached_filter.prepared
eligible_count = cached_filter.eligible_count
if eligible_count == 0:
@@ -1399,10 +1948,23 @@ class CuVSDenseIndex:
def close(self) -> None:
with self._lock:
+ if self._close_complete:
+ return
+ if not self._closed:
+ self._closed = True
+ self._records_generation += 1
+ self._filter_layout_generation = -1
+ self._cancel_preflight_flights_()
while self._active_searches:
self._idle_condition.wait()
with self._runtime_device_scope():
self._snapshot = None
self._filter_cache.clear()
self._preflight_filter_cache.clear()
- self._runtime.close()
+ try:
+ self._runtime.close()
+ finally:
+ # A failed recovery can otherwise keep this partially packed
+ # host shadow alive through the constructor traceback.
+ self._records.clear()
+ self._close_complete = True
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.py
index 9f7af96cd..f33949099 100644
--- a/openviking/storage/vectordb/index/local_index.py
+++ b/openviking/storage/vectordb/index/local_index.py
@@ -9,7 +9,7 @@ import threading
import time
from contextlib import contextmanager
from pathlib import Path
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union
import openviking.storage.vectordb.engine as engine
from openviking.storage.vectordb.index.cuvs_index import (
@@ -279,6 +279,51 @@ class IndexEngineProxy:
)
return result.bitset_words, result.eligible_count, result.native_filter_token
+ def evaluate_filter_for_routing(
+ self,
+ filters: Dict[str, Any],
+ native_threshold: int,
+ ) -> Tuple[List[int], int, int]:
+ """Evaluate only the projection needed for an adaptive route decision."""
+
+ if not self.index_engine:
+ raise RuntimeError("Index engine not initialized")
+ result = self.index_engine.evaluate_filter_for_routing(
+ json.dumps(filters),
+ native_threshold=native_threshold,
+ )
+ return result.bitset_words, result.eligible_count, result.native_filter_token
+
+ def evaluate_filter_packed(
+ self,
+ filters: Dict[str, Any],
+ max_cached_candidates: int = 0,
+ ) -> Tuple[Union[List[int], bytes], int, int]:
+ """Evaluate a cuVS filter using packed words when the engine supports it."""
+
+ if not self.index_engine:
+ raise RuntimeError("Index engine not initialized")
+ result = self.index_engine.evaluate_filter_packed(
+ json.dumps(filters),
+ max_cached_candidates=max_cached_candidates,
+ )
+ return result.bitset_words, result.eligible_count, result.native_filter_token
+
+ def evaluate_filter_for_routing_packed(
+ self,
+ filters: Dict[str, Any],
+ native_threshold: int,
+ ) -> Tuple[Union[List[int], bytes], int, int]:
+ """Evaluate a cuVS route using packed words when available."""
+
+ if not self.index_engine:
+ raise RuntimeError("Index engine not initialized")
+ result = self.index_engine.evaluate_filter_for_routing_packed(
+ json.dumps(filters),
+ native_threshold=native_threshold,
+ )
+ return result.bitset_words, result.eligible_count, result.native_filter_token
+
def drop(self):
"""Release the index engine resources.
@@ -310,7 +355,7 @@ class LocalIndex(IIndex):
index_path_or_json: str,
meta: Any,
dense_search_config: Optional[Dict[str, Any]] = None,
- initial_candidates: Optional[List[CandidateData]] = None,
+ initial_candidates: Optional[Iterable[CandidateData]] = None,
defer_dense_rebuild_start: bool = False,
):
"""Initialize a local index instance.
@@ -319,7 +364,7 @@ class LocalIndex(IIndex):
index_path_or_json (str): Path to index files or JSON configuration
meta: Index metadata object containing configuration
dense_search_config: Optional dense-search backend configuration.
- initial_candidates: Records used to initialize the dense-search shadow state.
+ initial_candidates: Records consumed to initialize the dense-search shadow state.
defer_dense_rebuild_start: Delay the background rebuild worker until the
caller has finished initializing the native index.
"""
@@ -352,6 +397,7 @@ class LocalIndex(IIndex):
dense_search_backend = dense_search_config.get("backend")
if dense_search_backend in {"cuvs", "auto_cuvs"}:
self._auto_cuvs = dense_search_backend == "auto_cuvs"
+ candidate_iterable = initial_candidates if initial_candidates is not None else ()
vector_meta = meta.inner_meta.get("VectorIndex", {})
field_types = {
name: DataProcessor.normalize_field_type(field_meta.get("FieldType", ""))
@@ -366,7 +412,7 @@ class LocalIndex(IIndex):
config=dense_search_config,
auto_memory=self._auto_cuvs,
)
- self.dense_search.add_candidates(initial_candidates or [])
+ self.dense_search.add_candidates(candidate_iterable)
self._auto_background_rebuild = self._auto_cuvs and bool(
dense_search_config.get("auto_background_rebuild", False)
)
@@ -374,10 +420,34 @@ class LocalIndex(IIndex):
max(0, int(dense_search_config.get("auto_rebuild_debounce_ms", 500))) / 1000.0
)
except CuVSUnavailableError:
+ failed_dense_search = self.dense_search
+ self.dense_search = None
+ if failed_dense_search is not None:
+ try:
+ failed_dense_search.close()
+ except Exception:
+ logger.warning(
+ "Failed to close unavailable cuVS dense search", exc_info=True
+ )
if not self._auto_cuvs:
raise
logger.info("cuVS auto mode unavailable; keeping native dense search")
+ except Exception:
+ failed_dense_search = self.dense_search
self.dense_search = None
+ if failed_dense_search is not None:
+ try:
+ failed_dense_search.close()
+ except Exception:
+ logger.warning("Failed to close partial cuVS dense search", exc_info=True)
+ raise
+ finally:
+ close_candidates = getattr(candidate_iterable, "close", None)
+ if callable(close_candidates):
+ try:
+ close_candidates()
+ except Exception:
+ logger.warning("Failed to close dense recovery iterator", exc_info=True)
if not defer_dense_rebuild_start:
self._start_dense_rebuild_worker()
@@ -783,7 +853,7 @@ class LocalIndex(IIndex):
) * 1000.0
native_count = self.dense_search.preflight_native_count(
filters,
- self._evaluate_cuvs_filter,
+ self._evaluate_cuvs_filter_for_routing,
self.engine_proxy.set_filter_layout,
telemetry=cuvs_telemetry,
)
@@ -879,6 +949,11 @@ class LocalIndex(IIndex):
) -> Tuple[List[int], List[float]]:
if self.dense_search is None or self.engine_proxy is None:
raise RuntimeError("cuVS search requires an initialized index")
+ filter_resolver = (
+ self._evaluate_cuvs_filter_for_routing
+ if self._auto_cuvs
+ else self._evaluate_cuvs_filter
+ )
if self._auto_background_rebuild:
queue_started = time.perf_counter()
with self._dense_search_lock.read():
@@ -892,7 +967,7 @@ class LocalIndex(IIndex):
query_vector,
limit,
filters,
- self._evaluate_cuvs_filter,
+ filter_resolver,
self.engine_proxy.set_filter_layout,
telemetry=telemetry,
)
@@ -906,7 +981,7 @@ class LocalIndex(IIndex):
query_vector,
limit,
filters,
- self._evaluate_cuvs_filter,
+ filter_resolver,
self.engine_proxy.set_filter_layout,
telemetry=telemetry,
)
@@ -920,7 +995,7 @@ class LocalIndex(IIndex):
query_vector,
limit,
filters,
- self._evaluate_cuvs_filter,
+ filter_resolver,
self.engine_proxy.set_filter_layout,
telemetry=telemetry,
)
@@ -962,14 +1037,26 @@ class LocalIndex(IIndex):
if telemetry.route_reason == "pending":
telemetry.route_reason = "native_fallback"
- def _evaluate_cuvs_filter(self, filters: Dict[str, Any]) -> Tuple[List[int], int, int]:
+ def _evaluate_cuvs_filter(
+ self, filters: Dict[str, Any]
+ ) -> Tuple[Union[List[int], bytes], int, int]:
if self.dense_search is None or self.engine_proxy is None:
raise RuntimeError("cuVS filter evaluation requires an initialized index")
- return self.engine_proxy.evaluate_filter(
+ return self.engine_proxy.evaluate_filter_packed(
filters,
max_cached_candidates=self.dense_search.native_filter_threshold(filters),
)
+ def _evaluate_cuvs_filter_for_routing(
+ self, filters: Dict[str, Any]
+ ) -> Tuple[Union[List[int], bytes], int, int]:
+ if self.dense_search is None or self.engine_proxy is None:
+ raise RuntimeError("cuVS filter evaluation requires an initialized index")
+ return self.engine_proxy.evaluate_filter_for_routing_packed(
+ filters,
+ native_threshold=self.dense_search.native_filter_threshold(filters),
+ )
+
@staticmethod
def _cuvs_telemetry_enabled() -> bool:
if logger.isEnabledFor(logging.DEBUG):
@@ -1247,7 +1334,7 @@ class PersistentIndex(LocalIndex):
name: str,
meta: Any,
path: str,
- cands_list: Optional[List[CandidateData]] = None,
+ cands_list: Optional[Iterable[CandidateData]] = None,
force_rebuild: bool = False,
initial_timestamp: Optional[int] = None,
dense_search_config: Optional[Dict[str, Any]] = None,
@@ -1262,7 +1349,9 @@ class PersistentIndex(LocalIndex):
name (str): Name identifier for the index (used as subdirectory name)
meta: Index metadata containing configuration
path (str): Parent directory path where index data will be stored
- cands_list (list): Initial data for creating a new index. Defaults to None.
+ cands_list: Initial records for a new native index or dense-search shadow.
+ Existing native snapshots consume this iterable only when the configured
+ dense-search backend needs to rehydrate its shadow state.
force_rebuild (bool): If True, rebuilds the index even if it exists.
Defaults to False.
initial_timestamp (Optional[int]): Timestamp to use if creating a new index
@@ -1282,7 +1371,7 @@ class PersistentIndex(LocalIndex):
- Apply any pending delta updates from collection
"""
if cands_list is None:
- cands_list = []
+ cands_list = ()
validate_name_str(name)
self.index_dir = str(safe_join_name(path, name))
@@ -1294,6 +1383,11 @@ class PersistentIndex(LocalIndex):
# At this point, there is no index, need to create a new one
if not newest_version or force_rebuild:
+ # Building a new native index needs both len() and another pass when
+ # initializing an optional dense shadow. Recovery of an existing
+ # snapshot stays single-pass and does not take this fallback.
+ if not isinstance(cands_list, list):
+ cands_list = list(cands_list)
self._create_new_index(name, meta, cands_list, initial_timestamp)
else:
self.now_version = str(newest_version)
diff --git a/openviking/storage/vectordb/store/local_store.py b/openviking/storage/vectordb/store/local_store.py
index 9f3728930..7c6b16183 100644
--- a/openviking/storage/vectordb/store/local_store.py
+++ b/openviking/storage/vectordb/store/local_store.py
@@ -1,6 +1,6 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-from typing import List, Tuple, Union
+from typing import Iterator, List, Tuple, Union
import openviking.storage.vectordb.engine as engine
from openviking.storage.vectordb.store.store import BatchOp, IMutiTableStore, Op, OpType
@@ -8,6 +8,8 @@ from openviking.storage.vectordb.utils.stale_lock import clean_stale_rocksdb_loc
# Constant for the maximum Unicode character, used for range queries to cover all possible keys
MAX_UNICODE_CHAR = "\U0010ffff"
+STORE_SCAN_PAGE_SIZE = 1024
+STORE_SCAN_PAGE_BYTES = 32 * 1024 * 1024
def create_store_engine_proxy(path: str = "") -> "StoreEngineProxy":
@@ -107,6 +109,60 @@ class StoreEngineProxy(IMutiTableStore):
if data[0].startswith(table_name)
]
+ def iter_all(
+ self,
+ table_name: str,
+ page_size: int = STORE_SCAN_PAGE_SIZE,
+ page_bytes: int = STORE_SCAN_PAGE_BYTES,
+ ) -> Iterator[Tuple[str, bytes]]:
+ """Scan a table in bounded native pages.
+
+ Pages are capped by row count and ``max(byte budget, one encoded row)``
+ so an oversized row cannot cause a false end-of-scan. Each native
+ page is internally consistent, but separate page calls do not share a
+ snapshot; collection recovery invokes this while the store is quiescent.
+ """
+ if page_size <= 0:
+ raise ValueError("Store scan page size must be positive")
+ if page_bytes <= 0:
+ raise ValueError("Store scan page byte budget must be positive")
+
+ seek_page = getattr(self.storage_engine, "seek_range_page", None)
+ if not callable(seek_page):
+ raise RuntimeError(
+ "The native VectorDB engine does not support bounded store scans; "
+ "rebuild or reinstall the matching OpenViking engine package"
+ )
+
+ end_key = table_name + MAX_UNICODE_CHAR
+ cursor = table_name
+ start_exclusive = False
+ try:
+ page = seek_page(cursor, end_key, page_size, page_bytes, start_exclusive)
+ except NotImplementedError as exc:
+ raise RuntimeError(
+ "The native VectorDB engine does not support bounded store scans; "
+ "rebuild or reinstall the matching OpenViking engine package"
+ ) from exc
+
+ while page:
+ next_cursor = page[-1][0]
+ if next_cursor < cursor or (start_exclusive and next_cursor <= cursor):
+ raise RuntimeError(
+ "Native bounded store scan did not advance its continuation cursor"
+ )
+ for full_key, value in page:
+ if full_key.startswith(table_name):
+ yield full_key[len(table_name) :], value
+ # Drop the page before the next native call so two encoded-vector
+ # pages are not live at the same time during recovery.
+ page = None
+ full_key = None
+ value = None
+ cursor = next_cursor
+ start_exclusive = True
+ page = seek_page(cursor, end_key, page_size, page_bytes, start_exclusive)
+
def begin_to_seek(self, end_key: str, table_name: str) -> List[Tuple[str, bytes]]:
"""Retrieve all entries from the beginning to a specific key.
diff --git a/openviking/storage/vectordb/store/store.py b/openviking/storage/vectordb/store/store.py
index efbeefc0d..54580b124 100644
--- a/openviking/storage/vectordb/store/store.py
+++ b/openviking/storage/vectordb/store/store.py
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: AGPL-3.0
from abc import ABC, abstractmethod
from enum import Enum
-from typing import Any, List, Tuple, Union
+from typing import Any, Iterator, List, Tuple, Union
class IKVStore(ABC):
@@ -189,6 +189,15 @@ class IMutiTableStore(ABC):
"""
pass
+ def iter_all(self, table_name: str) -> Iterator[Tuple[str, bytes]]:
+ """Iterate over a table, with a materializing compatibility fallback.
+
+ Store implementations with a cursor or paged range API should override
+ this method. Keeping the default in terms of ``read_all`` preserves
+ existing third-party and test-store behavior.
+ """
+ yield from self.read_all(table_name)
+
@abstractmethod
def seek_to_end(self, key: str, table_name: str) -> List[Tuple[str, bytes]]:
"""Retrieve all entries from a starting key to the end of the table.
diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.py
index 6a14d5f90..10c6b4c3a 100644
--- a/openviking/storage/vectordb/store/store_manager.py
+++ b/openviking/storage/vectordb/store/store_manager.py
@@ -1,7 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
import time
-from typing import List, Optional, Tuple
+from typing import Iterator, List, Optional, Tuple
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
from openviking.storage.vectordb.store.local_store import create_store_engine_proxy
@@ -225,9 +225,17 @@ class StoreManager:
Returns:
List[CandidateData]: List of all candidate data objects.
"""
- cands_kv_list = self.storage.read_all(StoreManager.CandsTable)
- cands_list = [CandidateData.from_bytes(data[1]) for data in cands_kv_list]
- return cands_list
+ return list(self.iter_all_cands_data())
+
+ def iter_all_cands_data(self) -> Iterator[CandidateData]:
+ """Iterate candidates without retaining deserialized full tables.
+
+ The local native store also bounds encoded rows by page. Generic store
+ implementations retain the ``IMutiTableStore.iter_all`` compatibility
+ fallback and may still materialize their encoded table.
+ """
+ for _, bytes_data in self.storage.iter_all(StoreManager.CandsTable):
+ yield CandidateData.from_bytes(bytes_data)
def clear(self):
"""Clear all data from the store."""
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 203f6818e..877030eb5 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -9,7 +9,7 @@ import uuid
from contextlib import asynccontextmanager
from typing import Any, AsyncIterator, Dict, List, Optional
-from openviking.core.namespace import canonicalize_uri, visible_roots
+from openviking.core.namespace import canonicalize_uri, uri_parts, visible_roots
from openviking.server.identity import RequestContext, Role
from openviking.storage.expr import And, Eq, FilterExpr, In, Or, PathScope, RawDSL
from openviking.storage.vectordb.collection.collection import Collection
@@ -1469,16 +1469,22 @@ class VikingVectorIndexBackend:
if context_type:
filters.append(Eq("context_type", context_type))
+ canonical_targets = [
+ canonicalize_uri(target_dir, ctx)
+ for target_dir in target_directories or []
+ if target_dir
+ ]
tenant_filter = self._tenant_filter(ctx, context_type=context_type)
+ if tenant_filter and self._targets_within_visible_roots(ctx, canonical_targets):
+ # The target scopes are already narrower than the tenant-visible
+ # roots. Keep account isolation, but avoid recursively evaluating
+ # the broader path scopes as an additional filter.
+ tenant_filter = Eq("account_id", ctx.account_id)
if tenant_filter:
filters.append(tenant_filter)
- if target_directories:
- uri_conds = [
- PathScope("uri", canonicalize_uri(target_dir, ctx), depth=-1)
- for target_dir in target_directories
- if target_dir
- ]
+ if canonical_targets:
+ uri_conds = [PathScope("uri", target_dir, depth=-1) for target_dir in canonical_targets]
if uri_conds:
filters.append(Or(uri_conds))
@@ -1493,6 +1499,20 @@ class VikingVectorIndexBackend:
return self._merge_filters(*filters)
+ @staticmethod
+ def _targets_within_visible_roots(ctx: RequestContext, canonical_targets: List[str]) -> bool:
+ if not canonical_targets:
+ return False
+
+ root_parts = [tuple(uri_parts(root)) for root in visible_roots(ctx)]
+ return all(
+ any(
+ len(target_parts) >= len(root) and target_parts[: len(root)] == root
+ for root in root_parts
+ )
+ for target_parts in (tuple(uri_parts(target)) for target in canonical_targets)
+ )
+
@staticmethod
def _tenant_filter(
ctx: RequestContext, context_type: Optional[str] = None
diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.py
index a15b444d8..1a5bfdee4 100644
--- a/openviking/telemetry/operation.py
+++ b/openviking/telemetry/operation.py
@@ -17,6 +17,7 @@ _CUVS_TIMING_FIELDS = (
"total_ms",
"preflight_ms",
"queue_ms",
+ "gpu_gate_queue_ms",
"build_ms",
"filter_prepare_ms",
"gpu_search_ms",
@@ -306,6 +307,12 @@ class TelemetrySummaryBuilder:
"routes": cls._counter_breakdown("vector.cuvs.routes", counters),
"filter_kinds": cls._counter_breakdown("vector.cuvs.filter_kinds", counters),
"filter_cache_hits": cls._i(counters.get("vector.cuvs.filter_cache_hits"), 0),
+ "filter_cache_eviction_fallbacks": cls._i(
+ counters.get("vector.cuvs.filter_cache_eviction_fallbacks"), 0
+ ),
+ "packed_filter_queries": cls._i(
+ counters.get("vector.cuvs.packed_filter_queries"), 0
+ ),
"native_filter_reuses": cls._i(
counters.get("vector.cuvs.native_filter_reuses"), 0
),
@@ -499,6 +506,10 @@ class OperationTelemetry:
self._counters["vector.cuvs.auto_mode_searches"] += 1
if TelemetrySummaryBuilder._bool(metrics.get("filter_cache_hit"), False):
self._counters["vector.cuvs.filter_cache_hits"] += 1
+ if TelemetrySummaryBuilder._bool(metrics.get("filter_cache_eviction_fallback"), False):
+ self._counters["vector.cuvs.filter_cache_eviction_fallbacks"] += 1
+ if TelemetrySummaryBuilder._bool(metrics.get("filter_words_packed"), False):
+ self._counters["vector.cuvs.packed_filter_queries"] += 1
if TelemetrySummaryBuilder._bool(metrics.get("native_filter_reused"), False):
self._counters["vector.cuvs.native_filter_reuses"] += 1
if TelemetrySummaryBuilder._bool(metrics.get("build_performed"), False):
diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cpp
index 6ae2ac906..d2a893442 100644
--- a/src/abi3_engine_backend.cpp
+++ b/src/abi3_engine_backend.cpp
@@ -1179,6 +1179,79 @@ PyObject* build_filter_result(const vdb::FilterResult& result) {
return payload;
}
+PyObject* build_packed_filter_result(const vdb::FilterResult& result) {
+ PyObject* payload = PyDict_New();
+ if (payload == nullptr) {
+ return nullptr;
+ }
+
+ if (result.bitset_words.size() >
+ static_cast<size_t>(PY_SSIZE_T_MAX) / sizeof(uint32_t)) {
+ Py_DECREF(payload);
+ PyErr_SetString(PyExc_OverflowError, "Packed filter bitmap is too large");
+ return nullptr;
+ }
+
+ // Encode explicitly instead of exposing the host byte order. This ABI is
+ // consumed with NumPy dtype "<u4" before the bitmap is copied to a device.
+ const Py_ssize_t packed_size = static_cast<Py_ssize_t>(
+ result.bitset_words.size() * sizeof(uint32_t));
+ PyObject* bitset_words_le =
+ PyBytes_FromStringAndSize(nullptr, packed_size);
+ if (bitset_words_le == nullptr) {
+ Py_DECREF(payload);
+ return nullptr;
+ }
+ char* packed_words = PyBytes_AsString(bitset_words_le);
+ if (packed_words == nullptr) {
+ Py_DECREF(bitset_words_le);
+ Py_DECREF(payload);
+ return nullptr;
+ }
+ auto pack_words = [&]() {
+ for (size_t index = 0; index < result.bitset_words.size(); ++index) {
+ const uint32_t word = result.bitset_words[index];
+ const size_t offset = index * sizeof(uint32_t);
+ packed_words[offset] = static_cast<char>(word & 0xffU);
+ packed_words[offset + 1] = static_cast<char>((word >> 8U) & 0xffU);
+ packed_words[offset + 2] = static_cast<char>((word >> 16U) & 0xffU);
+ packed_words[offset + 3] = static_cast<char>((word >> 24U) & 0xffU);
+ }
+ };
+ if (result.bitset_words.size() >= 256) {
+ call_without_gil(pack_words);
+ } else {
+ pack_words();
+ }
+
+ PyObject* eligible_count =
+ PyLong_FromUnsignedLongLong(result.eligible_count);
+ PyObject* native_filter_token =
+ PyLong_FromUnsignedLongLong(result.native_filter_token);
+ if (eligible_count == nullptr || native_filter_token == nullptr) {
+ Py_XDECREF(eligible_count);
+ Py_DECREF(bitset_words_le);
+ Py_XDECREF(native_filter_token);
+ Py_DECREF(payload);
+ return nullptr;
+ }
+
+ if (PyDict_SetItemString(payload, "eligible_count", eligible_count) < 0 ||
+ PyDict_SetItemString(payload, "bitset_words_le", bitset_words_le) < 0 ||
+ PyDict_SetItemString(payload, "native_filter_token",
+ native_filter_token) < 0) {
+ Py_DECREF(eligible_count);
+ Py_DECREF(bitset_words_le);
+ Py_DECREF(native_filter_token);
+ Py_DECREF(payload);
+ return nullptr;
+ }
+ Py_DECREF(eligible_count);
+ Py_DECREF(bitset_words_le);
+ Py_DECREF(native_filter_token);
+ return payload;
+}
+
PyObject* build_state_result(const vdb::StateResult& result) {
PyObject* payload = PyDict_New();
if (payload == nullptr) {
@@ -1403,6 +1476,28 @@ PyObject* py_index_engine_evaluate_filter(PyObject*, PyObject* args) {
}
}
+PyObject* py_index_engine_evaluate_filter_packed(PyObject*, PyObject* args) {
+ PyObject* capsule = nullptr;
+ const char* dsl = nullptr;
+ if (!PyArg_ParseTuple(args, "Os", &capsule, &dsl)) {
+ return nullptr;
+ }
+
+ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);
+ if (engine == nullptr) {
+ return nullptr;
+ }
+
+ try {
+ const vdb::FilterResult result =
+ call_without_gil([&]() { return engine->evaluate_filter(dsl); });
+ return build_packed_filter_result(result);
+ } catch (const std::exception& exc) {
+ raise_runtime_error(exc.what());
+ return nullptr;
+ }
+}
+
PyObject* py_index_engine_evaluate_filter_cached(PyObject*, PyObject* args) {
PyObject* capsule = nullptr;
const char* dsl = nullptr;
@@ -1429,6 +1524,85 @@ PyObject* py_index_engine_evaluate_filter_cached(PyObject*, PyObject* args) {
}
}
+PyObject* py_index_engine_evaluate_filter_cached_packed(PyObject*,
+ PyObject* args) {
+ PyObject* capsule = nullptr;
+ const char* dsl = nullptr;
+ unsigned long long max_cached_candidates = 0;
+ if (!PyArg_ParseTuple(args, "OsK", &capsule, &dsl,
+ &max_cached_candidates)) {
+ return nullptr;
+ }
+
+ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);
+ if (engine == nullptr) {
+ return nullptr;
+ }
+
+ try {
+ const vdb::FilterResult result = call_without_gil([&]() {
+ return engine->evaluate_filter(
+ dsl, static_cast<uint64_t>(max_cached_candidates));
+ });
+ return build_packed_filter_result(result);
+ } catch (const std::exception& exc) {
+ raise_runtime_error(exc.what());
+ return nullptr;
+ }
+}
+
+PyObject* py_index_engine_evaluate_filter_for_routing(PyObject*,
+ PyObject* args) {
+ PyObject* capsule = nullptr;
+ const char* dsl = nullptr;
+ unsigned long long native_threshold = 0;
+ if (!PyArg_ParseTuple(args, "OsK", &capsule, &dsl, &native_threshold)) {
+ return nullptr;
+ }
+
+ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);
+ if (engine == nullptr) {
+ return nullptr;
+ }
+
+ try {
+ const vdb::FilterResult result = call_without_gil([&]() {
+ return engine->evaluate_filter_for_routing(
+ dsl, static_cast<uint64_t>(native_threshold));
+ });
+ return build_filter_result(result);
+ } catch (const std::exception& exc) {
+ raise_runtime_error(exc.what());
+ return nullptr;
+ }
+}
+
+PyObject* py_index_engine_evaluate_filter_for_routing_packed(
+ PyObject*, PyObject* args) {
+ PyObject* capsule = nullptr;
+ const char* dsl = nullptr;
+ unsigned long long native_threshold = 0;
+ if (!PyArg_ParseTuple(args, "OsK", &capsule, &dsl, &native_threshold)) {
+ return nullptr;
+ }
+
+ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);
+ if (engine == nullptr) {
+ return nullptr;
+ }
+
+ try {
+ const vdb::FilterResult result = call_without_gil([&]() {
+ return engine->evaluate_filter_for_routing(
+ dsl, static_cast<uint64_t>(native_threshold));
+ });
+ return build_packed_filter_result(result);
+ } catch (const std::exception& exc) {
+ raise_runtime_error(exc.what());
+ return nullptr;
+ }
+}
+
PyObject* py_index_engine_dump(PyObject*, PyObject* args) {
PyObject* capsule = nullptr;
const char* path = nullptr;
@@ -1645,6 +1819,33 @@ PyObject* py_store_clear_data(PyObject*, PyObject* args) {
}
}
+PyObject* store_items_to_python(
+ const std::vector<std::pair<std::string, std::string>>& items) {
+ PyObject* list = PyList_New(static_cast<Py_ssize_t>(items.size()));
+ if (list == nullptr) {
+ return nullptr;
+ }
+ for (Py_ssize_t i = 0; i < static_cast<Py_ssize_t>(items.size()); ++i) {
+ const auto& item = items[static_cast<size_t>(i)];
+ PyObject* tuple = PyTuple_New(2);
+ PyObject* key = PyUnicode_FromStringAndSize(
+ item.first.data(), static_cast<Py_ssize_t>(item.first.size()));
+ PyObject* value = PyBytes_FromStringAndSize(
+ item.second.data(), static_cast<Py_ssize_t>(item.second.size()));
+ if (tuple == nullptr || key == nullptr || value == nullptr) {
+ Py_XDECREF(tuple);
+ Py_XDECREF(key);
+ Py_XDECREF(value);
+ Py_DECREF(list);
+ return nullptr;
+ }
+ PyTuple_SetItem(tuple, 0, key);
+ PyTuple_SetItem(tuple, 1, value);
+ PyList_SetItem(list, i, tuple);
+ }
+ return list;
+}
+
PyObject* py_store_seek_range(PyObject*, PyObject* args) {
PyObject* capsule = nullptr;
const char* start_key = nullptr;
@@ -1661,29 +1862,46 @@ PyObject* py_store_seek_range(PyObject*, PyObject* args) {
try {
const auto items = call_without_gil(
[&]() { return store->seek_range(start_key, end_key); });
- PyObject* list = PyList_New(static_cast<Py_ssize_t>(items.size()));
- if (list == nullptr) {
- return nullptr;
- }
- for (Py_ssize_t i = 0; i < static_cast<Py_ssize_t>(items.size()); ++i) {
- const auto& item = items[static_cast<size_t>(i)];
- PyObject* tuple = PyTuple_New(2);
- PyObject* key = PyUnicode_FromStringAndSize(
- item.first.data(), static_cast<Py_ssize_t>(item.first.size()));
- PyObject* value = PyBytes_FromStringAndSize(
- item.second.data(), static_cast<Py_ssize_t>(item.second.size()));
- if (tuple == nullptr || key == nullptr || value == nullptr) {
- Py_XDECREF(tuple);
- Py_XDECREF(key);
- Py_XDECREF(value);
- Py_DECREF(list);
- return nullptr;
- }
- PyTuple_SetItem(tuple, 0, key);
- PyTuple_SetItem(tuple, 1, value);
- PyList_SetItem(list, i, tuple);
- }
- return list;
+ return store_items_to_python(items);
+ } catch (const std::exception& exc) {
+ raise_runtime_error(exc.what());
+ return nullptr;
+ }
+}
+
+PyObject* py_store_seek_range_page(PyObject*, PyObject* args) {
+ PyObject* capsule = nullptr;
+ const char* start_key = nullptr;
+ const char* end_key = nullptr;
+ Py_ssize_t limit = 0;
+ Py_ssize_t max_bytes = 0;
+ int start_exclusive = 0;
+ if (!PyArg_ParseTuple(args, "Ossnnp", &capsule, &start_key, &end_key,
+ &limit, &max_bytes, &start_exclusive)) {
+ return nullptr;
+ }
+ if (limit <= 0) {
+ raise_value_error("seek_range_page limit must be positive");
+ return nullptr;
+ }
+ if (max_bytes <= 0) {
+ raise_value_error("seek_range_page max_bytes must be positive");
+ return nullptr;
+ }
+
+ auto* store = capsule_to_ptr<vdb::KVStore>(capsule, kStoreCapsuleName);
+ if (store == nullptr) {
+ return nullptr;
+ }
+
+ try {
+ const auto items = call_without_gil([&]() {
+ return store->seek_range_page(start_key, end_key,
+ static_cast<size_t>(limit),
+ static_cast<size_t>(max_bytes),
+ start_exclusive != 0);
+ });
+ return store_items_to_python(items);
} catch (const std::exception& exc) {
raise_runtime_error(exc.what());
return nullptr;
@@ -1722,9 +1940,21 @@ PyMethodDef kModuleMethods[] = {
METH_VARARGS, "Register an external label order for scalar filters."},
{"_index_engine_evaluate_filter", py_index_engine_evaluate_filter,
METH_VARARGS, "Evaluate a scalar filter in an external label order."},
+ {"_index_engine_evaluate_filter_packed",
+ py_index_engine_evaluate_filter_packed, METH_VARARGS,
+ "Evaluate a scalar filter and return little-endian packed words."},
{"_index_engine_evaluate_filter_cached",
py_index_engine_evaluate_filter_cached, METH_VARARGS,
"Evaluate and optionally retain a native scalar filter bitmap."},
+ {"_index_engine_evaluate_filter_cached_packed",
+ py_index_engine_evaluate_filter_cached_packed, METH_VARARGS,
+ "Evaluate and retain a filter while returning packed words."},
+ {"_index_engine_evaluate_filter_for_routing",
+ py_index_engine_evaluate_filter_for_routing, METH_VARARGS,
+ "Evaluate a scalar filter for an adaptive native/GPU route decision."},
+ {"_index_engine_evaluate_filter_for_routing_packed",
+ py_index_engine_evaluate_filter_for_routing_packed, METH_VARARGS,
+ "Route a scalar filter while returning little-endian packed words."},
{"_index_engine_dump", py_index_engine_dump, METH_VARARGS,
"Dump index state to disk."},
{"_index_engine_get_state", py_index_engine_get_state, METH_VARARGS,
@@ -1745,6 +1975,8 @@ PyMethodDef kModuleMethods[] = {
"Clear the store."},
{"_store_seek_range", py_store_seek_range, METH_VARARGS,
"Read a key range from the store."},
+ {"_store_seek_range_page", py_store_seek_range_page, METH_VARARGS,
+ "Read a bounded page from a key range."},
{nullptr, nullptr, 0, nullptr},
};
diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cpp
index f00c25981..a693e62d9 100644
--- a/src/index/detail/index_manager_impl.cpp
+++ b/src/index/detail/index_manager_impl.cpp
@@ -1,6 +1,7 @@
// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
// SPDX-License-Identifier: AGPL-3.0
#include "index/detail/index_manager_impl.h"
+#include <algorithm>
#include <stdexcept>
#include <memory>
#include <chrono>
@@ -19,6 +20,9 @@ const std::string kMetaFile = "manager_meta.json";
const std::string kVectorIndexDir = "vector_index";
const std::string kScalarIndexDir = "scalar_index";
constexpr size_t kFilterTokenCacheCapacity = 32;
+constexpr uint64_t kFilterLayoutInverseMaxSpanFactor = 4;
+constexpr uint32_t kMissingFilterLayoutOffset =
+ std::numeric_limits<uint32_t>::max();
IndexManagerImpl::IndexManagerImpl(const std::string& path_or_json) {
int ret = 0;
@@ -233,14 +237,51 @@ int IndexManagerImpl::search_with_filter_token(const SearchRequest& req,
int IndexManagerImpl::set_filter_layout(
const std::vector<uint64_t>& ordered_labels) {
std::unique_lock<std::shared_mutex> lock(rw_mutex_);
- filter_layout_offsets_.clear();
+ clear_filter_layout_();
filter_layout_offsets_.reserve(ordered_labels.size());
+ uint64_t valid_offset_count = 0;
+ uint32_t min_offset = kMissingFilterLayoutOffset;
+ uint32_t max_offset = 0;
for (const uint64_t label : ordered_labels) {
const int offset = vector_index_->get_offset_by_label(label);
- filter_layout_offsets_.push_back(
- offset >= 0 ? static_cast<uint32_t>(offset)
- : std::numeric_limits<uint32_t>::max());
+ if (offset < 0) {
+ filter_layout_offsets_.push_back(kMissingFilterLayoutOffset);
+ continue;
+ }
+ const auto logical_offset = static_cast<uint32_t>(offset);
+ filter_layout_offsets_.push_back(logical_offset);
+ ++valid_offset_count;
+ min_offset = std::min(min_offset, logical_offset);
+ max_offset = std::max(max_offset, logical_offset);
+ }
+
+ if (valid_offset_count == 0) {
+ return 0;
+ }
+ const uint64_t inverse_span =
+ static_cast<uint64_t>(max_offset) - min_offset + 1;
+ if (inverse_span >
+ kFilterLayoutInverseMaxSpanFactor * valid_offset_count) {
+ return 0;
+ }
+
+ filter_layout_rows_by_offset_.assign(
+ static_cast<size_t>(inverse_span), kMissingFilterLayoutOffset);
+ for (size_t row = 0; row < filter_layout_offsets_.size(); ++row) {
+ const uint32_t offset = filter_layout_offsets_[row];
+ if (offset == kMissingFilterLayoutOffset) {
+ continue;
+ }
+ const size_t inverse_row = static_cast<size_t>(offset - min_offset);
+ if (filter_layout_rows_by_offset_[inverse_row] !=
+ kMissingFilterLayoutOffset) {
+ filter_layout_rows_by_offset_.clear();
+ return 0;
+ }
+ filter_layout_rows_by_offset_[inverse_row] = static_cast<uint32_t>(row);
}
+ filter_layout_inverse_base_ = min_offset;
+ filter_layout_inverse_ready_ = true;
return 0;
}
@@ -284,6 +325,106 @@ int IndexManagerImpl::evaluate_filter(const std::string& dsl,
return 0;
}
+int IndexManagerImpl::evaluate_filter_for_routing(
+ const std::string& dsl, uint64_t native_threshold,
+ FilterResult& result) {
+ if (native_threshold == 0) {
+ return evaluate_filter(dsl, 0, result);
+ }
+
+ SearchContext ctx;
+ if (int ret = parse_dsl_query(dsl, ctx); ret != 0) {
+ SPDLOG_ERROR(
+ "IndexManagerImpl::evaluate_filter_for_routing [{}] parse failed",
+ dsl);
+ return ret;
+ }
+
+ std::shared_lock<std::shared_mutex> lock(rw_mutex_);
+ BitmapPtr bitmap = nullptr;
+ if (ctx.filter_op) {
+ bitmap = calculate_filter_bitmap(ctx, dsl);
+ if (!bitmap) {
+ SPDLOG_DEBUG(
+ "IndexManagerImpl::evaluate_filter_for_routing "
+ "calculate_filter_bitmap returned null");
+ return -1;
+ }
+ }
+
+ result.eligible_count = 0;
+ result.bitset_words.clear();
+ result.native_filter_token = 0;
+
+ // A selective scalar bitmap can be projected through the inverse layout in
+ // O(bitmap cardinality), without scanning every external dense-index row.
+ if (bitmap && filter_layout_inverse_ready_ &&
+ bitmap->nbit() <= native_threshold) {
+ std::vector<uint32_t> eligible_offsets;
+ bitmap->get_set_list(eligible_offsets);
+ for (const uint32_t offset : eligible_offsets) {
+ if (offset < filter_layout_inverse_base_) {
+ continue;
+ }
+ const uint64_t inverse_row =
+ static_cast<uint64_t>(offset) - filter_layout_inverse_base_;
+ if (inverse_row >= filter_layout_rows_by_offset_.size() ||
+ filter_layout_rows_by_offset_[inverse_row] ==
+ kMissingFilterLayoutOffset) {
+ continue;
+ }
+ ++result.eligible_count;
+ }
+ if (result.eligible_count > 0) {
+ result.native_filter_token = cache_filter_bitmap_(bitmap);
+ }
+ return 0;
+ }
+
+ // Keep at most threshold + 1 row ids while counting. If the filter is wide,
+ // allocate the full projection only after the route decision is known.
+ std::vector<uint32_t> narrow_rows;
+ const bool cannot_exceed_threshold =
+ native_threshold >= filter_layout_offsets_.size();
+ if (!cannot_exceed_threshold) {
+ narrow_rows.reserve(static_cast<size_t>(native_threshold + 1));
+ }
+ for (size_t row = 0; row < filter_layout_offsets_.size(); ++row) {
+ const uint32_t offset = filter_layout_offsets_[row];
+ const bool eligible = offset != kMissingFilterLayoutOffset &&
+ (!bitmap || bitmap->Isset(offset));
+ if (!eligible) {
+ continue;
+ }
+ ++result.eligible_count;
+ if (cannot_exceed_threshold) {
+ continue;
+ }
+ if (result.bitset_words.empty()) {
+ narrow_rows.push_back(static_cast<uint32_t>(row));
+ if (result.eligible_count <= native_threshold) {
+ continue;
+ }
+ result.bitset_words.assign((filter_layout_offsets_.size() + 31) / 32,
+ 0);
+ for (const uint32_t narrow_row : narrow_rows) {
+ result.bitset_words[narrow_row / 32] |=
+ static_cast<uint32_t>(1U << (narrow_row % 32));
+ }
+ narrow_rows.clear();
+ continue;
+ }
+ result.bitset_words[row / 32] |=
+ static_cast<uint32_t>(1U << (row % 32));
+ }
+
+ if (bitmap && result.eligible_count > 0 &&
+ result.eligible_count <= native_threshold) {
+ result.native_filter_token = cache_filter_bitmap_(bitmap);
+ }
+ return 0;
+}
+
uint64_t IndexManagerImpl::cache_filter_bitmap_(const BitmapPtr& bitmap) {
std::lock_guard<std::mutex> lock(filter_token_mutex_);
uint64_t token = next_filter_token_++;
@@ -305,6 +446,13 @@ void IndexManagerImpl::clear_filter_token_cache_() {
filter_token_order_.clear();
}
+void IndexManagerImpl::clear_filter_layout_() {
+ filter_layout_offsets_.clear();
+ filter_layout_rows_by_offset_.clear();
+ filter_layout_inverse_base_ = 0;
+ filter_layout_inverse_ready_ = false;
+}
+
BitmapPtr IndexManagerImpl::calculate_filter_bitmap(const SearchContext& ctx,
const std::string& dsl) {
auto bitmap = ctx.filter_op->calc_bitmap(scalar_index_->get_field_sets(),
@@ -415,7 +563,7 @@ int IndexManagerImpl::add_data(const std::vector<AddDataRequest>& data_list) {
parsed_old_fields_list[i]);
}
if (has_update) {
- filter_layout_offsets_.clear();
+ clear_filter_layout_();
clear_filter_token_cache_();
auto duration = std::chrono::system_clock::now().time_since_epoch();
manager_meta_->update_timestamp =
@@ -459,7 +607,7 @@ int IndexManagerImpl::delete_data(
vector_index_->stream_delete_data(data.label);
}
if (has_update) {
- filter_layout_offsets_.clear();
+ clear_filter_layout_();
clear_filter_token_cache_();
auto duration = std::chrono::system_clock::now().time_since_epoch();
manager_meta_->update_timestamp =
diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.h
index a510dc486..6180b0246 100644
--- a/src/index/detail/index_manager_impl.h
+++ b/src/index/detail/index_manager_impl.h
@@ -44,6 +44,10 @@ class IndexManagerImpl : public IndexManager {
uint64_t max_cached_candidates,
FilterResult& result) override;
+ int evaluate_filter_for_routing(const std::string& dsl,
+ uint64_t native_threshold,
+ FilterResult& result) override;
+
int add_data(const std::vector<AddDataRequest>& data_list) override;
int delete_data(const std::vector<DeleteDataRequest>& data_list) override;
@@ -73,12 +77,17 @@ class IndexManagerImpl : public IndexManager {
void clear_filter_token_cache_();
+ void clear_filter_layout_();
+
private:
std::shared_mutex rw_mutex_;
std::shared_ptr<ManagerMeta> manager_meta_;
std::shared_ptr<ScalarIndex> scalar_index_;
std::shared_ptr<VectorIndexAdapter> vector_index_;
std::vector<uint32_t> filter_layout_offsets_;
+ std::vector<uint32_t> filter_layout_rows_by_offset_;
+ uint32_t filter_layout_inverse_base_ = 0;
+ bool filter_layout_inverse_ready_ = false;
std::mutex filter_token_mutex_;
uint64_t next_filter_token_ = 1;
std::deque<uint64_t> filter_token_order_;
diff --git a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp
index 1ec660a7e..79dd2b1a5 100644
--- a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp
+++ b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.cpp
@@ -320,6 +320,12 @@ int FieldBitmapGroup::parse_from_stream(std::ifstream& input) {
dir_index_ = std::make_shared<DirIndex>();
dir_index_->parse_from_stream(input);
+ // Runtime leaf-to-bitmap bindings are intentionally not serialized. The
+ // on-disk format remains compatible; rebuild the non-owning links after
+ // both the bitmap map and trie have been loaded.
+ for (const auto& entry : bitmap_group_) {
+ dir_index_->add_key(entry.first, entry.second.get());
+ }
}
} catch (std::exception& e) {
// SPDLOG_ERROR("FieldBitmapGroup parse_from_stream exception {}",
@@ -469,20 +475,8 @@ BitmapPtr FieldBitmapGroupSet::make_path_field_copy(
}
std::vector<const Bitmap*> bitmaps_to_union;
- std::unordered_set<std::string> all_unique_bitmaps;
-
for (const auto& path_prefix : keys) {
- std::unordered_set<std::string> unique_bitmaps;
- dip->get_merged_bitmap(path_prefix, depth, unique_bitmaps);
- all_unique_bitmaps.insert(unique_bitmaps.begin(), unique_bitmaps.end());
- }
-
- bitmaps_to_union.reserve(all_unique_bitmaps.size());
- for (const auto& bitmap_key : all_unique_bitmaps) {
- const Bitmap* bm = group->get_bitmap(bitmap_key);
- if (bm) {
- bitmaps_to_union.push_back(bm);
- }
+ dip->get_merged_bitmaps(path_prefix, depth, bitmaps_to_union);
}
auto final_bitmap = std::make_shared<Bitmap>();
diff --git a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h
index d4804ff7c..de43c6f94 100644
--- a/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h
+++ b/src/index/detail/scalar/bitmap_holder/bitmap_field_group.h
@@ -137,14 +137,12 @@ class FieldBitmapGroup : public BitmapGroupBase {
for (auto& key_i : keys) {
const std::string norm_key =
dir_index_ ? normalize_path_key(key_i) : key_i;
- if (!exist_bitmap(norm_key)) {
- if (dir_index_) {
- dir_index_->add_key(norm_key);
- }
- }
-
+ const bool is_new_bitmap = !exist_bitmap(norm_key);
Bitmap* temp_p = get_editable_bitmap(norm_key);
if (temp_p) {
+ if (dir_index_ && is_new_bitmap) {
+ dir_index_->add_key(norm_key, temp_p);
+ }
temp_p->Set(offset);
}
}
@@ -152,13 +150,12 @@ class FieldBitmapGroup : public BitmapGroupBase {
} else {
const std::string norm_key =
dir_index_ ? normalize_path_key(field_str) : field_str;
- if (!exist_bitmap(norm_key)) {
- if (dir_index_) {
- dir_index_->add_key(norm_key);
- }
- }
+ const bool is_new_bitmap = !exist_bitmap(norm_key);
Bitmap* temp_p = get_editable_bitmap(norm_key);
if (temp_p) {
+ if (dir_index_ && is_new_bitmap) {
+ dir_index_->add_key(norm_key, temp_p);
+ }
temp_p->Set(offset);
}
}
diff --git a/src/index/detail/scalar/bitmap_holder/dir_index.cpp b/src/index/detail/scalar/bitmap_holder/dir_index.cpp
index 5c6817764..fc2ffc257 100644
--- a/src/index/detail/scalar/bitmap_holder/dir_index.cpp
+++ b/src/index/detail/scalar/bitmap_holder/dir_index.cpp
@@ -1,9 +1,6 @@
// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
// SPDX-License-Identifier: AGPL-3.0
#include "dir_index.h"
-#include <deque>
-#include <unordered_set>
-#include "index/detail/scalar/bitmap_holder/bitmap_field_group.h"
namespace vectordb {
@@ -38,38 +35,51 @@ TrieNode* DirIndex::find_node(const std::string& path) const {
return node;
}
-void DirIndex::get_merged_bitmap(
- const std::string& path_prefix, int depth,
- std::unordered_set<std::string>& unique_bitmaps) const {
- TrieNode* start_node = find_node(path_prefix);
- if (!start_node) {
+void DirIndex::bind_leaf_bitmap(TrieNode* node, const Bitmap* bitmap) {
+ if (!node || !bitmap) {
+ return;
+ }
+ if (!node->leaf_bitmap_) {
+ node->leaf_bitmap_ = bitmap;
+ return;
+ }
+ if (node->leaf_bitmap_ == bitmap) {
return;
}
- std::string path_buffer = path_prefix.empty() ? "" : path_prefix;
- if (!path_buffer.empty() && path_buffer[0] != '/') {
- path_buffer.insert(path_buffer.begin(), '/');
+ auto& collisions = leaf_bitmap_collisions_[node];
+ if (std::find(collisions.begin(), collisions.end(), bitmap) ==
+ collisions.end()) {
+ collisions.push_back(bitmap);
}
- while (path_buffer.size() > 1 && path_buffer.back() == '/') {
- path_buffer.pop_back();
+}
+
+void DirIndex::get_merged_bitmaps(
+ const std::string& path_prefix, int depth,
+ std::vector<const Bitmap*>& bitmaps) const {
+ TrieNode* start_node = find_node(path_prefix);
+ if (!start_node) {
+ return;
}
- collect_bitmaps_recursive_optimized(start_node, 0, depth, unique_bitmaps,
- path_buffer);
+ collect_bitmaps_recursive(start_node, 0, depth, bitmaps);
}
-void DirIndex::collect_bitmaps_recursive_optimized(
+void DirIndex::collect_bitmaps_recursive(
const TrieNode* node, int current_depth, int max_depth,
- std::unordered_set<std::string>& bitmaps, std::string& path_buffer) const {
+ std::vector<const Bitmap*>& bitmaps) const {
if (!node) {
return;
}
if (node->is_leaf_) {
- if (path_buffer.empty() || path_buffer == "/") {
- bitmaps.insert("/");
- } else {
- bitmaps.insert(path_buffer);
+ if (node->leaf_bitmap_) {
+ bitmaps.push_back(node->leaf_bitmap_);
+ }
+ const auto collision_it = leaf_bitmap_collisions_.find(node);
+ if (collision_it != leaf_bitmap_collisions_.end()) {
+ bitmaps.insert(bitmaps.end(), collision_it->second.begin(),
+ collision_it->second.end());
}
}
@@ -78,19 +88,9 @@ void DirIndex::collect_bitmaps_recursive_optimized(
}
for (const auto& child_pair : node->children_) {
- const auto& segment = child_pair.first;
const auto& child_node = child_pair.second;
- size_t original_size = path_buffer.length();
-
- if (path_buffer.empty() || path_buffer == "/") {
- path_buffer = "/" + segment;
- } else {
- path_buffer += "/" + segment;
- }
-
- collect_bitmaps_recursive_optimized(child_node.get(), current_depth + 1,
- max_depth, bitmaps, path_buffer);
- path_buffer.resize(original_size);
+ collect_bitmaps_recursive(child_node.get(), current_depth + 1, max_depth,
+ bitmaps);
}
}
@@ -129,7 +129,8 @@ std::unique_ptr<TrieNode> DirIndex::parse_recursive(std::ifstream& input,
return node;
}
void DirIndex::parse_from_stream(std::ifstream& input) {
+ leaf_bitmap_collisions_.clear();
root_ = parse_recursive(input, nullptr);
}
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/index/detail/scalar/bitmap_holder/dir_index.h b/src/index/detail/scalar/bitmap_holder/dir_index.h
index 1a7ef696a..acec3bc48 100644
--- a/src/index/detail/scalar/bitmap_holder/dir_index.h
+++ b/src/index/detail/scalar/bitmap_holder/dir_index.h
@@ -2,13 +2,11 @@
// SPDX-License-Identifier: AGPL-3.0
#pragma once
#include <algorithm>
-#include <string>
-#include <vector>
-#include <set>
#include <memory>
-#include <unordered_map>
-#include <unordered_set>
#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
#include "common/io_utils.h"
#include "index/detail/scalar/bitmap_holder/bitmap.h"
@@ -18,6 +16,10 @@ struct TrieNode {
std::string path_segment_;
TrieNode* parent_ = nullptr;
std::unordered_map<std::string, std::unique_ptr<TrieNode>> children_;
+ // Non-owning pointer into BitmapGroupBase::bitmap_group_. Bitmap objects are
+ // updated in place and remain stable during every live directory-index
+ // access under IndexManager's reader/writer lock.
+ const Bitmap* leaf_bitmap_ = nullptr;
bool is_leaf_ = false;
TrieNode() = default;
@@ -31,7 +33,7 @@ class DirIndex {
DirIndex() = default;
virtual ~DirIndex() = default;
- void add_key(const std::string& key) {
+ void add_key(const std::string& key, const Bitmap* bitmap) {
TrieNode* node = root_.get();
for (const auto& segment : split_path(key)) {
auto it = node->children_.find(segment);
@@ -45,29 +47,35 @@ class DirIndex {
}
}
node->is_leaf_ = true;
+ bind_leaf_bitmap(node, bitmap);
}
- void get_merged_bitmap(const std::string& path_prefix, int depth,
- std::unordered_set<std::string>& unique_bitmaps) const;
+ void get_merged_bitmaps(const std::string& path_prefix, int depth,
+ std::vector<const Bitmap*>& bitmaps) const;
virtual void serialize_to_stream(std::ofstream& output);
virtual void parse_from_stream(std::ifstream& input);
private:
std::unique_ptr<TrieNode> root_ = std::make_unique<TrieNode>("", nullptr);
+ // Canonically equivalent spellings can resolve to the same trie leaf. Keep
+ // their additional bitmaps out of TrieNode so the normal one-bitmap case
+ // does not pay per-node vector overhead.
+ std::unordered_map<const TrieNode*, std::vector<const Bitmap*>>
+ leaf_bitmap_collisions_;
TrieNode* find_node(const std::string& path) const;
+ void bind_leaf_bitmap(TrieNode* node, const Bitmap* bitmap);
std::vector<std::string> split_path(const std::string& path) const;
void serialize_recursive(const TrieNode* node, std::ofstream& output) const;
std::unique_ptr<TrieNode> parse_recursive(std::ifstream& input,
TrieNode* parent);
- void collect_bitmaps_recursive_optimized(
+ void collect_bitmaps_recursive(
const TrieNode* node, int current_depth, int max_depth,
- std::unordered_set<std::string>& unique_bitmaps,
- std::string& path_buffer) const;
+ std::vector<const Bitmap*>& bitmaps) const;
};
using DirIndexPtr = std::shared_ptr<DirIndex>;
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cpp
index da9924566..0ebd15e1f 100644
--- a/src/index/index_engine.cpp
+++ b/src/index/index_engine.cpp
@@ -50,6 +50,18 @@ FilterResult IndexEngine::evaluate_filter(
return result;
}
+FilterResult IndexEngine::evaluate_filter_for_routing(
+ const std::string& dsl, uint64_t native_threshold) {
+ FilterResult result;
+ const int ret =
+ impl_->evaluate_filter_for_routing(dsl, native_threshold, result);
+ if (ret != 0) {
+ throw std::runtime_error(
+ "Failed to evaluate native scalar filter for routing");
+ }
+ return result;
+}
+
int IndexEngine::add_data(const std::vector<AddDataRequest>& data_list) {
return impl_->add_data(data_list);
}
diff --git a/src/index/index_engine.h b/src/index/index_engine.h
index cea8004fd..1c82a9244 100644
--- a/src/index/index_engine.h
+++ b/src/index/index_engine.h
@@ -33,6 +33,9 @@ class IndexEngine {
FilterResult evaluate_filter(const std::string& dsl,
uint64_t max_cached_candidates = 0);
+ FilterResult evaluate_filter_for_routing(const std::string& dsl,
+ uint64_t native_threshold);
+
int64_t dump(const std::string& dir);
StateResult get_state();
diff --git a/src/index/index_manager.h b/src/index/index_manager.h
index 420112bd9..9b6e1f92f 100644
--- a/src/index/index_manager.h
+++ b/src/index/index_manager.h
@@ -26,6 +26,12 @@ class IndexManager {
uint64_t max_cached_candidates,
FilterResult& result) = 0;
+ // A narrow result may omit bitset_words because adaptive mode will route it
+ // to native search. A zero threshold retains evaluate_filter semantics.
+ virtual int evaluate_filter_for_routing(const std::string& dsl,
+ uint64_t native_threshold,
+ FilterResult& result) = 0;
+
virtual int add_data(const std::vector<AddDataRequest>& data_list) = 0;
virtual int delete_data(const std::vector<DeleteDataRequest>& data_list) = 0;
diff --git a/src/store/kv_store.h b/src/store/kv_store.h
index 7ba551a43..0cb2761f8 100644
--- a/src/store/kv_store.h
+++ b/src/store/kv_store.h
@@ -1,6 +1,7 @@
// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
// SPDX-License-Identifier: AGPL-3.0
#pragma once
+#include <cstddef>
#include <string>
#include <vector>
#include "store/common_structs.h"
@@ -25,6 +26,10 @@ class KVStore {
virtual std::vector<std::pair<std::string, std::string>> seek_range(
const std::string& start_key, const std::string& end_key) = 0;
+
+ virtual std::vector<std::pair<std::string, std::string>> seek_range_page(
+ const std::string& start_key, const std::string& end_key, size_t limit,
+ size_t max_bytes, bool start_exclusive) = 0;
};
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/store/persist_store.cpp b/src/store/persist_store.cpp
index dcdf44373..6bf28abf5 100644
--- a/src/store/persist_store.cpp
+++ b/src/store/persist_store.cpp
@@ -4,6 +4,7 @@
#include "spdlog/spdlog.h"
#include <stdexcept>
#include <filesystem>
+#include <memory>
#include "leveldb/write_batch.h"
namespace vectordb {
@@ -112,6 +113,43 @@ std::vector<std::pair<std::string, std::string>> PersistStore::seek_range(
return key_values;
}
+std::vector<std::pair<std::string, std::string>>
+PersistStore::seek_range_page(const std::string& start_key,
+ const std::string& end_key, size_t limit,
+ size_t max_bytes, bool start_exclusive) {
+ std::vector<std::pair<std::string, std::string>> key_values;
+ if (limit == 0 || max_bytes == 0) {
+ return key_values;
+ }
+ size_t page_bytes = 0;
+ std::unique_ptr<leveldb::Iterator> it(
+ db_->NewIterator(leveldb::ReadOptions()));
+ it->Seek(start_key);
+ if (start_exclusive && it->Valid() && it->key().ToString() == start_key) {
+ it->Next();
+ }
+ for (; it->Valid() && it->key().ToString() < end_key &&
+ key_values.size() < limit;
+ it->Next()) {
+ const auto key = it->key();
+ const auto value = it->value();
+ const size_t item_bytes = key.size() + value.size();
+ // Always admit one oversized row so pagination can make progress. Every
+ // additional row must fit within the encoded-byte budget.
+ if (!key_values.empty() &&
+ (page_bytes >= max_bytes || item_bytes > max_bytes - page_bytes)) {
+ break;
+ }
+ key_values.push_back({key.ToString(), value.ToString()});
+ page_bytes += item_bytes;
+ }
+ if (!it->status().ok()) {
+ throw std::runtime_error("PersistStore::seek_range_page iterate error: " +
+ it->status().ToString());
+ }
+ return key_values;
+}
+
int PersistStore::exec_op(const std::vector<StorageOp>& ops) {
leveldb::WriteBatch batch;
for (const auto& op : ops) {
@@ -134,4 +172,4 @@ int PersistStore::exec_op(const std::vector<StorageOp>& ops) {
return 0;
}
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/store/persist_store.h b/src/store/persist_store.h
index cc878d751..23e7bb99e 100644
--- a/src/store/persist_store.h
+++ b/src/store/persist_store.h
@@ -30,8 +30,12 @@ class PersistStore : public KVStore {
std::vector<std::pair<std::string, std::string>> seek_range(
const std::string& start_key, const std::string& end_key) override;
+ std::vector<std::pair<std::string, std::string>> seek_range_page(
+ const std::string& start_key, const std::string& end_key, size_t limit,
+ size_t max_bytes, bool start_exclusive) override;
+
private:
leveldb::DB* db_ = nullptr;
};
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/store/volatile_store.cpp b/src/store/volatile_store.cpp
index f3ee9e3ed..9b8a6b66b 100644
--- a/src/store/volatile_store.cpp
+++ b/src/store/volatile_store.cpp
@@ -56,6 +56,34 @@ std::vector<std::pair<std::string, std::string>> VolatileStore::seek_range(
return key_values;
}
+std::vector<std::pair<std::string, std::string>>
+VolatileStore::seek_range_page(const std::string& start_key,
+ const std::string& end_key, size_t limit,
+ size_t max_bytes, bool start_exclusive) {
+ std::shared_lock<std::shared_mutex> lock(mutex_);
+ std::vector<std::pair<std::string, std::string>> key_values;
+ if (limit == 0 || max_bytes == 0) {
+ return key_values;
+ }
+ size_t page_bytes = 0;
+ auto iter = start_exclusive ? data_.upper_bound(start_key)
+ : data_.lower_bound(start_key);
+ for (; iter != data_.end() && iter->first < end_key &&
+ key_values.size() < limit;
+ ++iter) {
+ const size_t item_bytes = iter->first.size() + iter->second.size();
+ // Always admit one oversized row so pagination can make progress. Every
+ // additional row must fit within the encoded-byte budget.
+ if (!key_values.empty() &&
+ (page_bytes >= max_bytes || item_bytes > max_bytes - page_bytes)) {
+ break;
+ }
+ key_values.push_back({iter->first, iter->second});
+ page_bytes += item_bytes;
+ }
+ return key_values;
+}
+
int VolatileStore::exec_op(const std::vector<StorageOp>& ops) {
std::unique_lock<std::shared_mutex> lock(mutex_);
for (const auto& op : ops) {
@@ -71,4 +99,4 @@ int VolatileStore::exec_op(const std::vector<StorageOp>& ops) {
return 0;
}
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/store/volatile_store.h b/src/store/volatile_store.h
index 07d76e347..afbf64440 100644
--- a/src/store/volatile_store.h
+++ b/src/store/volatile_store.h
@@ -31,9 +31,13 @@ class VolatileStore : public KVStore {
std::vector<std::pair<std::string, std::string>> seek_range(
const std::string& start_key, const std::string& end_key) override;
+ std::vector<std::pair<std::string, std::string>> seek_range_page(
+ const std::string& start_key, const std::string& end_key, size_t limit,
+ size_t max_bytes, bool start_exclusive) override;
+
private:
std::map<std::string, std::string> data_;
mutable std::shared_mutex mutex_;
};
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/tests/engine/test_index_engine.cpp b/tests/engine/test_index_engine.cpp
index 330b74723..9bd4186dd 100644
--- a/tests/engine/test_index_engine.cpp
+++ b/tests/engine/test_index_engine.cpp
@@ -1,6 +1,8 @@
// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
// SPDX-License-Identifier: AGPL-3.0
#include "index/index_engine.h"
+#include "store/persist_store.h"
+#include "store/volatile_store.h"
#include <iostream>
#include <vector>
#include <cassert>
@@ -16,6 +18,23 @@ bool is_close(float a, float b, float epsilon = 1e-5) {
return std::fabs(a - b) < epsilon;
}
+void expect_filter_projection(IndexEngine& engine, const std::string& dsl,
+ uint64_t expected_count,
+ uint32_t expected_first_word) {
+ FilterResult result = engine.evaluate_filter(dsl);
+ const uint32_t first_word =
+ result.bitset_words.empty() ? 0U : result.bitset_words[0];
+ if (result.eligible_count != expected_count ||
+ result.bitset_words.size() != 1 || first_word != expected_first_word) {
+ SPDLOG_ERROR(
+ "Unexpected filter projection: dsl={}, count={} (expected {}), "
+ "words={}, first_word={} (expected {})",
+ dsl, result.eligible_count, expected_count, result.bitset_words.size(),
+ first_word, expected_first_word);
+ exit(1);
+ }
+}
+
void test_basic_workflow() {
SPDLOG_INFO("[Running] test_basic_workflow...");
@@ -108,6 +127,39 @@ void test_basic_workflow() {
exit(1);
}
+ FilterResult routed_filter_res =
+ engine.evaluate_filter_for_routing(uri_filter, 10);
+ if (routed_filter_res.eligible_count != 1 ||
+ !routed_filter_res.bitset_words.empty() ||
+ routed_filter_res.native_filter_token == 0) {
+ SPDLOG_ERROR("Sparse routed filter projection failed");
+ exit(1);
+ }
+
+ FilterResult routed_threshold_zero =
+ engine.evaluate_filter_for_routing(uri_filter, 0);
+ if (routed_threshold_zero.eligible_count != filter_res.eligible_count ||
+ routed_threshold_zero.bitset_words != filter_res.bitset_words ||
+ routed_threshold_zero.native_filter_token != 0) {
+ SPDLOG_ERROR("Threshold-zero routed filter projection changed semantics");
+ exit(1);
+ }
+
+ const std::string wide_uri_filter =
+ R"({"op":"must","field":"uri","conds":["/docs","/other"],"para":"-d=-1"})";
+ FilterResult generic_wide_filter = engine.evaluate_filter(wide_uri_filter);
+ FilterResult routed_wide_filter =
+ engine.evaluate_filter_for_routing(wide_uri_filter, 1);
+ if (generic_wide_filter.eligible_count != 2 ||
+ generic_wide_filter.bitset_words.size() != 1 ||
+ generic_wide_filter.bitset_words[0] != 0b101U ||
+ routed_wide_filter.eligible_count != generic_wide_filter.eligible_count ||
+ routed_wide_filter.bitset_words != generic_wide_filter.bitset_words ||
+ routed_wide_filter.native_filter_token != 0) {
+ SPDLOG_ERROR("Wide routed filter projection changed generic semantics");
+ exit(1);
+ }
+
FilterResult cached_filter_res = engine.evaluate_filter(uri_filter, 10);
if (cached_filter_res.native_filter_token == 0) {
SPDLOG_ERROR("Native filter token was not retained");
@@ -159,11 +211,383 @@ void test_basic_workflow() {
exit(1);
}
+ std::filesystem::remove_all(db_path);
SPDLOG_INFO("[Passed] test_basic_workflow");
}
+void test_routed_filter_projection_edge_cases() {
+ SPDLOG_INFO("[Running] test_routed_filter_projection_edge_cases...");
+
+ const std::string config = R"({
+ "CollectionName": "routed_filter_projection",
+ "IndexName": "default",
+ "VectorIndex": {
+ "IndexType": "flat",
+ "ElementCount": 0,
+ "MaxElementCount": 32,
+ "Dimension": 2,
+ "Distance": "l2",
+ "Quant": "float"
+ },
+ "ScalarIndex": [
+ {"FieldName": "uri", "FieldType": "path"}
+ ]
+ })";
+
+ IndexEngine engine(config);
+ AddDataRequest early;
+ early.label = 3001;
+ early.vector = {1.0, 0.0};
+ early.fields_str = R"({"uri":"/keep/a"})";
+ if (engine.add_data({early}) != 0) {
+ SPDLOG_ERROR("Failed to add the early routed-filter record");
+ exit(1);
+ }
+
+ std::vector<AddDataRequest> fillers;
+ std::vector<DeleteDataRequest> filler_deletes;
+ for (uint64_t i = 0; i < 16; ++i) {
+ AddDataRequest filler;
+ filler.label = 3100 + i;
+ filler.vector = {0.0, 1.0};
+ filler.fields_str =
+ "{\"uri\":\"/drop/" + std::to_string(i) + "\"}";
+ fillers.push_back(filler);
+
+ DeleteDataRequest deleted;
+ deleted.label = filler.label;
+ deleted.old_fields_str = filler.fields_str;
+ filler_deletes.push_back(deleted);
+ }
+ if (engine.add_data(fillers) != 0 ||
+ engine.delete_data(filler_deletes) != 0) {
+ SPDLOG_ERROR("Failed to create sparse logical offsets");
+ exit(1);
+ }
+
+ AddDataRequest late;
+ late.label = 3002;
+ late.vector = {0.9, 0.1};
+ late.fields_str = R"({"uri":"/keep/b"})";
+ if (engine.add_data({late}) != 0) {
+ SPDLOG_ERROR("Failed to add the late routed-filter record");
+ exit(1);
+ }
+
+ const std::string keep_recursive =
+ R"({"op":"must","field":"uri","conds":["/keep"],"para":"-d=-1"})";
+ const std::string keep_a =
+ R"({"op":"must","field":"uri","conds":["/keep/a"],"para":"-d=0"})";
+
+ // The surviving logical offsets are deliberately far apart. The inverse
+ // layout must decline an oversized span and preserve the fallback result.
+ engine.set_filter_layout({3001, 9999, 3002});
+ FilterResult generic = engine.evaluate_filter(keep_recursive);
+ FilterResult routed = engine.evaluate_filter_for_routing(keep_recursive, 2);
+ if (generic.eligible_count != 2 || generic.bitset_words.size() != 1 ||
+ generic.bitset_words[0] != 0b101U || routed.eligible_count != 2 ||
+ !routed.bitset_words.empty() || routed.native_filter_token == 0) {
+ SPDLOG_ERROR("Large-span routed filter projection was incorrect");
+ exit(1);
+ }
+
+ FilterResult threshold_zero =
+ engine.evaluate_filter_for_routing(keep_recursive, 0);
+ if (threshold_zero.eligible_count != generic.eligible_count ||
+ threshold_zero.bitset_words != generic.bitset_words ||
+ threshold_zero.native_filter_token != 0) {
+ SPDLOG_ERROR("Threshold-zero routing changed generic projection semantics");
+ exit(1);
+ }
+
+ // Duplicate logical offsets cannot use the one-to-one inverse. The fallback
+ // must count and project every external row exactly as the generic API does.
+ engine.set_filter_layout({3001, 3001, 3002, 9999});
+ generic = engine.evaluate_filter(keep_a);
+ routed = engine.evaluate_filter_for_routing(keep_a, 1);
+ if (generic.eligible_count != 2 || generic.bitset_words.size() != 1 ||
+ generic.bitset_words[0] != 0b0011U ||
+ routed.eligible_count != generic.eligible_count ||
+ routed.bitset_words != generic.bitset_words ||
+ routed.native_filter_token != 0) {
+ SPDLOG_ERROR("Duplicate-offset routed filter projection was incorrect");
+ exit(1);
+ }
+
+ FilterResult narrow_duplicate =
+ engine.evaluate_filter_for_routing(keep_a, 2);
+ if (narrow_duplicate.eligible_count != 2 ||
+ !narrow_duplicate.bitset_words.empty() ||
+ narrow_duplicate.native_filter_token == 0) {
+ SPDLOG_ERROR("Narrow duplicate-offset filter did not retain native state");
+ exit(1);
+ }
+
+ AddDataRequest update = early;
+ update.fields_str = R"({"uri":"/other/a"})";
+ update.old_fields_str = early.fields_str;
+ if (engine.add_data({update}) != 0) {
+ SPDLOG_ERROR("Failed to mutate routed-filter data");
+ exit(1);
+ }
+ SearchRequest query;
+ query.query = {1.0, 0.0};
+ query.topk = 2;
+ if (engine.search_with_filter_token(
+ query, narrow_duplicate.native_filter_token)) {
+ SPDLOG_ERROR("A routed filter token survived mutation");
+ exit(1);
+ }
+ routed = engine.evaluate_filter_for_routing(keep_recursive, 2);
+ if (routed.eligible_count != 0 || !routed.bitset_words.empty() ||
+ routed.native_filter_token != 0) {
+ SPDLOG_ERROR("Mutation did not invalidate the registered filter layout");
+ exit(1);
+ }
+
+ engine.set_filter_layout({3001, 3002});
+ routed = engine.evaluate_filter_for_routing(keep_recursive, 2);
+ if (routed.eligible_count != 1 || !routed.bitset_words.empty() ||
+ routed.native_filter_token == 0) {
+ SPDLOG_ERROR("Routed filter was incorrect after layout registration");
+ exit(1);
+ }
+
+ DeleteDataRequest delete_late;
+ delete_late.label = late.label;
+ delete_late.old_fields_str = late.fields_str;
+ const uint64_t pre_delete_token = routed.native_filter_token;
+ if (engine.delete_data({delete_late}) != 0) {
+ SPDLOG_ERROR("Failed to delete routed-filter data");
+ exit(1);
+ }
+ if (engine.search_with_filter_token(query, pre_delete_token)) {
+ SPDLOG_ERROR("A routed filter token survived deletion");
+ exit(1);
+ }
+ engine.set_filter_layout({3001, 3002});
+ routed = engine.evaluate_filter_for_routing(keep_recursive, 2);
+ if (routed.eligible_count != 0 || !routed.bitset_words.empty() ||
+ routed.native_filter_token != 0) {
+ SPDLOG_ERROR("Missing-label routed filter projection was incorrect");
+ exit(1);
+ }
+
+ SPDLOG_INFO("[Passed] test_routed_filter_projection_edge_cases");
+}
+
+void test_path_bitmap_lifecycle_and_reload() {
+ SPDLOG_INFO("[Running] test_path_bitmap_lifecycle_and_reload...");
+
+ const std::string db_path = "test_data_cpp/path_bitmap_lifecycle";
+ if (std::filesystem::exists(db_path)) {
+ std::filesystem::remove_all(db_path);
+ }
+ std::filesystem::create_directories(db_path);
+
+ const std::string config = R"({
+ "CollectionName": "path_bitmap_lifecycle",
+ "IndexName": "default",
+ "VectorIndex": {
+ "IndexType": "flat",
+ "ElementCount": 0,
+ "MaxElementCount": 6,
+ "Dimension": 4,
+ "Distance": "l2",
+ "Quant": "float"
+ },
+ "ScalarIndex": [
+ {"FieldName": "uri", "FieldType": "path"}
+ ]
+ })";
+
+ IndexEngine engine(config);
+ if (!engine.is_valid()) {
+ SPDLOG_ERROR("Path bitmap lifecycle engine initialization failed");
+ exit(1);
+ }
+
+ std::vector<AddDataRequest> initial(5);
+ initial[0].label = 2001;
+ initial[0].vector = {1.0, 0.0, 0.0, 0.0};
+ initial[0].fields_str = R"({"uri":"/docs/a"})";
+ initial[1].label = 2002;
+ initial[1].vector = {0.9, 0.1, 0.0, 0.0};
+ initial[1].fields_str = R"({"uri":"/docs/deep/b"})";
+ initial[2].label = 2003;
+ initial[2].vector = {0.0, 1.0, 0.0, 0.0};
+ initial[2].fields_str = R"({"uri":"/other/c"})";
+ // These spellings resolve to one trie leaf. Both bitmap bindings must be
+ // retained without adding a vector to every ordinary TrieNode.
+ initial[3].label = 2005;
+ initial[3].vector = {0.0, 0.0, 1.0, 0.0};
+ initial[3].fields_str = R"({"uri":"/aliases/a"})";
+ initial[4].label = 2006;
+ initial[4].vector = {0.0, 0.0, 0.9, 0.1};
+ initial[4].fields_str = R"({"uri":"/aliases//a"})";
+ if (engine.add_data(initial) != 0) {
+ SPDLOG_ERROR("Initial path bitmap data add failed");
+ exit(1);
+ }
+
+ const std::string docs_recursive =
+ R"({"op":"must","field":"uri","conds":["/docs"],"para":"-d=-1"})";
+ const std::string docs_depth_one =
+ R"({"op":"must","field":"uri","conds":["/docs"],"para":"-d=1"})";
+ const std::string docs_overlapping =
+ R"({"op":"must","field":"uri","conds":["/docs","/docs/deep"],"para":"-d=-1"})";
+ const std::string other_recursive =
+ R"({"op":"must","field":"uri","conds":["/other"],"para":"-d=-1"})";
+ const std::string aliases_recursive =
+ R"({"op":"must","field":"uri","conds":["/aliases"],"para":"-d=-1"})";
+
+ engine.set_filter_layout({2001, 2002, 2003});
+ expect_filter_projection(engine, docs_recursive, 2, 0b011U);
+ expect_filter_projection(engine, docs_depth_one, 1, 0b001U);
+ expect_filter_projection(engine, docs_overlapping, 2, 0b011U);
+
+ engine.set_filter_layout({2005, 2006});
+ expect_filter_projection(engine, aliases_recursive, 2, 0b11U);
+
+ engine.set_filter_layout({2001, 2002, 2003});
+ FilterResult cached_docs = engine.evaluate_filter(docs_recursive, 10);
+ if (cached_docs.native_filter_token == 0) {
+ SPDLOG_ERROR("Path filter token was not retained before mutation");
+ exit(1);
+ }
+
+ AddDataRequest update;
+ update.label = 2001;
+ update.vector = {1.0, 0.0, 0.0, 0.0};
+ update.fields_str = R"({"uri":"/other/a"})";
+ update.old_fields_str = R"({"uri":"/docs/a"})";
+ if (engine.add_data({update}) != 0) {
+ SPDLOG_ERROR("Path bitmap update failed");
+ exit(1);
+ }
+ SearchRequest token_query;
+ token_query.query = {1.0, 0.0, 0.0, 0.0};
+ token_query.topk = 10;
+ if (engine.search_with_filter_token(token_query,
+ cached_docs.native_filter_token)) {
+ SPDLOG_ERROR("Path filter token survived an upsert mutation");
+ exit(1);
+ }
+
+ engine.set_filter_layout({2001, 2002, 2003});
+ expect_filter_projection(engine, docs_recursive, 1, 0b010U);
+ expect_filter_projection(engine, other_recursive, 2, 0b101U);
+
+ AddDataRequest added;
+ added.label = 2004;
+ added.vector = {0.8, 0.2, 0.0, 0.0};
+ added.fields_str = R"({"uri":"/docs/new"})";
+ if (engine.add_data({added}) != 0) {
+ SPDLOG_ERROR("New descendant add failed");
+ exit(1);
+ }
+ engine.set_filter_layout({2001, 2002, 2003, 2004});
+ expect_filter_projection(engine, docs_recursive, 2, 0b1010U);
+
+ DeleteDataRequest deleted;
+ deleted.label = 2002;
+ deleted.old_fields_str = R"({"uri":"/docs/deep/b"})";
+ if (engine.delete_data({deleted}) != 0) {
+ SPDLOG_ERROR("Path descendant delete failed");
+ exit(1);
+ }
+ engine.set_filter_layout({2001, 2003, 2004});
+ expect_filter_projection(engine, docs_recursive, 1, 0b100U);
+ expect_filter_projection(engine, other_recursive, 2, 0b011U);
+
+ if (engine.dump(db_path) <= 0) {
+ SPDLOG_ERROR("Path bitmap lifecycle dump failed");
+ exit(1);
+ }
+
+ {
+ IndexEngine reloaded(db_path);
+ if (!reloaded.is_valid()) {
+ SPDLOG_ERROR("Reloaded path bitmap engine is invalid");
+ exit(1);
+ }
+ reloaded.set_filter_layout({2001, 2003, 2004});
+ expect_filter_projection(reloaded, docs_recursive, 1, 0b100U);
+ expect_filter_projection(reloaded, other_recursive, 2, 0b011U);
+ reloaded.set_filter_layout({2005, 2006});
+ expect_filter_projection(reloaded, aliases_recursive, 2, 0b11U);
+ }
+
+ std::filesystem::remove_all(db_path);
+ SPDLOG_INFO("[Passed] test_path_bitmap_lifecycle_and_reload");
+}
+
+void expect_paged_store_scan(KVStore& store) {
+ const std::vector<std::string> keys = {
+ "candidate:1", "candidate:10", "candidate:2", "other:1"};
+ const std::vector<std::string> values = {"one", "ten", "two", "other"};
+ if (store.put_data(keys, values) != 0) {
+ SPDLOG_ERROR("Paged store test data add failed");
+ exit(1);
+ }
+
+ const auto first =
+ store.seek_range_page("candidate:", "candidate;", 2, 1024, false);
+ if (first.size() != 2 || first[0].first != "candidate:1" ||
+ first[1].first != "candidate:10") {
+ SPDLOG_ERROR("Paged store first page was incorrect");
+ exit(1);
+ }
+ const auto second =
+ store.seek_range_page(first.back().first, "candidate;", 2, 1024, true);
+ if (second.size() != 1 || second[0].first != "candidate:2") {
+ SPDLOG_ERROR("Paged store exclusive continuation was incorrect");
+ exit(1);
+ }
+ const auto byte_limited =
+ store.seek_range_page("candidate:", "candidate;", 10, 14, false);
+ if (byte_limited.size() != 1 ||
+ byte_limited[0].first != "candidate:1") {
+ SPDLOG_ERROR("Paged store byte budget was not enforced");
+ exit(1);
+ }
+ const auto oversized_first =
+ store.seek_range_page("candidate:", "candidate;", 10, 1, false);
+ if (oversized_first.size() != 1 ||
+ oversized_first[0].first != "candidate:1") {
+ SPDLOG_ERROR("Paged store did not admit one oversized row for progress");
+ exit(1);
+ }
+ const auto full = store.seek_range("candidate:", "candidate;");
+ if (full.size() != 3) {
+ SPDLOG_ERROR("Existing full store scan behavior changed");
+ exit(1);
+ }
+}
+
+void test_paged_store_scan() {
+ SPDLOG_INFO("[Running] test_paged_store_scan...");
+ VolatileStore volatile_store;
+ expect_paged_store_scan(volatile_store);
+
+ const std::string db_path = "test_data_cpp/paged_persist_store";
+ if (std::filesystem::exists(db_path)) {
+ std::filesystem::remove_all(db_path);
+ }
+ std::filesystem::create_directories(db_path);
+ {
+ PersistStore persist_store(db_path);
+ expect_paged_store_scan(persist_store);
+ }
+ std::filesystem::remove_all(db_path);
+ SPDLOG_INFO("[Passed] test_paged_store_scan");
+}
+
int main() {
init_logging("INFO", "stdout", "[%Y-%m-%d %H:%M:%S.%e] [%l] %v");
test_basic_workflow();
+ test_routed_filter_projection_edge_cases();
+ test_path_bitmap_lifecycle_and_reload();
+ test_paged_store_scan();
return 0;
}
diff --git a/tests/storage/test_viking_vector_scope_filter.py b/tests/storage/test_viking_vector_scope_filter.py
new file mode 100644
index 000000000..a250068e5
--- /dev/null
+++ b/tests/storage/test_viking_vector_scope_filter.py
@@ -0,0 +1,257 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from openviking.core.namespace import uri_parts
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.expr import And, Eq, In, Or, PathScope
+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+from openviking_cli.session.user_id import UserIdentifier
+
+
+def _ctx(*, role: Role = Role.USER, actor_peer_id: str | None = None) -> RequestContext:
+ return RequestContext(
+ user=UserIdentifier("acct", "alice"),
+ role=role,
+ actor_peer_id=actor_peer_id,
+ )
+
+
+def _build(
+ ctx: RequestContext,
+ targets: list[str] | None,
+ *,
+ context_type: str | None = "resource",
+ extra_filter=None,
+ level: list[int] | None = None,
+):
+ backend = object.__new__(VikingVectorIndexBackend)
+ return backend._build_scope_filter(
+ ctx=ctx,
+ context_type=context_type,
+ target_directories=targets,
+ extra_filter=extra_filter,
+ level=level,
+ )
+
+
+def _tenant_filter(ctx: RequestContext):
+ return VikingVectorIndexBackend._tenant_filter(ctx, context_type="resource")
+
+
+def test_descendant_target_elides_only_visible_root_path_filter():
+ ctx = _ctx()
+ target = "viking://resources/wiki/physics"
+
+ result = _build(
+ ctx,
+ [target],
+ extra_filter=Eq("status", "ready"),
+ level=[2],
+ )
+
+ assert result == And(
+ [
+ Eq("context_type", "resource"),
+ Eq("account_id", "acct"),
+ Or([PathScope("uri", target, depth=-1)]),
+ Eq("status", "ready"),
+ In("level", [2]),
+ ]
+ )
+
+
+def test_equal_visible_root_elides_only_visible_root_path_filter():
+ ctx = _ctx()
+
+ result = _build(ctx, ["viking://resources"])
+
+ assert result == And(
+ [
+ Eq("context_type", "resource"),
+ Eq("account_id", "acct"),
+ Or([PathScope("uri", "viking://resources", depth=-1)]),
+ ]
+ )
+
+
+def test_all_targets_may_be_under_different_visible_roots():
+ ctx = _ctx()
+ targets = [
+ "viking://resources/wiki/physics",
+ "viking://user/resources/private-notes",
+ "viking://agent/skills/research",
+ ]
+
+ result = _build(ctx, targets)
+
+ assert result == And(
+ [
+ Eq("context_type", "resource"),
+ Eq("account_id", "acct"),
+ Or(
+ [
+ PathScope("uri", "viking://resources/wiki/physics", depth=-1),
+ PathScope("uri", "viking://user/alice/resources/private-notes", depth=-1),
+ PathScope("uri", "viking://agent/skills/research", depth=-1),
+ ]
+ ),
+ ]
+ )
+
+
+def test_mixed_visible_and_outside_targets_keep_original_tenant_filter():
+ ctx = _ctx()
+ targets = ["viking://resources/wiki", "viking://upload/staged"]
+
+ result = _build(ctx, targets)
+
+ assert result == And(
+ [
+ Eq("context_type", "resource"),
+ _tenant_filter(ctx),
+ Or([PathScope("uri", target, depth=-1) for target in targets]),
+ ]
+ )
+
+
+async def test_cross_user_targets_cannot_bypass_visible_roots_in_tenant_search():
+ ctx = _ctx()
+ own_uri = "viking://user/alice/resources/notes"
+ cross_user_uri = "viking://user/bob/resources/notes"
+ records = [
+ {
+ "id": "own",
+ "uri": own_uri,
+ "account_id": "acct",
+ "context_type": "resource",
+ },
+ {
+ "id": "cross-user",
+ "uri": cross_user_uri,
+ "account_id": "acct",
+ "context_type": "resource",
+ },
+ ]
+ observed_filters = []
+
+ def matches(expr, record):
+ if isinstance(expr, And):
+ return all(matches(cond, record) for cond in expr.conds)
+ if isinstance(expr, Or):
+ return any(matches(cond, record) for cond in expr.conds)
+ if isinstance(expr, Eq):
+ return record.get(expr.field) == expr.value
+ if isinstance(expr, PathScope):
+ root = uri_parts(expr.path)
+ path = uri_parts(str(record.get(expr.field, "")))
+ if path[: len(root)] != root:
+ return False
+ return expr.depth == -1 or len(path) - len(root) <= expr.depth
+ raise AssertionError(f"Unexpected filter expression in test: {expr!r}")
+
+ async def fake_search(*, filter, **_kwargs):
+ observed_filters.append(filter)
+ return [record for record in records if matches(filter, record)]
+
+ backend = object.__new__(VikingVectorIndexBackend)
+ backend.search = fake_search
+
+ cross_user_only = await backend.search_in_tenant(
+ ctx=ctx,
+ query_vector=[1.0],
+ context_type="resource",
+ target_directories=[cross_user_uri],
+ )
+ mixed = await backend.search_in_tenant(
+ ctx=ctx,
+ query_vector=[1.0],
+ context_type="resource",
+ target_directories=[own_uri, cross_user_uri],
+ )
+
+ assert cross_user_only == []
+ assert [record["id"] for record in mixed] == ["own"]
+ assert observed_filters == [
+ And(
+ [
+ Eq("context_type", "resource"),
+ _tenant_filter(ctx),
+ Or([PathScope("uri", cross_user_uri, depth=-1)]),
+ ]
+ ),
+ And(
+ [
+ Eq("context_type", "resource"),
+ _tenant_filter(ctx),
+ Or(
+ [
+ PathScope("uri", own_uri, depth=-1),
+ PathScope("uri", cross_user_uri, depth=-1),
+ ]
+ ),
+ ]
+ ),
+ ]
+
+
+def test_segment_prefix_and_visible_root_ancestor_do_not_elide_tenant_filter():
+ ctx = _ctx()
+
+ segment_prefix = _build(ctx, ["viking://resources-other/wiki"])
+ ancestor = _build(ctx, ["viking://agent"])
+
+ assert segment_prefix == And(
+ [
+ Eq("context_type", "resource"),
+ _tenant_filter(ctx),
+ Or([PathScope("uri", "viking://resources-other/wiki", depth=-1)]),
+ ]
+ )
+ assert ancestor == And(
+ [
+ Eq("context_type", "resource"),
+ _tenant_filter(ctx),
+ Or([PathScope("uri", "viking://agent", depth=-1)]),
+ ]
+ )
+
+
+def test_no_target_keeps_original_tenant_filter():
+ ctx = _ctx()
+
+ assert _build(ctx, None) == And(
+ [
+ Eq("context_type", "resource"),
+ _tenant_filter(ctx),
+ ]
+ )
+
+
+def test_root_role_keeps_existing_target_only_behavior():
+ ctx = _ctx(role=Role.ROOT)
+ target = "viking://resources/wiki"
+
+ assert _build(ctx, [target]) == And(
+ [
+ Eq("context_type", "resource"),
+ Or([PathScope("uri", target, depth=-1)]),
+ ]
+ )
+
+
+def test_actor_peer_target_retains_account_and_exact_target_scope():
+ ctx = _ctx(actor_peer_id="visitor-a")
+ target = "viking://user/alice/peers/visitor-a/resources/cases"
+
+ result = _build(ctx, [target])
+
+ assert result == And(
+ [
+ Eq("context_type", "resource"),
+ Eq("account_id", "acct"),
+ Or([PathScope("uri", target, depth=-1)]),
+ ]
+ )
diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.py
index b4ad6ad85..16d487983 100644
--- a/tests/test_telemetry_runtime.py
+++ b/tests/test_telemetry_runtime.py
@@ -300,6 +300,8 @@ def test_cuvs_telemetry_aggregation_is_completion_order_independent():
"auto_mode": False,
"route_reason": "cuvs",
"filter_kind": "none",
+ "filter_cache_eviction_fallback": True,
+ "filter_words_packed": True,
"build_performed": True,
"records_generation": 2,
"index_size": 100,
@@ -308,6 +310,7 @@ def test_cuvs_telemetry_aggregation_is_completion_order_independent():
"memory_usable_bytes": 7000,
"total_ms": 12,
"queue_ms": 2,
+ "gpu_gate_queue_ms": 1.5,
"build_ms": 5,
"gpu_search_ms": 5,
},
@@ -343,8 +346,11 @@ def test_cuvs_telemetry_aggregation_is_completion_order_independent():
assert forward == reverse
assert forward["routes"] == {"cuvs": 1, "native_filter_threshold": 1}
assert forward["builds"] == 1
+ assert forward["filter_cache_eviction_fallbacks"] == 1
+ assert forward["packed_filter_queries"] == 1
assert forward["memory"]["free_bytes_min"] == 6000
assert forward["timings_ms"]["total"] == {"sum": 15.0, "max": 12.0}
+ assert forward["timings_ms"]["gpu_gate_queue"] == {"sum": 1.5, "max": 1.5}
def test_cuvs_telemetry_timing_sum_is_strictly_order_independent():
diff --git a/tests/vectordb/test_cuvs_collection.py b/tests/vectordb/test_cuvs_collection.py
index 9b6536fd4..7edfb4cba 100644
--- a/tests/vectordb/test_cuvs_collection.py
+++ b/tests/vectordb/test_cuvs_collection.py
@@ -1,14 +1,22 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+import gc
import threading
import traceback
+import weakref
from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import pytest
from openviking.storage.vectordb.collection.local_collection import (
get_or_create_local_collection,
)
from openviking.storage.vectordb.index import cuvs_index
+from openviking.storage.vectordb.store.data import CandidateData
+from openviking.storage.vectordb.store.local_store import StoreEngineProxy
+from openviking.storage.vectordb.store.store_manager import StoreManager
from openviking.telemetry.backends.memory import MemoryOperationTelemetry
from openviking.telemetry.context import bind_telemetry
@@ -356,6 +364,243 @@ def test_persistent_collection_rehydrates_cuvs_from_local_store(monkeypatch, tmp
reopened.close()
+def test_persistent_cuvs_recovery_deserializes_candidates_lazily(monkeypatch, tmp_path):
+ patch_cuvs_runtime(monkeypatch)
+ path = str(tmp_path / "cuvs-streaming-recovery")
+ config = {"dense_search": {"backend": "cuvs", "algorithm": "brute_force"}}
+ collection = get_or_create_local_collection(
+ meta_data={
+ "CollectionName": "cuvs_streaming_recovery",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},
+ ],
+ },
+ path=path,
+ config=config,
+ )
+ for index_name in ("first", "second"):
+ collection.create_index(
+ index_name,
+ {
+ "IndexName": index_name,
+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},
+ },
+ )
+ record_count = 64
+ collection.upsert_data(
+ [
+ {
+ "id": f"row-{offset}",
+ "vector": [float(offset + 1), 1.0, 0.0, 0.0],
+ }
+ for offset in range(record_count)
+ ]
+ )
+ collection.close()
+
+ original_from_bytes = CandidateData.from_bytes
+ original_pack_vector = cuvs_index.CuVSDenseIndex._pack_vector
+ candidate_refs = []
+ events = []
+ peak_live_candidates = 0
+
+ def tracked_from_bytes(data):
+ nonlocal peak_live_candidates
+ candidate = original_from_bytes(data)
+ candidate_refs.append(weakref.ref(candidate))
+ events.append("decode")
+ peak_live_candidates = max(
+ peak_live_candidates,
+ sum(candidate_ref() is not None for candidate_ref in candidate_refs),
+ )
+ return candidate
+
+ def tracked_pack_vector(self, vector):
+ events.append("pack")
+ return original_pack_vector(self, vector)
+
+ monkeypatch.setattr(CandidateData, "from_bytes", staticmethod(tracked_from_bytes))
+ monkeypatch.setattr(cuvs_index.CuVSDenseIndex, "_pack_vector", tracked_pack_vector)
+
+ reopened = get_or_create_local_collection(path=path, config=config)
+ try:
+ for index_name in ("first", "second"):
+ index = reopened.get_index(index_name)
+ assert index is not None
+ assert index.dense_search is not None
+ assert index.dense_search.size == record_count
+
+ assert events.count("decode") == record_count * 2
+ assert events.count("pack") == record_count * 2
+ assert events.index("pack") < len(events) - 1 - events[::-1].index("decode")
+ assert peak_live_candidates <= 3
+ gc.collect()
+ assert not any(candidate_ref() is not None for candidate_ref in candidate_refs)
+ finally:
+ reopened.close()
+
+
+def test_persistent_native_recovery_does_not_scan_candidates(monkeypatch, tmp_path):
+ path = str(tmp_path / "native-recovery-no-candidate-scan")
+ collection = get_or_create_local_collection(
+ meta_data={
+ "CollectionName": "native_recovery_no_candidate_scan",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},
+ ],
+ },
+ path=path,
+ )
+ collection.create_index(
+ "default",
+ {
+ "IndexName": "default",
+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},
+ },
+ )
+ collection.upsert_data([{"id": "persisted", "vector": [1.0, 0.0, 0.0, 0.0]}])
+ collection.close()
+
+ original_read_all = StoreEngineProxy.read_all
+
+ def reject_candidate_scan(self, table_name):
+ if table_name == StoreManager.CandsTable:
+ raise AssertionError("native snapshot recovery must not scan candidate rows")
+ return original_read_all(self, table_name)
+
+ monkeypatch.setattr(StoreEngineProxy, "read_all", reject_candidate_scan)
+ reopened = get_or_create_local_collection(path=path)
+ try:
+ result = reopened.search_by_vector("default", dense_vector=[1.0, 0.0, 0.0, 0.0], limit=1)
+ assert [item.id for item in result.data] == ["persisted"]
+ finally:
+ reopened.close()
+
+
+def test_persistent_cuvs_recovery_materializes_when_native_snapshot_is_missing(
+ monkeypatch, tmp_path
+):
+ patch_cuvs_runtime(monkeypatch)
+ path = str(tmp_path / "cuvs-missing-native-snapshot")
+ config = {"dense_search": {"backend": "cuvs", "algorithm": "brute_force"}}
+ collection = get_or_create_local_collection(
+ meta_data={
+ "CollectionName": "cuvs_missing_native_snapshot",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},
+ ],
+ },
+ path=path,
+ config=config,
+ )
+ collection.create_index(
+ "default",
+ {
+ "IndexName": "default",
+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},
+ },
+ )
+ collection.upsert_data([{"id": "persisted", "vector": [1.0, 0.0, 0.0, 0.0]}])
+ index = collection.get_index("default")
+ assert index is not None
+ version_dir = Path(index.version_dir)
+ collection.close()
+
+ markers = list(version_dir.glob("*.write_done"))
+ assert markers
+ for marker in markers:
+ marker.unlink()
+
+ reopened = get_or_create_local_collection(path=path, config=config)
+ try:
+ result = reopened.search_by_vector("default", dense_vector=[1.0, 0.0, 0.0, 0.0], limit=1)
+ assert [item.id for item in result.data] == ["persisted"]
+ recovered_index = reopened.get_index("default")
+ assert recovered_index is not None
+ assert recovered_index.dense_search is not None
+ assert recovered_index.dense_search.size == 1
+ finally:
+ reopened.close()
+
+
+def test_failed_cuvs_recovery_closes_candidate_iterator_and_partial_shadow(monkeypatch, tmp_path):
+ patch_cuvs_runtime(monkeypatch)
+ path = str(tmp_path / "cuvs-failed-streaming-recovery")
+ config = {"dense_search": {"backend": "cuvs", "algorithm": "brute_force"}}
+ collection = get_or_create_local_collection(
+ meta_data={
+ "CollectionName": "cuvs_failed_streaming_recovery",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},
+ ],
+ },
+ path=path,
+ config=config,
+ )
+ collection.create_index(
+ "default",
+ {
+ "IndexName": "default",
+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},
+ },
+ )
+ collection.upsert_data([{"id": "persisted", "vector": [1.0, 0.0, 0.0, 0.0]}])
+ collection.close()
+
+ iterator_closed = False
+ original_iter_all_cands_data = StoreManager.iter_all_cands_data
+
+ def broken_candidates(_self):
+ nonlocal iterator_closed
+ try:
+ yield CandidateData(label=1, vector=[1.0, 0.0, 0.0, 0.0])
+ yield CandidateData(label=2, vector=[1.0, 0.0])
+ finally:
+ iterator_closed = True
+
+ class TrackingRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__("inner_product")
+ self.closed = False
+
+ def close(self):
+ self.closed = True
+
+ runtime = TrackingRuntime()
+ monkeypatch.setattr(StoreManager, "iter_all_cands_data", broken_candidates)
+ monkeypatch.setattr(
+ cuvs_index,
+ "_CuVSRuntime",
+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,
+ )
+
+ with pytest.raises(ValueError, match="cuVS vector dimension mismatch"):
+ get_or_create_local_collection(path=path, config=config)
+
+ assert iterator_closed
+ assert runtime.closed
+
+ # Release must be complete when the constructor raises: restoring a valid
+ # stream and runtime should allow this same persistent collection to reopen
+ # immediately, without waiting for traceback GC to drop native locks.
+ monkeypatch.setattr(StoreManager, "iter_all_cands_data", original_iter_all_cands_data)
+ patch_cuvs_runtime(monkeypatch)
+ reopened = get_or_create_local_collection(path=path, config=config)
+ try:
+ result = reopened.search_by_vector(
+ "default",
+ dense_vector=[1.0, 0.0, 0.0, 0.0],
+ limit=1,
+ )
+ assert [item.id for item in result.data] == ["persisted"]
+ finally:
+ reopened.close()
+
+
def test_auto_cuvs_falls_back_then_retries_when_memory_is_available(monkeypatch):
runtimes = []
diff --git a/tests/vectordb/test_cuvs_filter_packed.py b/tests/vectordb/test_cuvs_filter_packed.py
new file mode 100644
index 000000000..5e8119fac
--- /dev/null
+++ b/tests/vectordb/test_cuvs_filter_packed.py
@@ -0,0 +1,244 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import gc
+import json
+
+import numpy as np
+import pytest
+
+from openviking.storage.vectordb.index.cuvs_index import (
+ CuVSDenseIndex,
+ CuVSSearchTelemetry,
+ _CuVSRuntime,
+)
+from openviking.storage.vectordb.store.data import CandidateData
+
+
+class _HostOnlyRuntime:
+ pass
+
+
+def _candidate(label: int) -> CandidateData:
+ return CandidateData(label=label, vector=[float(label)], fields=json.dumps({"group": "a"}))
+
+
+def _index(row_count: int) -> CuVSDenseIndex:
+ index = CuVSDenseIndex(
+ dimension=1,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"group": "string"},
+ config={},
+ runtime=_HostOnlyRuntime(),
+ )
+ index.add_candidates(_candidate(label) for label in range(1, row_count + 1))
+ with index._lock:
+ index._filter_layout_generation = index._records_generation
+ return index
+
+
+def test_packed_filter_rejects_malformed_byte_length():
+ index = _index(1)
+
+ with pytest.raises(ValueError, match="multiple of 4 bytes"):
+ index._resolve_native_filter({}, lambda _filters: (b"\x01", 1))
+
+
+def test_packed_filter_completeness_uses_word_count_not_byte_count():
+ index = _index(33)
+
+ with pytest.raises(RuntimeError, match=r"got 1 words.*expected at least 2"):
+ index._resolve_native_filter({}, lambda _filters: (b"\x01\x00\x00\x00", 1))
+
+
+def test_packed_filter_preserves_legacy_host_mask_fallback_and_telemetry():
+ index = _index(2)
+ filters = {"op": "must", "field": "group", "conds": ["a"]}
+ telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)
+ packed = b"\x02\x00\x00\x00"
+
+ host_filter = index._prepare_host_filter(
+ filters,
+ lambda _filters: (packed, 1),
+ lambda _labels: None,
+ telemetry,
+ )
+ assert host_filter is not None
+ resolved = host_filter.resolved_native_filter
+ assert resolved is not None
+ prepared = index._prepare_filter(
+ filters,
+ [1, 2],
+ lambda _filters: (packed, 1),
+ resolved,
+ )
+
+ assert resolved.bitset_words is packed
+ assert prepared.prepared == (False, True)
+ assert prepared.eligible_count == 1
+ assert telemetry.as_dict()["filter_words_packed"] is True
+
+ cache_hit_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)
+ cached_host_filter = index._prepare_host_filter(
+ filters,
+ lambda _filters: (_ for _ in ()).throw(AssertionError("resolver should not run")),
+ lambda _labels: None,
+ cache_hit_telemetry,
+ )
+ assert cached_host_filter is not None
+ assert cached_host_filter.cached_metadata is not None
+ assert cache_hit_telemetry.filter_cache_hit is True
+ assert cache_hit_telemetry.filter_words_packed is True
+
+
+def test_legacy_filter_words_keep_packed_origin_false_across_cache_hits():
+ index = _index(2)
+ filters = {"op": "must", "field": "group", "conds": ["a"]}
+ first_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)
+
+ first_host_filter = index._prepare_host_filter(
+ filters,
+ lambda _filters: ([0b01], 1),
+ lambda _labels: None,
+ first_telemetry,
+ )
+ assert first_host_filter is not None
+ resolved = first_host_filter.resolved_native_filter
+ assert resolved is not None
+ index._prepare_filter(filters, [1, 2], lambda _filters: ([0b01], 1), resolved)
+
+ cache_hit_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)
+ index._prepare_host_filter(
+ filters,
+ lambda _filters: (_ for _ in ()).throw(AssertionError("resolver should not run")),
+ lambda _labels: None,
+ cache_hit_telemetry,
+ )
+
+ assert first_telemetry.filter_words_packed is False
+ assert cache_hit_telemetry.filter_cache_hit is True
+ assert cache_hit_telemetry.filter_words_packed is False
+
+
+def test_in_gate_resolver_fallback_retains_packed_origin_and_native_token():
+ index = _index(2)
+ filters = {"op": "must", "field": "group", "conds": ["a"]}
+
+ prepared = index._prepare_filter(
+ filters,
+ [1, 2],
+ lambda _filters: (b"\x01\x00\x00\x00", 1, 23),
+ )
+
+ assert prepared.filter_words_packed is True
+ assert prepared.native_filter_token == 23
+ assert prepared.prepared == (True, False)
+
+
+def test_runtime_uses_numpy_frombuffer_for_packed_filter_words():
+ class _Device:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, _exc_type, _exc, _traceback):
+ return False
+
+ class _Cuda:
+ @staticmethod
+ def Device(_device_id):
+ return _Device()
+
+ class _CuPy:
+ uint32 = object()
+ cuda = _Cuda()
+
+ def __init__(self):
+ self.host_words = None
+ self.dtype = None
+
+ def empty(self, shape, dtype):
+ assert shape == (2,)
+ self.dtype = dtype
+ owner = self
+
+ class _DeviceWords:
+ def set(self, values):
+ owner.host_words = values
+
+ return _DeviceWords()
+
+ runtime = _CuVSRuntime.__new__(_CuVSRuntime)
+ runtime.cp = _CuPy()
+ runtime.device_id = 0
+ packed = b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa"
+
+ result = runtime.prepare_filter_words(packed)
+
+ assert result is not None
+ assert isinstance(runtime.cp.host_words, np.ndarray)
+ assert runtime.cp.host_words.dtype.str == "<u4"
+ assert runtime.cp.host_words.tolist() == [0x01020304, 0xAABBCCDD]
+ assert runtime.cp.host_words.base is packed
+ assert runtime.cp.dtype is runtime.cp.uint32
+
+
+def test_runtime_rejects_malformed_packed_words_before_cupy_conversion():
+ runtime = _CuVSRuntime.__new__(_CuVSRuntime)
+ runtime.cp = object()
+ runtime.device_id = 0
+
+ with pytest.raises(ValueError, match="multiple of 4 bytes"):
+ # Validation happens before touching CUDA, so malformed native output
+ # cannot enter the device-copy path.
+ runtime.prepare_filter_words(b"\x00")
+
+
[email protected]("failure_type", [RuntimeError, KeyboardInterrupt])
+def test_failed_packed_copy_releases_device_words_in_captured_scope(failure_type):
+ class _State:
+ current_device = 9
+ releases = []
+
+ class _Device:
+ def __init__(self, device_id):
+ self.device_id = device_id
+
+ def __enter__(self):
+ self.previous = _State.current_device
+ _State.current_device = self.device_id
+ return self
+
+ def __exit__(self, _exc_type, _exc, _traceback):
+ _State.current_device = self.previous
+ return False
+
+ class _DeviceWords:
+ def set(self, _values):
+ raise failure_type("injected packed copy failure")
+
+ def __del__(self):
+ _State.releases.append(_State.current_device)
+
+ class _Cuda:
+ Device = _Device
+
+ class _CuPy:
+ uint32 = object()
+ cuda = _Cuda()
+
+ @staticmethod
+ def empty(_shape, dtype):
+ assert dtype is _CuPy.uint32
+ return _DeviceWords()
+
+ runtime = _CuVSRuntime.__new__(_CuVSRuntime)
+ runtime.cp = _CuPy()
+ runtime.device_id = 3
+
+ with pytest.raises(failure_type, match="injected packed copy failure"):
+ runtime.prepare_filter_words(b"\x01\x00\x00\x00")
+ gc.collect()
+
+ assert _State.releases == [3]
+ assert _State.current_device == 9
diff --git a/tests/vectordb/test_cuvs_index.py b/tests/vectordb/test_cuvs_index.py
index a67930fa1..5ded0d3e9 100644
--- a/tests/vectordb/test_cuvs_index.py
+++ b/tests/vectordb/test_cuvs_index.py
@@ -3,10 +3,13 @@
import threading
import time
+from array import array
from concurrent.futures import ThreadPoolExecutor
+from contextlib import nullcontext
import pytest
+from openviking.storage.vectordb.index import cuvs_index as cuvs_index_module
from openviking.storage.vectordb.index.cuvs_index import (
CuVSDenseIndex,
CuVSMemoryBudgetError,
@@ -15,6 +18,7 @@ from openviking.storage.vectordb.index.cuvs_index import (
CuVSUnavailableError,
UnsupportedCuVSFilterError,
_CuVSRuntime,
+ _PackedFP32Rows,
estimate_cuvs_memory,
matches_filter,
)
@@ -33,6 +37,8 @@ class FakeCuVSRuntime:
self.free_memory_bytes = 1 << 60
self.total_memory_bytes = 1 << 60
self.release_count = 0
+ self.close_count = 0
+ self.close_error = None
def build(self, dataset):
self.dataset = [list(vector) for vector in dataset]
@@ -73,6 +79,11 @@ class FakeCuVSRuntime:
return False
def close(self):
+ self.close_count += 1
+ if self.close_error is not None:
+ error = self.close_error
+ self.close_error = None
+ raise error
self.closed = True
@@ -92,6 +103,7 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():
self.resource_count = 0
self.fail_build = False
self.fail_search = False
+ self.fail_host_copy = False
def set_current(self, device_id):
self.local.current = device_id
@@ -181,6 +193,8 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():
@staticmethod
def asnumpy(values):
spy.require_captured_device("asnumpy")
+ if spy.fail_host_copy:
+ raise RuntimeError("injected host copy failure")
return values
@staticmethod
@@ -247,8 +261,10 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():
runtime._owned_resources = []
runtime._resource_limit = 1
runtime._resources_closed = False
- runtime._use_explicit_resources = False
- runtime.set_max_concurrent_searches(2)
+ # The default serialized setting must still use an explicit synchronized
+ # resource. This is the production shape used when benchmark callers
+ # churn worker threads while GPU admission remains at one search.
+ runtime.set_max_concurrent_searches(1)
constructing_thread = threading.get_ident()
def use_runtime_from_worker():
@@ -293,6 +309,11 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():
assert spy.current() == 9
assert spy.releases == [("resources-1", 3)]
assert runtime.search(runtime_index, [1.0, 0.0], 1, [True]) == ([0], [0.25])
+ spy.fail_host_copy = True
+ with pytest.raises(RuntimeError, match="injected host copy failure"):
+ runtime.search(runtime_index, [1.0, 0.0], 1, [True])
+ spy.fail_host_copy = False
+ assert spy.releases == [("resources-1", 3), ("resources-2", 3)]
runtime.release_index()
assert spy.current() == 9
return runtime_index
@@ -317,18 +338,23 @@ def test_cuvs_runtime_uses_captured_device_from_worker_thread():
assert not thread.is_alive()
assert churn_errors == []
- assert spy.resource_count == 2
- assert len(runtime._owned_resources) <= 2
+ assert spy.resource_count == 3
+ assert len(runtime._owned_resources) <= 1
assert len(runtime._available_resources) == 1
- # A failed search discarded the first resource under device 3. The bounded
- # pool reused the replacement across short-lived threads and retains it
- # until close can also destroy it on the captured device.
- assert spy.releases == [("resources-1", 3)]
+ # Search and post-sync host-copy failures discarded the first two resources
+ # under device 3. The bounded pool reuses the replacement across
+ # short-lived threads and retains it until close can also destroy it on the
+ # captured device.
+ assert spy.releases == [("resources-1", 3), ("resources-2", 3)]
spy.set_current(9)
runtime.close()
assert spy.current() == 9
- assert spy.releases == [("resources-1", 3), ("resources-2", 3)]
+ assert spy.releases == [
+ ("resources-1", 3),
+ ("resources-2", 3),
+ ("resources-3", 3),
+ ]
assert {
"memory_info",
@@ -356,6 +382,170 @@ def delta(label, vector, **fields):
return DeltaRecord(label=label, vector=vector, fields=json.dumps(fields))
+def wait_for_preflight_participants(index, expected, timeout=5.0):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ with index._lock:
+ if any(flight.participants >= expected for flight in index._preflight_flights.values()):
+ return
+ time.sleep(0.001)
+ raise AssertionError(f"Timed out waiting for {expected} preflight participants")
+
+
+class _PreflightAbort(BaseException):
+ """Model an asynchronous owner abort that ordinary Exception misses."""
+
+
+def test_cuvs_host_shadow_is_immutable_compact_fp32():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={},
+ config={"dtype": "float16"},
+ runtime=runtime,
+ )
+ source = [0.1, -3.25]
+ index.add_candidates([candidate(10, source), candidate(20, [2.0, 4.0])])
+
+ assert index.host_shadow_nbytes == 2 * 2 * 4
+ stored = index._records[10].vector
+ assert isinstance(stored, bytes)
+ assert len(stored) == 2 * 4
+ assert list(memoryview(stored).cast("f")) == pytest.approx([0.1, -3.25])
+
+ # The host snapshot owns FP32 values instead of references to caller-owned
+ # Python floats, even when the configured device dataset uses FP16.
+ source[:] = [9.0, 9.0]
+ rebuild = index.prepare_rebuild()
+ assert rebuild is not None
+ assert rebuild.labels == (10, 20)
+ assert runtime.dataset[0] == pytest.approx([0.1, -3.25])
+ assert index.commit_rebuild(rebuild)
+
+ # Updating an existing label keeps dict/row order; delete plus reinsert
+ # retains the previous append-at-end behavior.
+ index.upsert([delta(10, [5.0, 6.0])])
+ replacement = index.prepare_rebuild()
+ assert replacement is not None
+ assert replacement.labels == (10, 20)
+ assert runtime.dataset[0] == pytest.approx([5.0, 6.0])
+ assert index.commit_rebuild(replacement)
+ index.delete([DeltaRecord(label=10)])
+ index.upsert([delta(10, [7.0, 8.0])])
+ reinserted = index.prepare_rebuild()
+ assert reinserted is not None
+ assert reinserted.labels == (20, 10)
+ assert runtime.dataset[1] == pytest.approx([7.0, 8.0])
+
+ index.close()
+ assert index.size == 0
+ assert index.host_shadow_nbytes == 0
+ assert runtime.closed
+
+
+ ("dtype", "expected_upload_dtype"),
+ [("float32", "float32"), ("float16", "float16")],
+)
+def test_cuvs_runtime_uploads_packed_rows_in_bounded_batches(
+ monkeypatch, dtype, expected_upload_dtype
+):
+ class DeviceDataset:
+ def __init__(self, shape, device_dtype):
+ self.rows = [[None] * shape[1] for _ in range(shape[0])]
+ self.dtype = device_dtype
+ self.upload_dtypes = []
+
+ def __getitem__(self, row_slice):
+ owner = self
+
+ class DeviceSlice:
+ def set(self, host_rows):
+ owner.upload_dtypes.append(str(host_rows.dtype))
+ owner.rows[row_slice] = host_rows.tolist()
+
+ return DeviceSlice()
+
+ class FakeCuPy:
+ float16 = "float16"
+ float32 = "float32"
+
+ @staticmethod
+ def empty(shape, dtype):
+ return DeviceDataset(shape, dtype)
+
+ class FakeBruteForce:
+ @staticmethod
+ def build(dataset, metric):
+ assert metric == "inner_product"
+ return dataset
+
+ runtime = _CuVSRuntime.__new__(_CuVSRuntime)
+ runtime.cp = FakeCuPy()
+ runtime.device_scope = lambda: nullcontext()
+ runtime.device_dtype = FakeCuPy.float16 if dtype == "float16" else FakeCuPy.float32
+ runtime.dtype = dtype
+ runtime.algorithm = "brute_force"
+ runtime.metric = "inner_product"
+ runtime.brute_force = FakeBruteForce()
+ runtime.cagra = object()
+ runtime.build_params = {}
+
+ rows = _PackedFP32Rows(
+ tuple(array("f", values).tobytes() for values in ([1, 2], [3, 4], [5, 6])),
+ dimension=2,
+ )
+ monkeypatch.setattr(cuvs_index_module, "_FP32_UPLOAD_BATCH_BYTES", 8)
+ built = runtime.build(rows)
+
+ assert built.dataset.rows == [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]
+ assert built.dataset.upload_dtypes == [expected_upload_dtype] * 3
+ assert rows.nbytes == 3 * 2 * 4
+ assert [end - start for start, end, _payload in rows.iter_packed_batches(8)] == [1, 1, 1]
+
+
+def test_mutation_during_packed_rebuild_keeps_candidate_snapshot_immutable():
+ class BlockingRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__()
+ self.first_build_started = threading.Event()
+ self.resume_first_build = threading.Event()
+
+ def build(self, dataset):
+ if self.build_count == 0:
+ self.first_build_started.set()
+ assert self.resume_first_build.wait(timeout=5)
+ return super().build(dataset)
+
+ runtime = BlockingRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={},
+ config={},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0])])
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ pending = executor.submit(index.prepare_rebuild)
+ assert runtime.first_build_started.wait(timeout=5)
+ index.upsert([delta(1, [0.0, 1.0])])
+ runtime.resume_first_build.set()
+ stale = pending.result(timeout=5)
+
+ assert stale is not None
+ assert runtime.dataset == [[1.0, 0.0]]
+ assert index.commit_rebuild(stale) is False
+ replacement = index.prepare_rebuild()
+ assert replacement is not None
+ assert runtime.dataset == [[0.0, 1.0]]
+ assert index.commit_rebuild(replacement)
+
+
def test_cuvs_rebuild_objects_are_released_on_the_captured_device():
class DeviceOwned:
def __init__(self, runtime, name):
@@ -432,6 +622,84 @@ def test_cuvs_rebuild_objects_are_released_on_the_captured_device():
assert runtime.current_device == 9
+def test_prepared_filter_lru_and_close_release_on_captured_device():
+ class DeviceOwnedFilter:
+ def __init__(self, runtime, name, values):
+ self.runtime = runtime
+ self.name = name
+ self.values = values
+
+ def __getitem__(self, index):
+ return self.values[index]
+
+ def __del__(self):
+ self.runtime.released.append((self.name, self.runtime.current_device))
+
+ class LifecycleRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__()
+ self.current_device = 9
+ self.prepared_count = 0
+ self.released = []
+
+ def device_scope(self):
+ runtime = self
+
+ class DeviceContext:
+ def __enter__(self):
+ self.previous = runtime.current_device
+ runtime.current_device = 3
+
+ def __exit__(self, _exc_type, _exc, _traceback):
+ runtime.current_device = self.previous
+
+ return DeviceContext()
+
+ def prepare_filter_words(self, words):
+ with self.device_scope():
+ self.prepared_count += 1
+ values = tuple(bool(words[0] & (1 << row)) for row in range(2))
+ return DeviceOwnedFilter(self, f"filter-{self.prepared_count}", values)
+
+ def close(self):
+ assert self.current_device == 3
+ super().close()
+
+ runtime = LifecycleRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"filter_cache_size": 1},
+ runtime=runtime,
+ )
+ index.add_candidates(
+ [
+ candidate(1, [1.0, 0.0], account_id="a"),
+ candidate(2, [0.0, 1.0], account_id="b"),
+ ]
+ )
+
+ def resolve(filters):
+ return ([0b01], 1) if filters["conds"][0] == "a" else ([0b10], 1)
+
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ filter_b = {"op": "must", "field": "account_id", "conds": ["b"]}
+
+ def registrar(_labels):
+ return None
+
+ assert index.search([1.0, 0.0], 1, filter_a, resolve, registrar)[0] == [1]
+ assert runtime.released == []
+ assert index.search([0.0, 1.0], 1, filter_b, resolve, registrar)[0] == [2]
+ assert runtime.released == [("filter-1", 3)]
+
+ index.close()
+ assert runtime.released == [("filter-1", 3), ("filter-2", 3)]
+ assert runtime.current_device == 9
+
+
def test_inflight_snapshot_is_released_on_captured_device_after_replacement():
class DeviceOwned:
def __init__(self, runtime, name):
@@ -736,28 +1004,385 @@ def test_cuvs_gpu_search_gate_serializes_kernels_by_default():
with self.active_lock:
self.active -= 1
- runtime = SerialRuntime()
+ runtime = SerialRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={},
+ config={},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0])])
+ assert index.search([1.0, 0.0], 1, None)[0] == [1]
+
+ telemetry = [CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False) for _ in range(4)]
+ with ThreadPoolExecutor(max_workers=4) as executor:
+ futures = [
+ executor.submit(index.search, [1.0, 0.0], 1, None, None, None, item)
+ for item in telemetry
+ ]
+ assert [future.result(timeout=5)[0] for future in futures] == [[1]] * 4
+
+ assert runtime.peak_active == 1
+ assert max(item.queue_ms for item in telemetry) > 0
+ assert max(item.gpu_gate_queue_ms for item in telemetry) > 0
+ assert all(item.queue_ms >= item.gpu_gate_queue_ms for item in telemetry)
+
+
+def test_native_filter_resolution_runs_before_gpu_search_admission():
+ class BlockingRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__()
+ self.block_next = False
+ self.search_started = threading.Event()
+ self.resume_search = threading.Event()
+
+ def search(self, index, query, limit, mask):
+ if self.block_next:
+ self.block_next = False
+ self.search_started.set()
+ assert self.resume_search.wait(timeout=5)
+ return super().search(index, query, limit, mask)
+
+ runtime = BlockingRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])
+ assert index.search([1.0, 0.0], 1, None)[0] == [1]
+
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ resolver_ran = threading.Event()
+ telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)
+
+ def resolve(_filters):
+ resolver_ran.set()
+ return [0b1], 1
+
+ runtime.block_next = True
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ occupying = executor.submit(index.search, [1.0, 0.0], 1, None)
+ assert runtime.search_started.wait(timeout=5)
+ filtered = executor.submit(
+ index.search,
+ [1.0, 0.0],
+ 1,
+ filter_a,
+ resolve,
+ lambda _labels: None,
+ telemetry,
+ )
+ # The resolver is host work and must complete while the sole GPU
+ # admission permit is still occupied by the first search.
+ assert resolver_ran.wait(timeout=5)
+ runtime.resume_search.set()
+ assert occupying.result(timeout=5)[0] == [1]
+ assert filtered.result(timeout=5)[0] == [1]
+
+ assert telemetry.gpu_gate_queue_ms > 0
+ assert telemetry.queue_ms >= telemetry.gpu_gate_queue_ms
+
+
+def test_explicit_same_filter_host_resolution_is_singleflight():
+ class WordPreparingRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__()
+ self.prepare_filter_words_count = 0
+
+ def prepare_filter_words(self, words):
+ self.prepare_filter_words_count += 1
+ return (bool(words[0] & 1),)
+
+ runtime = WordPreparingRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"filter_cache_size": 2},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ start = threading.Barrier(2)
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+ resolver_calls = 0
+
+ def resolve(_filters):
+ nonlocal resolver_calls
+ resolver_calls += 1
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ return [0b1], 1
+
+ def invoke():
+ start.wait(timeout=5)
+ return index.search(
+ [1.0, 0.0],
+ 1,
+ filter_a,
+ resolve,
+ lambda _labels: None,
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ futures = [executor.submit(invoke) for _ in range(2)]
+ assert resolver_started.wait(timeout=5)
+ wait_for_preflight_participants(index, 2)
+ release_resolver.set()
+ assert [future.result(timeout=5)[0] for future in futures] == [[1], [1]]
+
+ assert resolver_calls == 1
+ assert runtime.prepare_filter_words_count == 1
+ assert not index._preflight_flights
+
+
+def test_mutation_discards_host_filter_resolved_for_old_generation():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"filter_cache_size": 2},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="old")])
+ filter_new = {"op": "must", "field": "account_id", "conds": ["new"]}
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+ resolver_calls = 0
+ registered_layouts = []
+
+ def resolve(_filters):
+ nonlocal resolver_calls
+ resolver_calls += 1
+ if resolver_calls == 1:
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ return [0b0], 0
+ return [0b10], 1
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ future = executor.submit(
+ index.search,
+ [0.0, 1.0],
+ 1,
+ filter_new,
+ resolve,
+ lambda labels: registered_layouts.append(list(labels)),
+ )
+ assert resolver_started.wait(timeout=5)
+ index.add_candidates([candidate(2, [0.0, 1.0], account_id="new")])
+ release_resolver.set()
+ assert future.result(timeout=5)[0] == [2]
+
+ assert resolver_calls == 2
+ assert registered_layouts == [[1], [1, 2]]
+ assert runtime.build_count == 1
+
+
+def test_close_during_host_filter_resolution_rejects_search_before_gpu_admission():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+
+ def resolve(_filters):
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ return [0b1], 1
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ future = executor.submit(
+ index.search,
+ [1.0, 0.0],
+ 1,
+ filter_a,
+ resolve,
+ lambda _labels: None,
+ )
+ assert resolver_started.wait(timeout=5)
+ index.close()
+ release_resolver.set()
+ with pytest.raises(RuntimeError, match="dense index is closed"):
+ future.result(timeout=5)
+
+ assert runtime.build_count == 0
+ assert runtime.closed
+
+
+def test_close_while_warmed_filter_waits_for_gpu_gate_does_not_borrow_device_cache():
+ class BlockingWordRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__()
+ self.block_next = False
+ self.search_started = threading.Event()
+ self.resume_search = threading.Event()
+
+ def prepare_filter_words(self, words):
+ return (bool(words[0] & 1),)
+
+ def search(self, index, query, limit, mask):
+ if self.block_next:
+ self.block_next = False
+ self.search_started.set()
+ assert self.resume_search.wait(timeout=5)
+ return super().search(index, query, limit, mask)
+
+ runtime = BlockingWordRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={},
+ runtime=runtime,
+ )
+ index.add_candidates([candidate(1, [1.0, 0.0], account_id="a")])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+
+ def resolver(_filters):
+ return [0b1], 1
+
+ def registrar(_labels):
+ return None
+
+ assert index.search([1.0, 0.0], 1, filter_a, resolver, registrar)[0] == [1]
+
+ host_prepared = threading.Event()
+ original_prepare = index._prepare_host_filter
+
+ def observe_prepare(*args, **kwargs):
+ prepared = original_prepare(*args, **kwargs)
+ host_prepared.set()
+ return prepared
+
+ index._prepare_host_filter = observe_prepare
+ runtime.block_next = True
+ with ThreadPoolExecutor(max_workers=3) as executor:
+ occupying = executor.submit(index.search, [1.0, 0.0], 1, None)
+ assert runtime.search_started.wait(timeout=5)
+ waiting = executor.submit(
+ index.search,
+ [1.0, 0.0],
+ 1,
+ filter_a,
+ resolver,
+ registrar,
+ )
+ assert host_prepared.wait(timeout=5)
+
+ closing = executor.submit(index.close)
+ deadline = time.monotonic() + 5
+ while not index._closed and time.monotonic() < deadline:
+ time.sleep(0.001)
+ assert index._closed
+
+ runtime.resume_search.set()
+ assert occupying.result(timeout=5)[0] == [1]
+ with pytest.raises(RuntimeError, match="dense index is closed"):
+ waiting.result(timeout=5)
+ closing.result(timeout=5)
+
+ assert runtime.closed
+
+
+def test_device_cache_eviction_before_admission_uses_one_in_gate_fallback():
+ class WordPreparingRuntime(FakeCuVSRuntime):
+ def __init__(self):
+ super().__init__()
+ self.prepare_filter_words_count = 0
+
+ def prepare_filter_words(self, words):
+ self.prepare_filter_words_count += 1
+ return tuple(bool(words[0] & (1 << row)) for row in range(2))
+
+ runtime = WordPreparingRuntime()
index = CuVSDenseIndex(
dimension=2,
distance="ip",
normalize_vectors=False,
- field_types={},
- config={},
+ field_types={"account_id": "string"},
+ config={"filter_cache_size": 1, "max_concurrent_gpu_searches": 2},
runtime=runtime,
)
- index.add_candidates([candidate(1, [1.0, 0.0])])
- assert index.search([1.0, 0.0], 1, None)[0] == [1]
-
- telemetry = [CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False) for _ in range(4)]
- with ThreadPoolExecutor(max_workers=4) as executor:
- futures = [
- executor.submit(index.search, [1.0, 0.0], 1, None, None, None, item)
- for item in telemetry
+ index.add_candidates(
+ [
+ candidate(1, [1.0, 0.0], account_id="a"),
+ candidate(2, [0.0, 1.0], account_id="b"),
]
- assert [future.result(timeout=5)[0] for future in futures] == [[1]] * 4
+ )
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ filter_b = {"op": "must", "field": "account_id", "conds": ["b"]}
+ resolver_calls = {"a": 0, "b": 0}
- assert runtime.peak_active == 1
- assert max(item.queue_ms for item in telemetry) > 0
+ def resolve(filters):
+ account_id = filters["conds"][0]
+ resolver_calls[account_id] += 1
+ return ([0b01], 1) if account_id == "a" else ([0b10], 1)
+
+ def registrar(_labels):
+ return None
+
+ # Warm A so the delayed search below initially observes a device-cache hit.
+ assert index.search([1.0, 0.0], 1, filter_a, resolve, registrar)[0] == [1]
+ assert resolver_calls == {"a": 1, "b": 0}
+
+ a_host_prepared = threading.Event()
+ release_a = threading.Event()
+ original_prepare = index._prepare_host_filter
+
+ def pause_a_after_host_prepare(filters, *args, **kwargs):
+ prepared = original_prepare(filters, *args, **kwargs)
+ if filters == filter_a:
+ a_host_prepared.set()
+ assert release_a.wait(timeout=5)
+ return prepared
+
+ index._prepare_host_filter = pause_a_after_host_prepare
+ eviction_telemetry = CuVSSearchTelemetry(algorithm="brute_force", auto_mode=False)
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ delayed_a = executor.submit(
+ index.search,
+ [1.0, 0.0],
+ 1,
+ filter_a,
+ resolve,
+ registrar,
+ eviction_telemetry,
+ )
+ assert a_host_prepared.wait(timeout=5)
+
+ # Materializing B evicts A from the size-one device LRU after A's host
+ # context was captured but before A consumes an admission permit.
+ assert index.search([0.0, 1.0], 1, filter_b, resolve, registrar)[0] == [2]
+ release_a.set()
+ assert delayed_a.result(timeout=5)[0] == [1]
+
+ # A resolves once to warm and exactly once in the admitted eviction
+ # fallback; it does not loop under alternating hot-key churn.
+ assert resolver_calls == {"a": 2, "b": 1}
+ assert runtime.prepare_filter_words_count == 3
+ assert eviction_telemetry.filter_cache_hit is False
+ assert eviction_telemetry.filter_cache_eviction_fallback is True
+ assert eviction_telemetry.as_dict()["filter_cache_eviction_fallback"] is True
def test_inflight_search_keeps_immutable_snapshot_during_rebuild():
@@ -1133,6 +1758,245 @@ def test_auto_mode_caches_native_route_for_selective_filter():
assert runtime.prepare_filter_count == 0
+ ("evaluation", "expected_route"),
+ [(([], 1), 1), (([0b11], 2), None)],
+ ids=["narrow-native", "broad-gpu"],
+)
+def test_auto_mode_same_key_preflight_is_singleflight(evaluation, expected_route):
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates(
+ [
+ candidate(10, [1.0, 0.0], account_id="a"),
+ candidate(20, [0.0, 1.0], account_id="a"),
+ ]
+ )
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ start = threading.Barrier(4)
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+ resolver_calls = 0
+
+ def resolve(_filters):
+ nonlocal resolver_calls
+ resolver_calls += 1
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ return evaluation
+
+ def invoke():
+ start.wait(timeout=5)
+ return index.preflight_native_count(
+ filter_a,
+ resolve,
+ lambda _labels: None,
+ )
+
+ with ThreadPoolExecutor(max_workers=4) as executor:
+ futures = [executor.submit(invoke) for _ in range(4)]
+ assert resolver_started.wait(timeout=5)
+ wait_for_preflight_participants(index, 4)
+ release_resolver.set()
+ routes = [future.result(timeout=5) for future in futures]
+
+ assert routes == [expected_route] * 4
+ assert resolver_calls == 1
+ assert not index._preflight_flights
+
+
+ "error_type",
+ [RuntimeError, _PreflightAbort],
+ ids=["exception", "base-exception"],
+)
+def test_auto_mode_same_key_preflight_broadcasts_error_and_allows_retry(error_type):
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates([candidate(10, [1.0, 0.0], account_id="a")])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ start = threading.Barrier(4)
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+ resolver_calls = 0
+
+ def failing_resolve(_filters):
+ nonlocal resolver_calls
+ resolver_calls += 1
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ raise error_type("preflight failed")
+
+ def invoke():
+ start.wait(timeout=5)
+ return index.preflight_native_count(
+ filter_a,
+ failing_resolve,
+ lambda _labels: None,
+ )
+
+ with ThreadPoolExecutor(max_workers=4) as executor:
+ futures = [executor.submit(invoke) for _ in range(4)]
+ assert resolver_started.wait(timeout=5)
+ wait_for_preflight_participants(index, 4)
+ release_resolver.set()
+ for future in futures:
+ with pytest.raises(error_type, match="preflight failed"):
+ future.result(timeout=5)
+
+ assert resolver_calls == 1
+ assert not index._preflight_flights
+ assert (
+ index.preflight_native_count(
+ filter_a,
+ lambda _filters: ([], 1),
+ lambda _labels: None,
+ )
+ == 1
+ )
+
+
+def test_auto_mode_mutation_wakes_same_key_preflight_waiters_and_drops_old_result():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates([candidate(10, [1.0, 0.0], account_id="a")])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ start = threading.Barrier(2)
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+ resolver_calls = 0
+
+ def resolve(_filters):
+ nonlocal resolver_calls
+ resolver_calls += 1
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ return [], 1
+
+ def invoke():
+ start.wait(timeout=5)
+ return index.preflight_native_count(
+ filter_a,
+ resolve,
+ lambda _labels: None,
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ futures = [executor.submit(invoke) for _ in range(2)]
+ assert resolver_started.wait(timeout=5)
+ wait_for_preflight_participants(index, 2)
+ index.add_candidates([candidate(20, [0.0, 1.0], account_id="a")])
+
+ deadline = time.monotonic() + 5
+ while not any(future.done() for future in futures) and time.monotonic() < deadline:
+ time.sleep(0.001)
+ assert any(future.done() for future in futures)
+
+ release_resolver.set()
+ assert [future.result(timeout=5) for future in futures] == [None, None]
+
+ assert resolver_calls == 1
+ assert not index._preflight_flights
+ assert (
+ index.preflight_native_count(
+ filter_a,
+ lambda _filters: ([0b11], 2),
+ lambda _labels: None,
+ )
+ is None
+ )
+
+
+def test_auto_mode_close_wakes_same_key_preflight_waiters():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates([candidate(10, [1.0, 0.0], account_id="a")])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+ start = threading.Barrier(2)
+ resolver_started = threading.Event()
+ release_resolver = threading.Event()
+
+ def resolve(_filters):
+ resolver_started.set()
+ assert release_resolver.wait(timeout=5)
+ return [], 1
+
+ def invoke():
+ start.wait(timeout=5)
+ return index.preflight_native_count(
+ filter_a,
+ resolve,
+ lambda _labels: None,
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ futures = [executor.submit(invoke) for _ in range(2)]
+ assert resolver_started.wait(timeout=5)
+ wait_for_preflight_participants(index, 2)
+ index.close()
+ assert runtime.closed
+ assert not index._preflight_flights
+ release_resolver.set()
+ assert [future.result(timeout=5) for future in futures] == [None, None]
+
+
+def test_close_retries_runtime_cleanup_and_rejects_new_searches():
+ runtime = FakeCuVSRuntime()
+ runtime.close_error = RuntimeError("close failed")
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={},
+ config={},
+ runtime=runtime,
+ )
+
+ with pytest.raises(RuntimeError, match="close failed"):
+ index.close()
+ assert not index._close_complete
+ with pytest.raises(RuntimeError, match="dense index is closed"):
+ index.search([1.0, 0.0], 1, None)
+
+ index.close()
+ index.close()
+ assert index._close_complete
+ assert runtime.closed
+ assert runtime.close_count == 2
+
+
def test_auto_mode_preflights_different_filters_concurrently():
runtime = FakeCuVSRuntime()
index = CuVSDenseIndex(
@@ -1366,6 +2230,105 @@ def test_auto_mode_wide_filter_reuses_preflight_bitmap_after_build():
assert runtime.build_count == 1
[email protected]("words", [[], [0xFFFFFFFF]], ids=["empty", "short"])
+def test_auto_mode_rejects_incomplete_gpu_filter_bitset(words):
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(33)])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+
+ with pytest.raises(RuntimeError, match="incomplete bitset for GPU routing"):
+ index.preflight_native_count(
+ filter_a,
+ lambda _filters: (words, 33),
+ lambda _labels: None,
+ )
+
+
+def test_explicit_mode_rejects_short_gpu_bitset_below_auto_threshold():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 100},
+ runtime=runtime,
+ auto_memory=False,
+ )
+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(33)])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+
+ with pytest.raises(RuntimeError, match="incomplete bitset for GPU routing"):
+ index.search(
+ [1.0, 0.0],
+ 1,
+ filter_a,
+ lambda _filters: ([0b1], 1),
+ lambda _labels: None,
+ )
+
+
+def test_auto_mode_accepts_omitted_bitset_for_native_route():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(33)])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+
+ assert (
+ index.preflight_native_count(
+ filter_a,
+ lambda _filters: ([], 1),
+ lambda _labels: None,
+ )
+ == 1
+ )
+
+
+def test_auto_mode_does_not_validate_stale_filter_projection_after_mutation():
+ runtime = FakeCuVSRuntime()
+ index = CuVSDenseIndex(
+ dimension=2,
+ distance="ip",
+ normalize_vectors=False,
+ field_types={"account_id": "string"},
+ config={"auto_filter_native_threshold": 1},
+ runtime=runtime,
+ auto_memory=True,
+ )
+ index.add_candidates([candidate(label, [1.0, 0.0], account_id="a") for label in range(32)])
+ filter_a = {"op": "must", "field": "account_id", "conds": ["a"]}
+
+ def resolve_and_mutate(_filters):
+ index.add_candidates([candidate(1000, [0.0, 1.0], account_id="a")])
+ return [0xFFFFFFFF], 32
+
+ assert (
+ index.preflight_native_count(
+ filter_a,
+ resolve_and_mutate,
+ lambda _labels: None,
+ )
+ is None
+ )
+
+
def test_auto_mode_retains_native_filter_token_for_selective_route():
runtime = FakeCuVSRuntime()
index = CuVSDenseIndex(
diff --git a/tests/vectordb/test_engine_filter_packed_native_abi.py b/tests/vectordb/test_engine_filter_packed_native_abi.py
new file mode 100644
index 000000000..426b957b2
--- /dev/null
+++ b/tests/vectordb/test_engine_filter_packed_native_abi.py
@@ -0,0 +1,77 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import json
+
+import pytest
+
+import openviking.storage.vectordb.engine as engine
+
+
+def _native_packed_abi_available() -> bool:
+ backend = getattr(engine, "_BACKEND", None)
+ return backend is not None and hasattr(backend, "_index_engine_evaluate_filter_packed")
+
+
+ not _native_packed_abi_available(),
+ reason="the packaged native extension predates the additive packed filter ABI",
+)
+def test_native_packed_filter_abi_has_exact_little_endian_layout_and_shapes():
+ config = json.dumps(
+ {
+ "CollectionName": "packed_filter_abi_test",
+ "IndexName": "default",
+ "VectorIndex": {
+ "IndexType": "flat",
+ "ElementCount": 0,
+ "MaxElementCount": 96,
+ "Dimension": 1,
+ "Distance": "l2",
+ "Quant": "float",
+ },
+ "ScalarIndex": [{"FieldName": "uri", "FieldType": "path"}],
+ }
+ )
+ index = engine.IndexEngine(config)
+ selected_rows = {0, 8, 16, 24, 31, 32, 63, 64}
+ labels = list(range(1000, 1065))
+ requests = []
+ for row, label in enumerate(labels):
+ request = engine.AddDataRequest()
+ request.label = label
+ request.vector = [float(row)]
+ request.fields_str = json.dumps(
+ {"uri": f"/keep/item-{row}" if row in selected_rows else f"/drop/item-{row}"}
+ )
+ requests.append(request)
+ assert index.add_data(requests) == 0
+ assert index.set_filter_layout(labels) == 0
+
+ keep_filter = json.dumps({"op": "must", "field": "uri", "conds": ["/keep"], "para": "-d=-1"})
+ missing_filter = json.dumps(
+ {"op": "must", "field": "uri", "conds": ["/missing"], "para": "-d=-1"}
+ )
+ expected_words = [0x81010101, 0x80000001, 0x00000001]
+ expected_bytes = b"\x01\x01\x01\x81\x01\x00\x00\x80\x01\x00\x00\x00"
+
+ legacy = index.evaluate_filter(keep_filter)
+ generic = index.evaluate_filter_packed(keep_filter)
+ cached = index.evaluate_filter_packed(keep_filter, max_cached_candidates=10)
+ routed_narrow = index.evaluate_filter_for_routing_packed(keep_filter, native_threshold=10)
+ routed_wide = index.evaluate_filter_for_routing_packed(keep_filter, native_threshold=1)
+ empty = index.evaluate_filter_packed(missing_filter)
+
+ assert legacy.bitset_words == expected_words
+ assert generic.bitset_words == expected_bytes
+ assert generic.eligible_count == 8
+ assert generic.native_filter_token == 0
+ assert cached.bitset_words == expected_bytes
+ assert cached.native_filter_token > 0
+ assert routed_narrow.bitset_words == b""
+ assert routed_narrow.eligible_count == 8
+ assert routed_narrow.native_filter_token > 0
+ assert routed_wide.bitset_words == expected_bytes
+ assert routed_wide.native_filter_token == 0
+ assert empty.bitset_words == b"\x00" * 12
+ assert empty.eligible_count == 0
diff --git a/tests/vectordb/test_engine_filter_routing_abi.py b/tests/vectordb/test_engine_filter_routing_abi.py
new file mode 100644
index 000000000..b07d7ea66
--- /dev/null
+++ b/tests/vectordb/test_engine_filter_routing_abi.py
@@ -0,0 +1,190 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import pytest
+
+from openviking.storage.vectordb.engine._python_api import build_abi3_exports
+
+
+class _CurrentBackend:
+ def __init__(self):
+ self.calls = []
+
+ def _new_index_engine(self, path_or_json):
+ self.calls.append(("new", path_or_json))
+ return "engine-handle"
+
+ def _index_engine_evaluate_filter(self, handle, dsl):
+ self.calls.append(("generic", handle, dsl))
+ return {"eligible_count": 1, "bitset_words": [1], "native_filter_token": 0}
+
+ def _index_engine_evaluate_filter_cached(self, handle, dsl, threshold):
+ self.calls.append(("cached", handle, dsl, threshold))
+ return {"eligible_count": 1, "bitset_words": [1], "native_filter_token": 7}
+
+ def _index_engine_evaluate_filter_for_routing(self, handle, dsl, threshold):
+ self.calls.append(("routed", handle, dsl, threshold))
+ if threshold == 0:
+ return {"eligible_count": 1, "bitset_words": [1], "native_filter_token": 0}
+ return {"eligible_count": 1, "bitset_words": [], "native_filter_token": 11}
+
+
+class _LegacyBackend:
+ def __init__(self):
+ self.calls = []
+
+ def _new_index_engine(self, path_or_json):
+ return "legacy-handle"
+
+ def _index_engine_evaluate_filter(self, handle, dsl):
+ self.calls.append(("generic", handle, dsl))
+ return {"eligible_count": 1, "bitset_words": [2], "native_filter_token": 0}
+
+ def _index_engine_evaluate_filter_cached(self, handle, dsl, threshold):
+ self.calls.append(("cached", handle, dsl, threshold))
+ return {"eligible_count": 1, "bitset_words": [2], "native_filter_token": 13}
+
+
+class _PackedBackend(_CurrentBackend):
+ def _index_engine_evaluate_filter_packed(self, handle, dsl):
+ self.calls.append(("generic-packed", handle, dsl))
+ return {
+ "eligible_count": 2,
+ "bitset_words_le": b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa",
+ "native_filter_token": 0,
+ }
+
+ def _index_engine_evaluate_filter_cached_packed(self, handle, dsl, threshold):
+ self.calls.append(("cached-packed", handle, dsl, threshold))
+ return {
+ "eligible_count": 2,
+ "bitset_words_le": b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa",
+ "native_filter_token": 17,
+ }
+
+ def _index_engine_evaluate_filter_for_routing_packed(self, handle, dsl, threshold):
+ self.calls.append(("routed-packed", handle, dsl, threshold))
+ if threshold >= 2:
+ return {
+ "eligible_count": 2,
+ "bitset_words_le": b"",
+ "native_filter_token": 19,
+ }
+ return {
+ "eligible_count": 2,
+ "bitset_words_le": b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa",
+ "native_filter_token": 0,
+ }
+
+
+def test_routed_filter_uses_additive_abi_without_changing_generic_calls():
+ backend = _CurrentBackend()
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ routed = index_engine.evaluate_filter_for_routing("dsl", native_threshold=5)
+ generic = index_engine.evaluate_filter("dsl", max_cached_candidates=5)
+
+ assert routed.eligible_count == 1
+ assert routed.bitset_words == []
+ assert routed.native_filter_token == 11
+ assert generic.bitset_words == [1]
+ assert generic.native_filter_token == 7
+ assert ("routed", "engine-handle", "dsl", 5) in backend.calls
+ assert ("cached", "engine-handle", "dsl", 5) in backend.calls
+
+
+def test_routed_filter_threshold_zero_reaches_new_abi():
+ backend = _CurrentBackend()
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ result = index_engine.evaluate_filter_for_routing("dsl", native_threshold=0)
+
+ assert result.eligible_count == 1
+ assert result.bitset_words == [1]
+ assert result.native_filter_token == 0
+ assert ("routed", "engine-handle", "dsl", 0) in backend.calls
+
+
+def test_routed_filter_falls_back_with_a_legacy_extension():
+ backend = _LegacyBackend()
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ result = index_engine.evaluate_filter_for_routing("dsl", native_threshold=5)
+ threshold_zero = index_engine.evaluate_filter_for_routing("dsl", native_threshold=0)
+
+ assert result.eligible_count == 1
+ assert result.bitset_words == [2]
+ assert result.native_filter_token == 13
+ assert threshold_zero.bitset_words == [2]
+ assert threshold_zero.native_filter_token == 0
+ assert backend.calls == [
+ ("cached", "legacy-handle", "dsl", 5),
+ ("generic", "legacy-handle", "dsl"),
+ ]
+
+
+def test_packed_filter_abi_preserves_little_endian_bytes_and_route_shapes():
+ backend = _PackedBackend()
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ generic = index_engine.evaluate_filter_packed("wide")
+ cached = index_engine.evaluate_filter_packed("wide", max_cached_candidates=4)
+ routed = index_engine.evaluate_filter_for_routing_packed("narrow", native_threshold=2)
+
+ expected = b"\x04\x03\x02\x01\xdd\xcc\xbb\xaa"
+ assert generic.bitset_words == expected
+ assert cached.bitset_words == expected
+ assert cached.native_filter_token == 17
+ assert routed.bitset_words == b""
+ assert routed.eligible_count == 2
+ assert routed.native_filter_token == 19
+ assert backend.calls[-3:] == [
+ ("generic-packed", "engine-handle", "wide"),
+ ("cached-packed", "engine-handle", "wide", 4),
+ ("routed-packed", "engine-handle", "narrow", 2),
+ ]
+
+
+def test_packed_filter_methods_fall_back_to_legacy_backend_results():
+ backend = _LegacyBackend()
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ generic = index_engine.evaluate_filter_packed("wide")
+ cached = index_engine.evaluate_filter_packed("wide", max_cached_candidates=5)
+ routed = index_engine.evaluate_filter_for_routing_packed("narrow", native_threshold=5)
+
+ assert generic.bitset_words == [2]
+ assert cached.bitset_words == [2]
+ assert cached.native_filter_token == 13
+ assert routed.bitset_words == [2]
+ assert routed.native_filter_token == 13
+ assert backend.calls == [
+ ("generic", "legacy-handle", "wide"),
+ ("cached", "legacy-handle", "wide", 5),
+ ("cached", "legacy-handle", "narrow", 5),
+ ]
+
+
+def test_packed_filter_abi_rejects_a_non_bytes_payload():
+ backend = _PackedBackend()
+ backend._index_engine_evaluate_filter_packed = lambda _handle, _dsl: {
+ "eligible_count": 1,
+ "bitset_words_le": [1],
+ "native_filter_token": 0,
+ }
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ with pytest.raises(TypeError, match="bitset_words_le as bytes"):
+ index_engine.evaluate_filter_packed("dsl")
+
+
+def test_packed_filter_abi_rejects_a_missing_payload():
+ backend = _PackedBackend()
+ backend._index_engine_evaluate_filter_packed = lambda _handle, _dsl: {
+ "eligible_count": 0,
+ "native_filter_token": 0,
+ }
+ index_engine = build_abi3_exports(backend)["IndexEngine"]("config")
+
+ with pytest.raises(KeyError, match="missing bitset_words_le"):
+ index_engine.evaluate_filter_packed("dsl")
diff --git a/tests/vectordb/test_store_streaming.py b/tests/vectordb/test_store_streaming.py
new file mode 100644
index 000000000..7add1d426
--- /dev/null
+++ b/tests/vectordb/test_store_streaming.py
@@ -0,0 +1,221 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import gc
+import weakref
+
+import pytest
+
+import openviking.storage.vectordb.engine as engine
+from openviking.storage.vectordb.store.data import CandidateData
+from openviking.storage.vectordb.store.local_store import (
+ STORE_SCAN_PAGE_SIZE,
+ StoreEngineProxy,
+)
+from openviking.storage.vectordb.store.store_manager import StoreManager
+
+
+class _TrackedPage(list):
+ pass
+
+
+class _PagedStorageEngine:
+ def __init__(self, rows):
+ self.rows = sorted(rows)
+ self.page_refs = []
+ self.page_sizes = []
+ self.peak_live_pages = 0
+ self.full_scan_calls = 0
+
+ def seek_range(self, _start_key, _end_key):
+ self.full_scan_calls += 1
+ raise AssertionError("paged candidate recovery must not use a full range scan")
+
+ def seek_range_page(
+ self,
+ start_key,
+ end_key,
+ limit,
+ max_bytes,
+ start_exclusive=False,
+ ):
+ gc.collect()
+ selected = []
+ selected_bytes = 0
+ for row in self.rows:
+ after_start = row[0] > start_key if start_exclusive else row[0] >= start_key
+ if after_start and row[0] < end_key:
+ row_bytes = len(row[0].encode("utf-8")) + len(row[1])
+ if selected and (
+ selected_bytes >= max_bytes or row_bytes > max_bytes - selected_bytes
+ ):
+ break
+ selected.append(row)
+ selected_bytes += row_bytes
+ if len(selected) == limit:
+ break
+ page = _TrackedPage(selected)
+ self.page_refs.append(weakref.ref(page))
+ self.page_sizes.append(len(page))
+ self.peak_live_pages = max(
+ self.peak_live_pages,
+ sum(page_ref() is not None for page_ref in self.page_refs),
+ )
+ return page
+
+
+class _LegacyStorageEngine:
+ def __init__(self, rows):
+ self.rows = sorted(rows)
+ self.full_scan_calls = 0
+
+ def seek_range(self, start_key, end_key):
+ self.full_scan_calls += 1
+ return [row for row in self.rows if start_key <= row[0] < end_key]
+
+
+def _decode_candidate_payloads(monkeypatch):
+ candidate_refs = []
+ peak_live_candidates = 0
+
+ def decode(payload):
+ nonlocal peak_live_candidates
+ candidate = CandidateData(label=int(payload.decode("ascii")), vector=[1.0, 0.0])
+ candidate_refs.append(weakref.ref(candidate))
+ peak_live_candidates = max(
+ peak_live_candidates,
+ sum(candidate_ref() is not None for candidate_ref in candidate_refs),
+ )
+ return candidate
+
+ monkeypatch.setattr(CandidateData, "from_bytes", staticmethod(decode))
+ return candidate_refs, lambda: peak_live_candidates
+
+
+def test_candidate_recovery_bounds_encoded_pages_and_deserialized_rows(monkeypatch):
+ prefix = StoreManager.CandsTable
+ row_count = STORE_SCAN_PAGE_SIZE + 7
+ keys = [str(index) for index in range(row_count)]
+ rows = [(prefix + key, key.encode("ascii")) for key in keys]
+ storage_engine = _PagedStorageEngine(rows)
+ store = StoreEngineProxy(storage_engine)
+ manager = StoreManager(store)
+ candidate_refs, peak_live_candidates = _decode_candidate_payloads(monkeypatch)
+
+ labels = []
+ for candidate in manager.iter_all_cands_data():
+ labels.append(candidate.label)
+
+ assert labels == [int(key) for key in sorted(keys)]
+ assert storage_engine.full_scan_calls == 0
+ assert storage_engine.page_sizes == [STORE_SCAN_PAGE_SIZE, 7, 0]
+ assert storage_engine.peak_live_pages == 1
+ assert peak_live_candidates() <= 2
+ del candidate
+ gc.collect()
+ assert not any(candidate_ref() is not None for candidate_ref in candidate_refs)
+
+
+def test_store_proxy_pages_without_retaining_the_previous_encoded_page():
+ rows = [
+ ("candidate:" + key, ("value-" + key).encode("ascii"))
+ for key in ("1", "10", "2", "20", "3", "30", "4")
+ ]
+ storage_engine = _PagedStorageEngine(rows)
+ store = StoreEngineProxy(storage_engine)
+
+ keys = []
+ for key, value in store.iter_all("candidate:", page_size=100, page_bytes=40):
+ keys.append(key)
+ assert value == ("value-" + key).encode("ascii")
+
+ assert keys == ["1", "10", "2", "20", "3", "30", "4"]
+ assert storage_engine.page_sizes == [2, 2, 2, 1, 0]
+ assert storage_engine.peak_live_pages == 1
+ assert storage_engine.full_scan_calls == 0
+
+ oversized_engine = _PagedStorageEngine(rows)
+ oversized_items = list(
+ StoreEngineProxy(oversized_engine).iter_all(
+ "candidate:",
+ page_size=100,
+ page_bytes=1,
+ )
+ )
+ assert [key for key, _value in oversized_items] == keys
+ assert oversized_engine.page_sizes == [1] * len(rows) + [0]
+
+
+def test_store_proxy_rejects_older_engines_without_bounded_scan():
+ prefix = StoreManager.CandsTable
+ rows = [(prefix + "1", b"1"), (prefix + "2", b"2")]
+ storage_engine = _LegacyStorageEngine(rows)
+ manager = StoreManager(StoreEngineProxy(storage_engine))
+
+ with pytest.raises(RuntimeError, match="does not support bounded store scans"):
+ manager.get_all_cands_data()
+
+ assert storage_engine.full_scan_calls == 0
+
+
+def test_store_proxy_rejects_a_nonadvancing_native_cursor():
+ class StalledStorageEngine:
+ def seek_range_page(
+ self,
+ _start_key,
+ _end_key,
+ _limit,
+ _max_bytes,
+ _start_exclusive=False,
+ ):
+ return [("candidate:1", b"value")]
+
+ store = StoreEngineProxy(StalledStorageEngine())
+
+ with pytest.raises(RuntimeError, match="did not advance its continuation cursor"):
+ list(store.iter_all("candidate:", page_size=1))
+
+
+def test_store_proxy_accepts_the_table_prefix_as_the_first_inclusive_key():
+ storage_engine = _PagedStorageEngine([("candidate:", b"root"), ("candidate:1", b"child")])
+
+ items = list(StoreEngineProxy(storage_engine).iter_all("candidate:", page_size=1))
+
+ assert items == [("", b"root"), ("1", b"child")]
+ assert storage_engine.page_sizes == [1, 1, 0]
+
+
+def test_closing_store_stream_releases_the_current_encoded_page():
+ rows = [(f"candidate:{index}", b"value") for index in range(4)]
+ storage_engine = _PagedStorageEngine(rows)
+ stream = StoreEngineProxy(storage_engine).iter_all("candidate:", page_size=2)
+
+ assert next(stream)[0] == "0"
+ stream.close()
+ gc.collect()
+
+ assert not any(page_ref() is not None for page_ref in storage_engine.page_refs)
+
+
+def test_native_volatile_store_paged_scan_abi():
+ if engine.ENGINE_VARIANT == "unavailable":
+ pytest.skip("No native VectorDB engine is packaged in this test environment")
+
+ store = engine.VolatileStore()
+ assert (
+ store.put_data(
+ ["candidate:1", "candidate:10", "candidate:2"],
+ [b"one", b"ten", b"two"],
+ )
+ == 0
+ )
+
+ first = store.seek_range_page("candidate:", "candidate;", 2, 1024, False)
+ second = store.seek_range_page(first[-1][0], "candidate;", 2, 1024, True)
+ byte_limited = store.seek_range_page("candidate:", "candidate;", 10, 14, False)
+ oversized_first = store.seek_range_page("candidate:", "candidate;", 10, 1, False)
+
+ assert first == [("candidate:1", b"one"), ("candidate:10", b"ten")]
+ assert second == [("candidate:2", b"two")]
+ assert byte_limited == [("candidate:1", b"one")]
+ assert oversized_first == [("candidate:1", b"one")]