Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(vectordb): coalesce auto cuVS rebuilds during bulk ingest (#3277)
Yuanqing ZHAOfa19ac0a753e
diff --git a/benchmark/vectordb_perf/README.md b/benchmark/vectordb_perf/README.mdindex 6a470f3e7..c2a1b8ca5 100644--- a/benchmark/vectordb_perf/README.md+++ b/benchmark/vectordb_perf/README.md@@ -204,6 +204,7 @@ runner 会把配置里的 `name` 改成 `<name>_bench_<run-id>`,避免覆盖| --- | --- || `setup` | 创建 collection 和索引 schema || `ingest` | 通过 backend upsert OV context row |+| `prepare` | Auto background 模式等待最终 derived GPU index ready;不计入 search QPS || `validate` | count、get、过滤 count || `vector_search` | `search_in_tenant`,无指定目录 || `filtered_vector_search` | `search_in_tenant`,带 `target_directories` |@@ -215,11 +216,18 @@ runner 会把配置里的 `name` 改成 `<name>_bench_<run-id>`,避免覆盖而不是逐行调用 `upsert`。对本地后端,这会把每批记录合并到一次 store 持久化操作;store 与 vector index 之间不提供跨组件事务。远端 adapter 还可根据请求上限进一步拆批,因此一次 `upsert_many` 也不代表一个远端请求或跨子批事务。批量接口只提供完整记录-upsert;需要保留未提供字段的 partial update 仍使用单条 `upsert`。当前 runner 是该接口-在仓库内的首个调用方;本改动不自动改变现有 embedding、migration 或 ovpack 写入流程。+upsert;需要保留未提供字段的 partial update 仍使用单条 `upsert`。runner 会在整个多批+ingest 外层使用 `bulk_ingest` maintenance scope:native index 和数据持久化仍逐批更新,+但 Auto background cuVS rebuild 会延迟到最外层 scope 退出后再合并触发一次。这个 scope+不是事务或原子性边界,退出时只调度 rebuild,本身不等待 GPU ready。正式计时 search 前,+runner 还会显式等待该最终 GPU snapshot ready;+readiness wait 单独记录为 `prepare/wait_for_auto_derived_index`,失败时不会继续产出可能混合+CPU fallback 的 search QPS。当前 runner 是这些批量接口在仓库内的首个调用方;本改动不+自动改变现有 embedding、migration 或 ovpack 写入流程。`--mode write-only` 只会出现 `setup` / `ingest` / 可选 `cleanup` 阶段。-`--mode read-only` 只会出现 `validate` / `vector_search` / `filtered_vector_search` 阶段。+`--mode read-only` 不执行 `setup` / `ingest`;Auto background 配置下会先出现 `prepare`,+随后执行 `validate` / `vector_search` / `filtered_vector_search`。运行中会向 stderr 输出进度,包括当前 collection、写入条数、查询条数和速率。进度输出示例:diff --git a/benchmark/vectordb_perf/run.py b/benchmark/vectordb_perf/run.pyindex d37e2c275..2439e1820 100644--- a/benchmark/vectordb_perf/run.py+++ b/benchmark/vectordb_perf/run.py@@ -44,6 +44,7 @@ DIR_VECTOR_DATASET_URL = "https://github.com/KurtPatrickHere/dir-vector-dataset"DEFAULT_DIR_VECTOR_DATASETS = ("wiki", "arxiv")PROGRESS_INTERVAL_SECONDS = 5.0DIR_VECTOR_QUERY_SCOPES = ("dataset", "derived_gt_lca_v1")+DERIVED_INDEX_READY_TIMEOUT_SECONDS = 600.0PROFILE_DEFAULTS: dict[str, dict[str, Any]] = {@@ -1196,6 +1197,30 @@ async def run_search_phase(return events, validation_errors, quality+async def wait_for_auto_derived_index(+ backend: VikingVectorIndexBackend,+ *,+ timeout: float = DERIVED_INDEX_READY_TIMEOUT_SECONDS,+) -> bool:+ """Wait for Auto's background GPU snapshot before timed search phases."""++ def wait() -> bool:+ adapter = backend._shared_adapter+ collection = adapter.get_collection()+ index = collection.get_index(adapter.index_name)+ if index is None:+ raise RuntimeError(f"benchmark index not found: {adapter.index_name}")+ wait_for_rebuild = getattr(index, "wait_for_background_rebuild", None)+ if not callable(wait_for_rebuild):+ raise RuntimeError("Auto backend does not expose background rebuild readiness")+ return bool(wait_for_rebuild(timeout=timeout))++ ready = await asyncio.to_thread(wait)+ if not ready:+ raise TimeoutError(f"Auto derived index was not ready within {timeout:.0f}s")+ return True++def result_hits_ground_truth(rows: list[dict[str, Any]], ground_truth_ids: list[str]) -> bool:returned_ids = {str(row.get("id")) for row in rows if row.get("id") is not None}return any(str(item) in returned_ids for item in ground_truth_ids)@@ -1306,28 +1331,29 @@ async def run_benchmark_async(options: BenchOptions) -> RunResult:progress(f"ingest: start batch_size={batch_size}")ingest_started_at = time.perf_counter()last_ingest_reported_at = 0.0- for batch in batched(workload.records(), batch_size):- if len(fetch_ids) < 20:- fetch_ids.extend(str(row["id"]) for row in batch[: 20 - len(fetch_ids)])- event, ids = await async_timed_event(- "ingest",- "upsert_rows_batch",- lambda batch=batch: upsert_records(backend, ctx, batch),- count=len(batch),- )- events.append(event)- if not event.success:- validation_errors.append(event.error or "upsert_rows_batch failed")- break- inserted += len(ids or [])- last_ingest_reported_at = progress_count(- "ingest",- inserted,- workload.expected_rows,- ingest_started_at,- last_ingest_reported_at,- force=inserted == workload.expected_rows,- )+ async with backend.bulk_ingest(ctx=ctx):+ for batch in batched(workload.records(), batch_size):+ if len(fetch_ids) < 20:+ fetch_ids.extend(str(row["id"]) for row in batch[: 20 - len(fetch_ids)])+ event, ids = await async_timed_event(+ "ingest",+ "upsert_rows_batch",+ lambda batch=batch: upsert_records(backend, ctx, batch),+ count=len(batch),+ )+ events.append(event)+ if not event.success:+ validation_errors.append(event.error or "upsert_rows_batch failed")+ break+ inserted += len(ids or [])+ last_ingest_reported_at = progress_count(+ "ingest",+ inserted,+ workload.expected_rows,+ ingest_started_at,+ last_ingest_reported_at,+ force=inserted == workload.expected_rows,+ )if options.mode != "read-only" and inserted == 0:validation_errors.append("no records inserted")@@ -1363,6 +1389,33 @@ async def run_benchmark_async(options: BenchOptions) -> RunResult:kept_collection=kept_collection,)+ cuvs_config = getattr(vectordb_config, "cuvs", None)+ auto_background_enabled = bool(+ cuvs_config is not None+ and getattr(cuvs_config, "auto_enable", False)+ and getattr(cuvs_config, "auto_background_rebuild", False)+ )+ if auto_background_enabled:+ progress("prepare: wait for Auto derived index")+ event, _ = await async_timed_event(+ "prepare",+ "wait_for_auto_derived_index",+ lambda: wait_for_auto_derived_index(backend),+ )+ events.append(event)+ if not event.success:+ validation_errors.append(event.error or "Auto derived index readiness failed")+ await close_backend(backend)+ return finalize_result(+ options,+ workload,+ collection_name,+ events,+ validation_errors,+ quality,+ kept_collection=True,+ )+if options.mode == "read-only":fetch_ids = sample_record_ids(workload, 20)diff --git a/docs/design/openviking-cuvs-integration-plan.md b/docs/design/openviking-cuvs-integration-plan.mdindex 50c6c1a44..bb48122b1 100644--- a/docs/design/openviking-cuvs-integration-plan.md+++ b/docs/design/openviking-cuvs-integration-plan.md@@ -362,6 +362,10 @@ URI/path 使用更低阈值,是因为宽路径需要 native Trie traversal 和- mutation 不会破坏已被查询持有的 snapshot;同步 rebuild 仍由写锁合并为一次;- 同一 GPU 上不同 collection 的 admission/build 由进程级 coordinator 串行,避免并发超卖显存;- native fallback 不持有 GPU 锁,可以继续并发读取;+- 边界明确的多调用 bulk ingest 可显式延迟 derived GPU maintenance;native/store 仍逐批+ 可见,最外层 scope 退出后只调度一次 rebuild,该 scope 不提供事务或原子性;+- background worker 在 debounce 到期、candidate build 后和 commit 前都校验 generation;+ index 替换会先继承 bulk suspension、停止旧 maintenance worker,再启动新 worker;- Store 是最终事实来源,重启后派生状态会重新收敛。### 10.2 错误矩阵diff --git a/docs/en/guides/16-cuvs.md b/docs/en/guides/16-cuvs.mdindex 984cd8ed1..6e54960fe 100644--- a/docs/en/guides/16-cuvs.md+++ b/docs/en/guides/16-cuvs.md@@ -89,7 +89,16 @@ Explicit `backend: "cuvs"` continues to use cuVS for supported dense queries.mutations are coalesced for `auto_rebuild_debounce_ms`, and a worker builds thenew immutable GPU snapshot without holding the cross-backend mutation lock.The 500 ms default avoids rebuilding most intermediate batches during normal-ingestion; use a larger value for bulk loads with longer gaps between batches.+interactive ingestion. For a known multi-call bulk load, wrap all writes in+`async with backend.bulk_ingest(ctx=ctx):`: native visibility and persistence+still advance per call, while derived GPU maintenance is deferred until the+outermost scope exits and then scheduled once. This scope is a maintenance hint,+not a transaction or atomicity boundary; exiting it schedules the rebuild but+does not itself wait for GPU readiness. The vector backend benchmark adds that+explicit readiness wait before timing search. Changing the debounce remains+useful for callers that cannot identify a bulk-load boundary. Auto remains+opt-in; with Auto/background rebuild disabled, the scope is a no-op for derived+maintenance and native CPU behavior and dtype are unchanged.Queries use the current native index while the snapshot is dirty, so GPU buildtime does not become request queue time. The worker installs the new labellayout and GPU snapshot atomically only if its record generation is stilldiff --git a/docs/zh/guides/16-cuvs.md b/docs/zh/guides/16-cuvs.mdindex c62f9ec7f..a54de4c43 100644--- a/docs/zh/guides/16-cuvs.md+++ b/docs/zh/guides/16-cuvs.md@@ -121,7 +121,14 @@ dense query 仍固定使用 cuVS。`auto_background_rebuild` 默认关闭。开启后,连续 mutation 会按`auto_rebuild_debounce_ms` 合并,worker 在不持有跨后端 mutation 锁的情况下构建新的 immutable GPU snapshot。默认 500 ms 用于避免普通 ingest 的中间 batch-反复触发构建;batch 间隔更长的 bulk load 应配置更大的值。snapshot dirty 期间查询直接使用当前 native index,+反复触发构建。对于边界明确、由多次调用组成的 bulk load,可把所有写入放在+`async with backend.bulk_ingest(ctx=ctx):` scope 内:native 可见性和持久化仍按+每次调用推进,但 derived GPU maintenance 会延迟到最外层 scope 退出后只调度一次。+该 scope 只是 maintenance hint,不提供事务或原子性;退出 scope 只负责调度 rebuild,+本身不等待 GPU ready。vector backend benchmark 会额外在正式计时 search 前显式等待+最终 snapshot;无法识别 bulk 边界的调用方仍可按实际 batch 间隔调整 debounce。Auto+仍为显式启用;未开启 Auto/background rebuild 时,该 scope 对派生维护为 no-op,不改变+原生 CPU 检索、写入与 dtype 行为。snapshot dirty 期间查询直接使用当前 native index,不会把 GPU build 时间转化成请求排队时间。worker 只在 record generation 仍匹配时原子提交 label layout 和 GPU snapshot;过期 build 会被丢弃,并只重建最新一代。diff --git a/openviking/storage/vectordb/collection/collection.py b/openviking/storage/vectordb/collection/collection.pyindex 0104ae304..b4499d08b 100644--- a/openviking/storage/vectordb/collection/collection.py+++ b/openviking/storage/vectordb/collection/collection.py@@ -38,6 +38,12 @@ class ICollection(ABC):def drop(self):raise NotImplementedError+ def begin_bulk_ingest(self) -> None:+ """Suspend optional derived-index maintenance across a bulk write scope."""++ def end_bulk_ingest(self) -> None:+ """Resume optional derived-index maintenance after a bulk write scope."""+@abstractmethoddef create_index(self, index_name: str, meta_data: Dict[str, Any]) -> IIndex:raise NotImplementedError@@ -244,6 +250,22 @@ class Collection:raise RuntimeError("Collection is closed")return self.__collection.drop()+ def begin_bulk_ingest(self) -> None:+ """Coalesce derived-index maintenance across multiple write calls.++ This is a maintenance hint, not a transaction or atomicity boundary.+ Backends that do not maintain derived indexes may treat it as a no-op.+ """+ if self.__collection is None:+ raise RuntimeError("Collection is closed")+ self.__collection.begin_bulk_ingest()++ def end_bulk_ingest(self) -> None:+ """End a matching :meth:`begin_bulk_ingest` scope."""+ if self.__collection is None:+ raise RuntimeError("Collection is closed")+ self.__collection.end_bulk_ingest()+def get_meta_data(self) -> Dict[str, Any]:"""Retrieve the full metadata of the collection.diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 49bcd27e9..0e9c61493 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -5,9 +5,11 @@ import jsonimport osimport randomimport shutil+import threadingimport time+from contextlib import contextmanagerfrom itertools import zip_longest-from typing import Any, Dict, List, Optional+from typing import Any, Dict, Iterator, List, Optionalfrom apscheduler.schedulers.background import BackgroundScheduler@@ -54,6 +56,106 @@ from openviking_cli.utils.logger import default_logger as loggerAUTO_ID_KEY = SpecialFields.AUTO_ID.value+class _MutationMarker:+ def __init__(self):+ self.changed = False++ def mark_changed(self) -> None:+ self.changed = True+++class _IndexMutationBarrier:+ """Coordinate index replacement with concurrent Store/index mutations.++ Normal mutations share the barrier so independent writers retain their+ existing concurrency. Index publication is exclusive: it waits for writes+ already in flight and prevents a new write from selecting the index being+ replaced. A monotonically increasing generation lets an off-lock rebuild+ reject a Store snapshot that became stale while the replacement was built.+ """++ def __init__(self):+ self._condition = threading.Condition(threading.Lock())+ self._active_mutations = 0+ self._publisher = False+ self._waiting_publishers = 0+ self._generation = 0++ @contextmanager+ def mutation(self) -> Iterator[_MutationMarker]:+ with self._condition:+ while self._publisher or self._waiting_publishers:+ self._condition.wait()+ self._active_mutations += 1+ marker = _MutationMarker()+ try:+ yield marker+ except BaseException:+ # The Store may already have committed before a later step raises.+ marker.mark_changed()+ raise+ finally:+ with self._condition:+ if marker.changed:+ self._generation += 1+ self._active_mutations -= 1+ if self._active_mutations == 0:+ self._condition.notify_all()++ @contextmanager+ def exclusive_mutation(self) -> Iterator[None]:+ with self._exclusive() as _:+ try:+ yield+ finally:+ with self._condition:+ self._generation += 1++ def snapshot_generation(self) -> int:+ """Return a boundary ordered after any exclusive collection mutation."""++ with self._condition:+ while self._publisher or self._waiting_publishers:+ self._condition.wait()+ return self._generation++ def is_current(self, expected_generation: int) -> bool:+ """Check whether a snapshot is already known to be stale."""++ with self._condition:+ return (+ not self._publisher+ and self._active_mutations == 0+ and self._generation == expected_generation+ )++ @contextmanager+ def publish_if_current(self, expected_generation: int) -> Iterator[bool]:+ """Reserve publication and report whether the snapshot is still current."""++ with self._exclusive():+ with self._condition:+ current = self._generation == expected_generation+ yield current++ @contextmanager+ def _exclusive(self) -> Iterator[None]:+ with self._condition:+ self._waiting_publishers += 1+ try:+ while self._publisher or self._active_mutations:+ self._condition.wait()+ self._publisher = True+ finally:+ self._waiting_publishers -= 1+ try:+ yield+ finally:+ with self._condition:+ self._publisher = False+ self._condition.notify_all()++def get_or_create_local_collection(meta_data: Optional[Dict[str, Any]] = None,path: str = "",@@ -122,6 +224,9 @@ class LocalCollection(ICollection):config: Optional[Dict[str, Any]] = None,):self.indexes = ThreadSafeDictManager[IIndex]()+ self._index_mutation_barrier = _IndexMutationBarrier()+ self._bulk_ingest_lock = threading.RLock()+ self._bulk_ingest_depth = 0self.meta: CollectionMeta = metaself.collection_name = ""@@ -156,15 +261,16 @@ class LocalCollection(ICollection):meta_data["Description"] = descriptionif not meta_data:return- self.meta.update(meta_data)- self.data_processor = DataProcessor(- self.meta.fields_dict, collection_name=self.meta.collection_name- )+ with self._index_mutation_barrier.exclusive_mutation():+ self.meta.update(meta_data)+ self.data_processor = DataProcessor(+ self.meta.fields_dict, collection_name=self.meta.collection_name+ )def get_meta_data(self):return self.meta.get_meta_data()- def close(self):+ def _close_unlocked(self) -> None:self._delete_scheduler_job()# Shutdown scheduler@@ -184,6 +290,125 @@ class LocalCollection(ICollection):self.indexes.iterate(close_index)self.indexes.clear()+ def close(self):+ with self._index_mutation_barrier.exclusive_mutation():+ self._close_unlocked()++ def begin_bulk_ingest(self) -> None:+ """Defer optional derived-index rebuilds until the outer scope exits."""++ with self._bulk_ingest_lock:+ if self._bulk_ingest_depth > 0:+ self._bulk_ingest_depth += 1+ return+ suspended: List[IIndex] = []+ try:+ for index in self.indexes.get_all().values():+ index.begin_bulk_ingest()+ suspended.append(index)+ except Exception:+ for index in reversed(suspended):+ try:+ index.end_bulk_ingest()+ except Exception:+ logger.warning(+ "Failed to roll back bulk-ingest suspension",+ exc_info=True,+ )+ raise+ self._bulk_ingest_depth = 1++ def end_bulk_ingest(self) -> None:+ """Resume derived-index maintenance after a matching begin call."""++ with self._bulk_ingest_lock:+ if self._bulk_ingest_depth <= 0:+ raise RuntimeError("bulk ingest scope is not active")+ if self._bulk_ingest_depth > 1:+ self._bulk_ingest_depth -= 1+ return+ first_error: Optional[Exception] = None+ for index in self.indexes.get_all().values():+ try:+ index.end_bulk_ingest()+ except Exception as exc:+ if first_error is None:+ first_error = exc+ logger.warning("Failed to end bulk-ingest suspension", exc_info=True)+ self._bulk_ingest_depth = 0+ if first_error is not None:+ raise first_error++ @staticmethod+ def _start_index_background_rebuild(index: IIndex) -> None:+ start = getattr(index, "_start_dense_rebuild_worker", None)+ if callable(start):+ start()++ @staticmethod+ def _stop_index_background_rebuild(index: IIndex) -> None:+ stop = getattr(index, "_stop_dense_rebuild_worker", None)+ if callable(stop):+ stop()++ def _publish_index(+ self,+ index_name: str,+ index: IIndex,+ ) -> Optional[IIndex]:+ """Publish an index while the caller holds the mutation barrier."""++ with self._bulk_ingest_lock:+ replaced = self.indexes.get(index_name)+ if self._bulk_ingest_depth > 0:+ index.begin_bulk_ingest()+ self.indexes.set(index_name, index)+ return replaced++ def _install_index(+ self,+ index_name: str,+ index: IIndex,+ *,+ expected_generation: int,+ expected_replaced: IIndex,+ start_background_rebuild: bool = False,+ ) -> bool:+ """Publish a replacement only if its Store snapshot is still current."""++ replaced: Optional[IIndex] = None+ with self._index_mutation_barrier.publish_if_current(expected_generation) as current:+ if not current or self.indexes.get(index_name) is not expected_replaced:+ return False+ replaced = self._publish_index(index_name, index)+ if replaced is not None and replaced is not index:+ try:+ # Retire the old worker before starting the replacement so two+ # full GPU builds do not overlap. This happens after the map+ # swap, so Store writes keep flowing to the new index while a+ # possibly in-flight old build winds down.+ self._stop_index_background_rebuild(replaced)+ except Exception:+ # Publication is the commit point. Post-swap maintenance+ # failures must not make the caller discard the installed index.+ logger.warning(+ "Failed to stop background rebuild for replaced index '%s'",+ index_name,+ exc_info=True,+ )+ if start_background_rebuild:+ try:+ self._start_index_background_rebuild(index)+ except Exception:+ # Publication is the commit point. A worker-start failure must+ # not be reported as though the old index remained installed.+ logger.warning(+ "Failed to start background rebuild for installed index '%s'",+ index_name,+ exc_info=True,+ )+ return True+def drop(self):self.close()@@ -193,10 +418,43 @@ class LocalCollection(ICollection):meta_data = {}if not self.store_mgr:raise RuntimeError("Store manager is not initialized")- cands_list: List[CandidateData] = self.store_mgr.get_all_cands_data()- index = self._new_index(index_name, meta_data, cands_list)- self.indexes.set(index_name, index)- self._delete_expire_delta_record()+ # Index creation has no older index that can absorb concurrent writes.+ # Keep its initial Store snapshot and publication in one exclusive+ # mutation scope; later steady-state rebuilds use generation validation+ # instead and remain off the write hot path.+ replaced: Optional[IIndex] = None+ published = False+ try:+ with self._index_mutation_barrier.exclusive_mutation():+ cands_list: List[CandidateData] = self.store_mgr.get_all_cands_data()+ index = self._new_index(+ index_name,+ meta_data,+ cands_list,+ defer_dense_rebuild_start=True,+ )+ replaced = self._publish_index(index_name, index)+ published = True+ self._delete_expire_delta_record()+ finally:+ if replaced is not None and replaced is not index:+ try:+ self._stop_index_background_rebuild(replaced)+ except Exception:+ logger.warning(+ "Failed to stop background rebuild for replaced index '%s'",+ index_name,+ exc_info=True,+ )+ if published:+ try:+ self._start_index_background_rebuild(index)+ except Exception:+ logger.warning(+ "Failed to start background rebuild for installed index '%s'",+ index_name,+ exc_info=True,+ )return indexdef has_index(self, index_name: str) -> bool:@@ -206,9 +464,10 @@ class LocalCollection(ICollection):return self.indexes.get(index_name)def drop_index(self, index_name: str) -> None:- index = self.indexes.remove(index_name)- if index:- index.drop()+ with self._index_mutation_barrier.exclusive_mutation():+ index = self.indexes.remove(index_name)+ if index:+ index.drop()def get_indexes(self) -> Dict[str, IIndex]:return self.indexes.get_all()@@ -219,10 +478,12 @@ class LocalCollection(ICollection):scalar_index: Optional[Dict[str, Any]] = None,description: Optional[str] = None,) -> None:- index = self.indexes.get(index_name)- if not index:- return- index.update(scalar_index, description)+ with self._index_mutation_barrier.mutation() as mutation:+ index = self.indexes.get(index_name)+ if not index:+ return+ index.update(scalar_index, description)+ mutation.mark_changed()def get_index_meta_data(self, index_name: str) -> Optional[Dict[str, Any]]:index = self.indexes.get(index_name)@@ -567,13 +828,15 @@ class LocalCollection(ICollection):if not self.store_mgr:raise RuntimeError("Store manager is not initialized")- need_record_delta = True if self.indexes.count() > 0 else False- delta_list = self.store_mgr.add_cands_data(cands_list, ttl, need_record_delta)+ with self._index_mutation_barrier.mutation() as mutation:+ need_record_delta = True if self.indexes.count() > 0 else False+ delta_list = self.store_mgr.add_cands_data(cands_list, ttl, need_record_delta)+ mutation.mark_changed()- def upsert_to_index(name, index):- index.upsert_data(delta_list)+ def upsert_to_index(name, index):+ index.upsert_data(delta_list)- self.indexes.iterate(upsert_to_index)+ self.indexes.iterate(upsert_to_index)for i, data in enumerate(data_list):data[vk] = list(cands_list[i].vector) if cands_list[i].vector else []@@ -629,13 +892,15 @@ class LocalCollection(ICollection):)if not self.store_mgr:raise RuntimeError("Store manager is not initialized")- need_record_delta = True if self.indexes.count() > 0 else False- delta_list = self.store_mgr.delete_data(labels_list, need_record_delta)+ with self._index_mutation_barrier.mutation() as mutation:+ need_record_delta = True if self.indexes.count() > 0 else False+ delta_list = self.store_mgr.delete_data(labels_list, need_record_delta)+ mutation.mark_changed()- def delete_from_index(name, index):- index.delete_data(delta_list)+ def delete_from_index(name, index):+ index.delete_data(delta_list)- self.indexes.iterate(delete_from_index)+ self.indexes.iterate(delete_from_index)def delete_all_data(self):"""Delete all data and rebuild indexes (thread-safe).@@ -648,53 +913,65 @@ class LocalCollection(ICollection):Uses locks to ensure no concurrent read/write requests cause errors during the operation."""- # Use get_all_with_lock() to ensure atomicity of the entire operation- with self.indexes.get_all_with_lock() as indexes_dict:- # 1. Save metadata and names for all indexes- indexes_metadata = []- for index_name, index in indexes_dict.items():- try:- meta_data = index.get_meta_data()- indexes_metadata.append((index_name, meta_data))- logger.debug(f"Saved metadata for index: {index_name}")- except Exception as e:- logger.error(f"Failed to get metadata for index {index_name}: {e}")-- # 2. Delete all indexes- index_names = list(indexes_dict.keys())- for index_name in index_names:+ # Keep one lock order everywhere: mutation barrier, bulk-ingest state,+ # then the index map.+ # This also lets replacement indexes inherit an active suspension before+ # they become visible to concurrent readers or writers.+ with self._index_mutation_barrier.exclusive_mutation():+ with self._bulk_ingest_lock, self.indexes.get_all_with_lock() as indexes_dict:+ # 1. Save metadata and names for all indexes+ indexes_metadata = []+ for index_name, index in indexes_dict.items():+ try:+ meta_data = index.get_meta_data()+ indexes_metadata.append((index_name, meta_data))+ logger.debug(f"Saved metadata for index: {index_name}")+ except Exception as e:+ logger.error(f"Failed to get metadata for index {index_name}: {e}")++ # 2. Delete all indexes+ index_names = list(indexes_dict.keys())+ for index_name in index_names:+ try:+ index = indexes_dict.pop(index_name, None)+ if index:+ index.drop()+ logger.debug(f"Dropped index: {index_name}")+ except Exception as e:+ logger.error(f"Failed to drop index {index_name}: {e}")++ # 3. Clear storage datatry:- index = indexes_dict.pop(index_name, None)- if index:- index.drop()- logger.debug(f"Dropped index: {index_name}")+ if self.store_mgr:+ self.store_mgr.clear()+ logger.info(+ "Storage cleared successfully",+ extra={"collection": self.collection_name},+ )except Exception as e:- logger.error(f"Failed to drop index {index_name}: {e}")-- # 3. Clear storage data- try:- if self.store_mgr:- self.store_mgr.clear()- logger.info(- "Storage cleared successfully", extra={"collection": self.collection_name}- )- except Exception as e:- logger.error(f"Failed to clear storage: {e}")- raise+ logger.error(f"Failed to clear storage: {e}")+ raise- # 4. Rebuild empty indexes using saved metadata- for index_name, meta_data in indexes_metadata:- try:- # Rebuild index with empty data list- empty_cands_list: List[CandidateData] = []- new_index = self._new_index(index_name, meta_data, empty_cands_list)- indexes_dict[index_name] = new_index- logger.info(f"Rebuilt index: {index_name}")- except Exception as e:- logger.error(f"Failed to rebuild index {index_name}: {e}")- # Continue rebuilding other indexes, don't interrupt the process+ # 4. Rebuild empty indexes using saved metadata+ for index_name, meta_data in indexes_metadata:+ try:+ empty_cands_list: List[CandidateData] = []+ new_index = self._new_index(+ index_name,+ meta_data,+ empty_cands_list,+ defer_dense_rebuild_start=True,+ )+ if self._bulk_ingest_depth > 0:+ new_index.begin_bulk_ingest()+ indexes_dict[index_name] = new_index+ self._start_index_background_rebuild(new_index)+ logger.info(f"Rebuilt index: {index_name}")+ except Exception as e:+ logger.error(f"Failed to rebuild index {index_name}: {e}")+ # Continue rebuilding other indexes, don't interrupt the process- logger.info(f"delete_all_data completed. Rebuilt {len(indexes_dict)} indexes")+ logger.info(f"delete_all_data completed. Rebuilt {len(indexes_dict)} indexes")def _delete_expire_delta_record(self):oldest_version = 0@@ -708,12 +985,15 @@ class LocalCollection(ICollection):def _expire_timeout_data(self):if not self.store_mgr:return- delta_list = self.store_mgr.expire_data()+ with self._index_mutation_barrier.mutation() as mutation:+ delta_list = self.store_mgr.expire_data()+ if delta_list:+ mutation.mark_changed()- def delete_from_index(name, index):- index.delete_data(delta_list)+ def delete_from_index(name, index):+ index.delete_data(delta_list)- self.indexes.iterate(delete_from_index)+ self.indexes.iterate(delete_from_index)def _register_scheduler_job(self):if self.ttl_cleanup_seconds > 0:@@ -793,28 +1073,76 @@ class LocalCollection(ICollection):index_name: Name of the indexold_index: Old index object"""+ # PersistentIndex deliberately reports need_rebuild=False. Its builder+ # writes version markers into the live index directory before map+ # publication, so it cannot use the optimistic discard protocol below+ # without a separate staging directory and atomic filesystem commit.+ if not isinstance(old_index, VolatileIndex):+ logger.debug("Skipping optimistic replacement for persistent index '%s'", index_name)+ return++ new_index: Optional[IIndex] = None+ installed = Falsetry:+ expected_generation = self._index_mutation_barrier.snapshot_generation()+# 1. Retrieve all dataif not self.store_mgr:raise RuntimeError("Store manager is not initialized")cands_list: List[CandidateData] = self.store_mgr.get_all_cands_data()+ if not self._index_mutation_barrier.is_current(expected_generation):+ logger.debug(+ "Skipping stale replacement build for index '%s' after Store snapshot",+ index_name,+ )+ return# 2. Get index metadatameta_data = old_index.get_meta_data()# 3. Create new index (this process is safe and doesn't affect the old index)- new_index = self._new_index(index_name, meta_data, cands_list, True)+ new_index = self._new_index(+ index_name,+ meta_data,+ cands_list,+ True,+ defer_dense_rebuild_start=True,+ )- # 4. Atomically replace the old index (ThreadSafeDictManager ensures thread safety)- self.indexes.set(index_name, new_index)+ # 4. Publish only if no Store/index mutation completed after the+ # snapshot boundary and this is still the index being rebuilt.+ # Otherwise the continuously updated old index remains authoritative+ # and a later maintenance pass can retry compaction.+ installed = self._install_index(+ index_name,+ new_index,+ expected_generation=expected_generation,+ expected_replaced=old_index,+ start_background_rebuild=True,+ )+ if not installed:+ logger.debug(+ "Discarding stale replacement for index '%s' after a concurrent mutation",+ index_name,+ )- # 5. Don't manually close the old index, let Python GC automatically reclaim it- # This avoids errors for threads currently using old_index- # The object will be automatically destructed when all references are released+ # 5. _install_index stops only the old background maintenance+ # worker. It deliberately leaves the old search object usable+ # until concurrent readers release their references.except Exception as e:logger.error(f"Failed to rebuild index {index_name}: {e}")# Rebuild failed, keep the old index unchanged+ finally:+ if new_index is not None and not installed:+ try:+ new_index.close()+ except Exception:+ logger.warning(+ "Failed to close unpublished replacement index '%s'",+ index_name,+ exc_info=True,+ )def aggregate_data(self,@@ -886,6 +1214,7 @@ class LocalCollection(ICollection):meta_data: Dict[str, Any],cands_list: List[CandidateData],force_rebuild: bool = False,+ defer_dense_rebuild_start: bool = False,):raise NotImplementedError@@ -906,6 +1235,7 @@ class VolatileCollection(LocalCollection):meta_data: Dict[str, Any],cands_list: List[CandidateData],force_rebuild: bool = False,+ defer_dense_rebuild_start: bool = False,):meta = create_index_meta(self.meta, "", meta_data)index = VolatileIndex(@@ -913,6 +1243,7 @@ class VolatileCollection(LocalCollection):meta=meta,cands_list=cands_list,dense_search_config=self.dense_search_config,+ defer_dense_rebuild_start=defer_dense_rebuild_start,)return index@@ -1100,8 +1431,9 @@ class PersistCollection(LocalCollection):def close(self):"""Close the collection and release resources."""- self.flush_all_indexes()- super().close() # Call parent close (includes TTL scheduling deletion)+ with self._index_mutation_barrier.exclusive_mutation():+ self.flush_all_indexes()+ self._close_unlocked()def flush_all_indexes(self):"""Manually trigger persistence of all indexes.@@ -1132,6 +1464,7 @@ class PersistCollection(LocalCollection):meta_data: Dict[str, Any],cands_list: List[CandidateData],force_rebuild: bool = False,+ defer_dense_rebuild_start: bool = False,):new_index_dir = str(safe_join_name(self.index_dir, index_name))os.makedirs(new_index_dir, exist_ok=True)@@ -1144,6 +1477,7 @@ class PersistCollection(LocalCollection):cands_list=cands_list,force_rebuild=force_rebuild,dense_search_config=self.dense_search_config,+ defer_dense_rebuild_start=defer_dense_rebuild_start,)return indexdiff --git a/openviking/storage/vectordb/index/cuvs_index.py b/openviking/storage/vectordb/index/cuvs_index.pyindex db98a9778..c6c173078 100644--- a/openviking/storage/vectordb/index/cuvs_index.py+++ b/openviking/storage/vectordb/index/cuvs_index.py@@ -1044,6 +1044,39 @@ class CuVSDenseIndex:self._cache_filter(cache_key, cached)return cached+ def _check_auto_memory_budget(+ self,+ vector_count: int,+ telemetry: Optional[CuVSSearchTelemetry],+ ) -> None:+ """Raise before a cuVS build when the estimated GPU peak is unsafe."""++ estimate = estimate_cuvs_memory(+ vector_count=vector_count,+ dimension=self.dimension,+ algorithm=self.algorithm,+ build_params=self._build_params,+ filter_cache_size=self.filter_cache_size,+ safety_factor=self.auto_memory_safety_factor,+ dtype=self.dtype,+ )+ if telemetry is not None:+ telemetry.memory_estimated_peak_bytes = estimate.estimated_peak_bytes+ try:+ free_bytes, total_bytes = self._runtime.memory_info()+ except Exception as exc:+ raise CuVSMemoryBudgetError("cuVS auto mode could not read free GPU memory") from exc+ usable_bytes = max(0, free_bytes - self.auto_memory_reserve_bytes)+ if telemetry is not None:+ telemetry.memory_free_bytes = free_bytes+ telemetry.memory_usable_bytes = usable_bytes+ if estimate.estimated_peak_bytes > usable_bytes:+ raise CuVSMemoryBudgetError(+ "cuVS auto mode kept native search because the estimated GPU peak "+ f"({estimate.estimated_peak_bytes} bytes) exceeds the usable free "+ f"memory ({usable_bytes} of {total_bytes} bytes after reserve)"+ )+def prepare_rebuild(self,telemetry: Optional[CuVSSearchTelemetry] = None,@@ -1051,9 +1084,8 @@ class CuVSDenseIndex:with self._lock:if not self._dirty:return None- labels = tuple(self._records)+ vector_count = len(self._records)generation = self._records_generation- dataset = [self._records[label].vector for label in labels]can_release_unused = self._active_searches == 0if can_release_unused:with self._runtime_device_scope():@@ -1061,44 +1093,40 @@ class CuVSDenseIndex:build_started = time.perf_counter()try:- if not labels:+ if vector_count == 0:return _CuVSBuildCandidate(runtime_index=None,- labels=labels,+ labels=(),generation=generation,)- with _CUVS_MEMORY_COORDINATOR.build_lock(self._runtime):- if self.auto_memory:++ # Reject stable, known-insufficient budgets before copying every+ # Python vector into a build dataset. Do not hold ``self._lock``+ # while taking the device build lock: foreground rebuilds already+ # use the opposite order through ``_search_admitted``.+ if self.auto_memory:+ with _CUVS_MEMORY_COORDINATOR.build_lock(self._runtime):release_index = getattr(self._runtime, "release_index", None)if release_index is not None and can_release_unused:release_index()- estimate = estimate_cuvs_memory(- vector_count=len(labels),- dimension=self.dimension,- algorithm=self.algorithm,- build_params=self._build_params,- filter_cache_size=self.filter_cache_size,- safety_factor=self.auto_memory_safety_factor,- dtype=self.dtype,+ self._check_auto_memory_budget(vector_count, telemetry)++ with self._lock:+ if self._records_generation != generation:+ return _CuVSBuildCandidate(+ runtime_index=None,+ labels=(),+ generation=generation,)- if telemetry is not None:- telemetry.memory_estimated_peak_bytes = estimate.estimated_peak_bytes- try:- free_bytes, total_bytes = self._runtime.memory_info()- except Exception as exc:- raise CuVSMemoryBudgetError(- "cuVS auto mode could not read free GPU memory"- ) from exc- usable_bytes = max(0, free_bytes - self.auto_memory_reserve_bytes)- if telemetry is not None:- telemetry.memory_free_bytes = free_bytes- telemetry.memory_usable_bytes = usable_bytes- if estimate.estimated_peak_bytes > usable_bytes:- raise CuVSMemoryBudgetError(- "cuVS auto mode kept native search because the estimated GPU peak "- f"({estimate.estimated_peak_bytes} bytes) exceeds the usable free "- f"memory ({usable_bytes} of {total_bytes} bytes after reserve)"- )+ labels = tuple(self._records)+ dataset = [self._records[label].vector for label in labels]++ with _CUVS_MEMORY_COORDINATOR.build_lock(self._runtime):+ # Admission is checked again while the per-device build lock is+ # held. Another collection may have allocated GPU memory while+ # this collection materialized its host dataset.+ if self.auto_memory:+ self._check_auto_memory_budget(len(labels), telemetry)try:if telemetry is not None:telemetry.build_performed = Truediff --git a/openviking/storage/vectordb/index/index.py b/openviking/storage/vectordb/index/index.pyindex 0e3bae0ab..a0499dbf8 100644--- a/openviking/storage/vectordb/index/index.py+++ b/openviking/storage/vectordb/index/index.py@@ -273,6 +273,12 @@ class IIndex(ABC):"""return True+ def begin_bulk_ingest(self) -> None:+ """Suspend optional derived-index rebuilds across a bulk write scope."""++ def end_bulk_ingest(self) -> None:+ """Resume optional derived-index rebuilds after a bulk write scope."""+class Index:"""diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex b2241ba59..9f7af96cd 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -31,6 +31,9 @@ from openviking.storage.vectordb.utils.path_safety import (from openviking.storage.vectordb.utils.validation import validate_name_strfrom openviking_cli.utils.logger import default_logger as logger+_DENSE_REBUILD_MEMORY_RETRY_BASE_SECONDS = 1.0+_DENSE_REBUILD_MEMORY_RETRY_MAX_SECONDS = 30.0+def normalize_vector(vector: List[float]) -> List[float]:"""Perform L2 normalization on a vector.@@ -338,8 +341,12 @@ class LocalIndex(IIndex):self._dense_rebuild_state_lock = threading.Lock()self._dense_rebuild_generation = 0self._dense_rebuild_debounce_deadline = 0.0+ self._dense_rebuild_suspend_count = 0+ self._dense_rebuild_deferred = Falseself._dense_rebuild_failure: Optional[Tuple[type[Exception], str]] = Noneself._dense_rebuild_memory_blocked = False+ self._dense_rebuild_memory_retry_attempts = 0+ self._dense_rebuild_memory_retry_not_before = 0.0self._dense_rebuild_thread: Optional[threading.Thread] = Nonedense_search_config = dict(dense_search_config or {})dense_search_backend = dense_search_config.get("backend")@@ -416,14 +423,58 @@ class LocalIndex(IIndex):returnwith self._dense_rebuild_state_lock:self._dense_rebuild_generation += 1- self._dense_rebuild_debounce_deadline = (- time.monotonic() + self._dense_rebuild_debounce_seconds- )self._dense_rebuild_failure = Noneself._dense_rebuild_memory_blocked = False+ self._dense_rebuild_memory_retry_attempts = 0+ self._dense_rebuild_memory_retry_not_before = 0.0self._dense_rebuild_completed.clear()+ if self._dense_rebuild_suspend_count > 0:+ self._dense_rebuild_deferred = True+ return+ self._dense_rebuild_debounce_deadline = (+ time.monotonic() + self._dense_rebuild_debounce_seconds+ )+ self._dense_rebuild_event.set()++ def begin_bulk_ingest(self) -> None:+ """Defer background GPU rebuilds while native and shadow writes continue."""++ if not self._auto_background_rebuild or self.dense_search is None:+ return+ with self._dense_rebuild_state_lock:+ self._dense_rebuild_suspend_count += 1+ if self.dense_search.needs_rebuild:+ self._dense_rebuild_deferred = True+ self._dense_rebuild_completed.clear()+ # Wake a worker that may already be inside its debounce wait so it can+ # observe the suspension before starting a build.self._dense_rebuild_event.set()+ def end_bulk_ingest(self) -> None:+ """Schedule one trailing rebuild when the outermost bulk scope exits."""++ if not self._auto_background_rebuild or self.dense_search is None:+ return+ should_wake = False+ with self._dense_rebuild_state_lock:+ if self._dense_rebuild_suspend_count <= 0:+ raise RuntimeError("bulk ingest scope is not active")+ self._dense_rebuild_suspend_count -= 1+ if self._dense_rebuild_suspend_count == 0:+ should_wake = self._dense_rebuild_deferred or self.dense_search.needs_rebuild+ self._dense_rebuild_deferred = False+ if should_wake:+ self._dense_rebuild_debounce_deadline = (+ time.monotonic() + self._dense_rebuild_debounce_seconds+ )+ self._dense_rebuild_failure = None+ self._dense_rebuild_memory_blocked = False+ self._dense_rebuild_memory_retry_attempts = 0+ self._dense_rebuild_memory_retry_not_before = 0.0+ self._dense_rebuild_completed.clear()+ if should_wake:+ self._dense_rebuild_event.set()+def _start_dense_rebuild_worker(self) -> None:"""Start the worker once native and dense initial state are aligned."""@@ -442,26 +493,52 @@ class LocalIndex(IIndex):self._dense_rebuild_thread.start()self._schedule_dense_rebuild()- def _wake_dense_rebuild_worker(self) -> None:+ def _wake_dense_rebuild_worker(self) -> bool:"""Wake the worker without extending the mutation debounce window."""+ if not self._auto_background_rebuild or self.dense_search is None:+ return False+ with self._dense_rebuild_state_lock:+ if self._dense_rebuild_memory_blocked:+ return False+ self._dense_rebuild_completed.clear()+ if self._dense_rebuild_suspend_count > 0:+ self._dense_rebuild_deferred = True+ return False+ self._dense_rebuild_event.set()+ return True++ def _rearm_dense_rebuild_after_stale_candidate(self) -> None:+ """Retry a stale build only after a fresh trailing-edge debounce."""+if not self._auto_background_rebuild or self.dense_search is None:returnwith self._dense_rebuild_state_lock:- self._dense_rebuild_memory_blocked = Falseself._dense_rebuild_completed.clear()+ if self._dense_rebuild_suspend_count > 0:+ self._dense_rebuild_deferred = True+ return+ self._dense_rebuild_debounce_deadline = max(+ self._dense_rebuild_debounce_deadline,+ time.monotonic() + self._dense_rebuild_debounce_seconds,+ )self._dense_rebuild_event.set()def _retry_memory_blocked_rebuild(self) -> bool:- """Let the first later query retry an expected memory-budget fallback."""+ """Let a later query retry memory admission after bounded backoff."""if not self._auto_background_rebuild or self.dense_search is None:return Falsewith self._dense_rebuild_state_lock:if not self._dense_rebuild_memory_blocked:return False+ if time.monotonic() < self._dense_rebuild_memory_retry_not_before:+ return Falseself._dense_rebuild_memory_blocked = Falseself._dense_rebuild_completed.clear()+ if self._dense_rebuild_suspend_count > 0:+ self._dense_rebuild_deferred = True+ return Trueself._dense_rebuild_event.set()return True@@ -479,37 +556,62 @@ class LocalIndex(IIndex):def _dense_rebuild_loop(self) -> None:while True:self._dense_rebuild_event.wait()+ self._dense_rebuild_event.clear()if self._dense_rebuild_stop.is_set():return- self._dense_rebuild_event.clear()- while self._dense_rebuild_debounce_seconds > 0:+ suspended = False+ rebuild_generation = 0+ while True:with self._dense_rebuild_state_lock:+ suspended = self._dense_rebuild_suspend_count > 0+ if suspended:+ self._dense_rebuild_deferred = (+ bool(self.dense_search is not None and self.dense_search.needs_rebuild)+ or self._dense_rebuild_deferred+ )remaining = self._dense_rebuild_debounce_deadline - time.monotonic()- if remaining <= 0:+ if not suspended and remaining <= 0:+ # Claim the generation in the same critical section as+ # the deadline decision. A mutation cannot move the+ # deadline between those two observations.+ rebuild_generation = self._dense_rebuild_generation+ if suspended:break- changed = self._dense_rebuild_event.wait(timeout=remaining)- if not changed:+ if remaining <= 0:break+ self._dense_rebuild_event.wait(timeout=remaining)self._dense_rebuild_event.clear()if self._dense_rebuild_stop.is_set():return+ # Always re-read the deadline after both notifications and+ # timeouts. A mutation can move it at the timeout boundary.++ if suspended:+ continue- with self._dense_rebuild_state_lock:- rebuild_generation = self._dense_rebuild_generationtry:- committed = self._run_background_rebuild()+ committed = self._run_background_rebuild(rebuild_generation)except CuVSMemoryBudgetError as exc:with self._dense_rebuild_state_lock:if self._dense_rebuild_generation == rebuild_generation:self._dense_rebuild_failure = Noneself._dense_rebuild_memory_blocked = True+ self._dense_rebuild_memory_retry_attempts += 1+ exponent = min(self._dense_rebuild_memory_retry_attempts - 1, 5)+ retry_delay = min(+ _DENSE_REBUILD_MEMORY_RETRY_BASE_SECONDS * (2**exponent),+ _DENSE_REBUILD_MEMORY_RETRY_MAX_SECONDS,+ )+ self._dense_rebuild_memory_retry_not_before = time.monotonic() + retry_delaylogger.debug("cuVS background rebuild kept native search: %s", exc)except Exception as exc:with self._dense_rebuild_state_lock:if self._dense_rebuild_generation == rebuild_generation:self._dense_rebuild_failure = (type(exc), str(exc))self._dense_rebuild_memory_blocked = False+ self._dense_rebuild_memory_retry_attempts = 0+ self._dense_rebuild_memory_retry_not_before = 0.0logger.warning("cuVS background rebuild failed", exc_info=True)else:if committed:@@ -517,44 +619,109 @@ class LocalIndex(IIndex):if self._dense_rebuild_generation == rebuild_generation:self._dense_rebuild_failure = Noneself._dense_rebuild_memory_blocked = False+ self._dense_rebuild_memory_retry_attempts = 0+ self._dense_rebuild_memory_retry_not_before = 0.0finally:self._dense_rebuild_completed.set()- def _run_background_rebuild(self) -> bool:+ def _run_background_rebuild(self, expected_generation: int) -> bool:dense_search = self.dense_searchif dense_search is None or self.engine_proxy is None:return False+ if self._dense_rebuild_stop.is_set():+ return False+ with self._dense_rebuild_state_lock:+ if self._dense_rebuild_suspend_count > 0:+ self._dense_rebuild_deferred = dense_search.needs_rebuild+ return False+ if self._dense_rebuild_generation != expected_generation:+ return False+candidate = dense_search.prepare_rebuild()if candidate is None:return not dense_search.needs_rebuild- if self._dense_rebuild_stop.is_set():++ with self._dense_rebuild_state_lock:+ candidate_is_stale = (+ self._dense_rebuild_generation != expected_generation+ or self._dense_rebuild_suspend_count > 0+ )+ if self._dense_rebuild_stop.is_set() or candidate_is_stale:dense_search.discard_rebuild(candidate)+ if not self._dense_rebuild_stop.is_set():+ self._rearm_dense_rebuild_after_stale_candidate()return False+with self._dense_search_lock.write():+ with self._dense_rebuild_state_lock:+ candidate_is_stale = (+ self._dense_rebuild_generation != expected_generation+ or self._dense_rebuild_suspend_count > 0+ )if (self.dense_search is not dense_searchor self.engine_proxy is Noneor self._dense_rebuild_stop.is_set()+ or candidate_is_stale):dense_search.discard_rebuild(candidate)+ if not self._dense_rebuild_stop.is_set():+ self._rearm_dense_rebuild_after_stale_candidate()return Falsecommitted = dense_search.commit_rebuild(candidate,self.engine_proxy.set_filter_layout,)if not committed:- self._wake_dense_rebuild_worker()+ self._rearm_dense_rebuild_after_stale_candidate()return committeddef _stop_dense_rebuild_worker(self) -> None:thread = self._dense_rebuild_thread- if thread is None:- return+ # Retirement is durable even if publication has not started this+ # worker yet. A concurrent replacement can otherwise stop this index+ # in the map-swap/start window, only for the earlier publisher to start+ # an orphan worker afterwards.self._dense_rebuild_stop.set()self._dense_rebuild_event.set()+ if thread is None:+ returnthread.join()self._dense_rebuild_thread = None+ def wait_for_background_rebuild(self, timeout: float = 5.0) -> bool:+ if not self._auto_background_rebuild:+ return False+ with self._dense_rebuild_state_lock:+ if self._dense_rebuild_suspend_count > 0:+ return False+ deadline = time.monotonic() + max(0.0, timeout)+ memory_retry_attempted = False+ while self.dense_search is not None and self.dense_search.needs_rebuild:+ self._raise_dense_rebuild_failure()+ with self._dense_rebuild_state_lock:+ memory_blocked = self._dense_rebuild_memory_blocked+ if memory_blocked:+ # A readiness wait may retry a previously blocked admission at+ # most once. A second rejection returns immediately instead of+ # rebuilding an O(N) host candidate in a tight timeout loop.+ if memory_retry_attempted:+ return False+ if self._retry_memory_blocked_rebuild():+ memory_retry_attempted = True+ elif not self._wake_dense_rebuild_worker():+ # A mutation may have cleared the blocked state between+ # the observation and retry. In that case the normal wake+ # path owns the newer generation; otherwise backoff holds.+ return False+ elif not self._wake_dense_rebuild_worker():+ return False+ remaining = deadline - time.monotonic()+ if remaining <= 0 or not self._dense_rebuild_completed.wait(remaining):+ return False+ self._raise_dense_rebuild_failure()+ return self.dense_search is not None+def search(self,query_vector: Optional[List[float]],@@ -979,6 +1146,7 @@ class VolatileIndex(LocalIndex):meta: Any,cands_list: Optional[List[CandidateData]] = None,dense_search_config: Optional[Dict[str, Any]] = None,+ defer_dense_rebuild_start: bool = False,):"""Initialize a volatile (in-memory) index.@@ -989,6 +1157,8 @@ class VolatileIndex(LocalIndex):meta: Index metadata containing configuration (dimensions, distance metric, etc.)cands_list (list): Initial list of CandidateData records to populate the index.Defaults to None (empty index).+ defer_dense_rebuild_start: Delay the optional background dense+ rebuild worker until collection-level publication.Note:The index is immediately built in memory with the provided data.@@ -1014,7 +1184,8 @@ class VolatileIndex(LocalIndex):self.engine_proxy.add_data(self._convert_candidate_list_for_index(cands_list))# Native add_data() invalidates its filter layout, so publish the first# dense snapshot only after the native records are present.- self._start_dense_rebuild_worker()+ if not defer_dense_rebuild_start:+ self._start_dense_rebuild_worker()def need_rebuild(self) -> bool:"""Determine if rebuild is needed.diff --git a/openviking/storage/vectordb_adapters/base.py b/openviking/storage/vectordb_adapters/base.pyindex 9d2287f34..1b69237c1 100644--- a/openviking/storage/vectordb_adapters/base.py+++ b/openviking/storage/vectordb_adapters/base.py@@ -208,6 +208,16 @@ class CollectionAdapter(ABC):self._collection.close()self._collection = None+ def begin_bulk_ingest(self) -> None:+ """Begin a derived-index maintenance suppression scope.++ Remote adapters and backends without derived indexes intentionally use+ this default no-op implementation.+ """++ def end_bulk_ingest(self) -> None:+ """End a matching bulk-ingest maintenance scope."""+def get_collection_info(self) -> Optional[Dict[str, Any]]:if not self.collection_exists():return Nonediff --git a/openviking/storage/vectordb_adapters/local_adapter.py b/openviking/storage/vectordb_adapters/local_adapter.pyindex e85528a3a..06c564e8f 100644--- a/openviking/storage/vectordb_adapters/local_adapter.py+++ b/openviking/storage/vectordb_adapters/local_adapter.py@@ -87,6 +87,12 @@ class LocalCollectionAdapter(CollectionAdapter):result = collection.update_data(data_list)return list(result.ids or [])+ def begin_bulk_ingest(self) -> None:+ self.get_collection().begin_bulk_ingest()++ def end_bulk_ingest(self) -> None:+ self.get_collection().end_bulk_ingest()+class CuVSCollectionAdapter(LocalCollectionAdapter):"""Embedded OpenViking storage with dense search executed by NVIDIA cuVS."""diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 7b1096356..203f6818e 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -6,7 +6,8 @@ from __future__ import annotationsimport asyncioimport uuid-from typing import Any, Dict, List, Optional+from contextlib import asynccontextmanager+from typing import Any, AsyncIterator, Dict, List, Optionalfrom openviking.core.namespace import canonicalize_uri, visible_rootsfrom openviking.server.identity import RequestContext, Role@@ -69,6 +70,28 @@ URI_REWRITE_OUTPUT_FIELDS = [VIKINGDB_CONTENT_MAX_SIZE = 1024 * 1024+async def _wait_for_task_completion_despite_cancellation(+ task: asyncio.Task[Any],+) -> Optional[asyncio.CancelledError]:+ """Wait for an offloaded lifecycle operation without leaking its state."""++ pending_cancellation: Optional[asyncio.CancelledError] = None+ while not task.done():+ try:+ await asyncio.shield(task)+ except asyncio.CancelledError as exc:+ # A child that cancels itself did not complete the lifecycle+ # operation and must still be reported by ``task.result()``.+ if not task.cancelled() and pending_cancellation is None:+ pending_cancellation = exc+ except BaseException:+ # Read the task's exception below so the caller can apply the+ # lifecycle-specific exception ordering.+ if not task.done():+ raise+ return pending_cancellation++class _AsyncVectorAdapter:"""Thread-offloaded facade for sync vector adapters."""@@ -353,6 +376,12 @@ class _SingleAccountBackend:)return normalized_ids+ async def begin_bulk_ingest(self) -> None:+ await self._async_adapter.call("begin_bulk_ingest")++ async def end_bulk_ingest(self) -> None:+ await self._async_adapter.call("end_bulk_ingest")+async def update(self, data: Dict[str, Any]) -> UpdateResult:"""Strict update path. The target record must already exist."""try:@@ -864,6 +893,36 @@ class VikingVectorIndexBackend:)return result+ @asynccontextmanager+ async def bulk_ingest(self, *, ctx: RequestContext) -> AsyncIterator[None]:+ """Coalesce optional derived-index rebuilds across many write calls.++ The scope is a performance hint only. It does not make the enclosed+ writes transactional or atomic, and adapters that do not maintain a+ derived local index treat it as a no-op.+ """+ backend = self._get_backend_for_context(ctx)+ begin_task = asyncio.create_task(backend.begin_bulk_ingest())+ entry_cancellation = await _wait_for_task_completion_despite_cancellation(begin_task)+ # A failed or self-cancelled begin did not acquire the scope, so it+ # must not be balanced with an end call.+ begin_task.result()+ if entry_cancellation is not None:+ end_task = asyncio.create_task(backend.end_bulk_ingest())+ await _wait_for_task_completion_despite_cancellation(end_task)+ # Cleanup failures take priority because they mean the suspension+ # may still be live. Otherwise preserve the original cancellation.+ end_task.result()+ raise entry_cancellation+ try:+ yield+ finally:+ end_task = asyncio.create_task(backend.end_bulk_ingest())+ exit_cancellation = await _wait_for_task_completion_despite_cancellation(end_task)+ end_task.result()+ if exit_cancellation is not None:+ raise exit_cancellation+async def update(self, data: Dict[str, Any], *, ctx: RequestContext) -> UpdateResult:"""Strict update path. The target record must already exist."""logger.debug(diff --git a/openviking/storage/vikingdb_manager.py b/openviking/storage/vikingdb_manager.pyindex 3231f6c4f..f1b074bca 100644--- a/openviking/storage/vikingdb_manager.py+++ b/openviking/storage/vikingdb_manager.py@@ -4,7 +4,8 @@VikingDB Manager class that extends VikingVectorIndexBackend with queue management functionality."""-from typing import Any, Dict, List, Optional, Tuple+from contextlib import asynccontextmanager+from typing import Any, AsyncIterator, Dict, List, Optional, Tuplefrom openviking.server.identity import RequestContextfrom openviking.storage.expr import FilterExpr@@ -298,6 +299,12 @@ class VikingDBManagerProxy:"""Bulk full-record upsert with the proxy's bound request context."""return await self._manager.upsert_many(data_list, ctx=self._ctx)+ @asynccontextmanager+ async def bulk_ingest(self) -> AsyncIterator[None]:+ """Bind the proxy context to a bulk-ingest maintenance scope."""+ async with self._manager.bulk_ingest(ctx=self._ctx):+ yield+async def get(self, ids: List[str]) -> List[Dict[str, Any]]:return await self._manager.get(ids, ctx=self._ctx)diff --git a/tests/benchmark/test_vectordb_perf_run.py b/tests/benchmark/test_vectordb_perf_run.pyindex 76bc97a33..8d4e07fd3 100644--- a/tests/benchmark/test_vectordb_perf_run.py+++ b/tests/benchmark/test_vectordb_perf_run.py@@ -16,6 +16,7 @@ from benchmark.vectordb_perf.run import (recall_scope,record_id,upsert_records,+ wait_for_auto_derived_index,)@@ -98,6 +99,124 @@ def _write_arxiv_public_shape_fixture(tmp_path):_write_fvecs(tmp_path / "arxiv_query_vectors.fvecs", vectors)+def run_to_thread_inline(monkeypatch):+ async def run_inline(fn):+ return fn()++ monkeypatch.setattr("benchmark.vectordb_perf.run.asyncio.to_thread", run_inline)+++async def test_wait_for_auto_derived_index_uses_configured_index_and_timeout(+ run_to_thread_inline,+):+ del run_to_thread_inline+ calls = []++ class _Index:+ def wait_for_background_rebuild(self, *, timeout):+ calls.append(("wait_for_background_rebuild", timeout))+ return True++ index = _Index()++ class _Collection:+ def get_index(self, index_name):+ calls.append(("get_index", index_name))+ return index++ class _Adapter:+ index_name = "configured-index"++ def get_collection(self):+ calls.append(("get_collection",))+ return _Collection()++ class _Backend:+ _shared_adapter = _Adapter()++ assert await wait_for_auto_derived_index(_Backend(), timeout=12.5) is True+ assert calls == [+ ("get_collection",),+ ("get_index", "configured-index"),+ ("wait_for_background_rebuild", 12.5),+ ]+++async def test_wait_for_auto_derived_index_converts_false_to_timeout_error(+ run_to_thread_inline,+):+ del run_to_thread_inline++ class _Index:+ def wait_for_background_rebuild(self, *, timeout):+ assert timeout == 7+ return False++ class _Collection:+ def get_index(self, index_name):+ assert index_name == "default"+ return _Index()++ class _Adapter:+ index_name = "default"++ def get_collection(self):+ return _Collection()++ class _Backend:+ _shared_adapter = _Adapter()++ with pytest.raises(TimeoutError, match="not ready within 7s"):+ await wait_for_auto_derived_index(_Backend(), timeout=7)+++async def test_wait_for_auto_derived_index_rejects_missing_index(run_to_thread_inline):+ del run_to_thread_inline++ class _Collection:+ def get_index(self, index_name):+ assert index_name == "missing-index"+ return None++ class _Adapter:+ index_name = "missing-index"++ def get_collection(self):+ return _Collection()++ class _Backend:+ _shared_adapter = _Adapter()++ with pytest.raises(RuntimeError, match="benchmark index not found: missing-index"):+ await wait_for_auto_derived_index(_Backend())+++async def test_wait_for_auto_derived_index_rejects_unsupported_index(run_to_thread_inline):+ del run_to_thread_inline++ class _Collection:+ def get_index(self, index_name):+ assert index_name == "native-only"+ return object()++ class _Adapter:+ index_name = "native-only"++ def get_collection(self):+ return _Collection()++ class _Backend:+ _shared_adapter = _Adapter()++ with pytest.raises(RuntimeError, match="does not expose background rebuild readiness"):+ await wait_for_auto_derived_index(_Backend())++@pytest.mark.asyncioasync def test_upsert_records_uses_one_bulk_backend_call():calls = []diff --git a/tests/storage/test_bulk_upsert.py b/tests/storage/test_bulk_upsert.pyindex 80ffdba2a..4702767b3 100644--- a/tests/storage/test_bulk_upsert.py+++ b/tests/storage/test_bulk_upsert.py@@ -1,6 +1,8 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+import asyncio+import threadingimport uuidfrom types import SimpleNamespace@@ -199,6 +201,123 @@ async def test_viking_vector_index_backend_upsert_many_delegates_once():assert calls == [records]+async def test_viking_vector_index_backend_bulk_ingest_balances_scope_on_error():+ backend = object.__new__(VikingVectorIndexBackend)+ ctx = SimpleNamespace(account_id="acc1")+ calls = []++ class _BoundBackend:+ async def begin_bulk_ingest(self):+ calls.append("begin")++ async def end_bulk_ingest(self):+ calls.append("end")++ bound_backend = _BoundBackend()+ backend._get_backend_for_context = lambda _ctx: bound_backend++ with pytest.raises(RuntimeError, match="injected"):+ async with backend.bulk_ingest(ctx=ctx):+ calls.append("body")+ raise RuntimeError("injected")++ assert calls == ["begin", "body", "end"]+++async def test_bulk_ingest_cancellation_during_threaded_begin_balances_scope():+ backend = object.__new__(VikingVectorIndexBackend)+ ctx = SimpleNamespace(account_id="acc1")+ begin_started = threading.Event()+ allow_begin = threading.Event()+ begin_finished = threading.Event()+ calls = []++ class _BoundBackend:+ async def begin_bulk_ingest(self):+ def blocking_begin():+ begin_started.set()+ assert allow_begin.wait(timeout=5)+ calls.append("begin")+ begin_finished.set()++ await asyncio.to_thread(blocking_begin)++ async def end_bulk_ingest(self):+ calls.append("end")++ backend._get_backend_for_context = lambda _ctx: _BoundBackend()++ async def enter_scope():+ async with backend.bulk_ingest(ctx=ctx):+ calls.append("body")++ task = asyncio.create_task(enter_scope())+ try:+ assert await asyncio.to_thread(begin_started.wait, 5)+ task.cancel()+ await asyncio.sleep(0)+ task.cancel()+ allow_begin.set()+ assert await asyncio.to_thread(begin_finished.wait, 5)+ with pytest.raises(asyncio.CancelledError):+ await task+ assert calls == ["begin", "end"]+ finally:+ allow_begin.set()+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+++async def test_bulk_ingest_cancellation_during_threaded_end_waits_for_cleanup():+ backend = object.__new__(VikingVectorIndexBackend)+ ctx = SimpleNamespace(account_id="acc1")+ end_started = threading.Event()+ allow_end = threading.Event()+ end_finished = threading.Event()+ calls = []++ class _BoundBackend:+ async def begin_bulk_ingest(self):+ calls.append("begin")++ async def end_bulk_ingest(self):+ def blocking_end():+ end_started.set()+ assert allow_end.wait(timeout=5)+ calls.append("end")+ end_finished.set()++ await asyncio.to_thread(blocking_end)++ backend._get_backend_for_context = lambda _ctx: _BoundBackend()++ async def use_scope():+ async with backend.bulk_ingest(ctx=ctx):+ calls.append("body")++ task = asyncio.create_task(use_scope())+ try:+ assert await asyncio.to_thread(end_started.wait, 5)+ task.cancel()+ await asyncio.sleep(0)+ task.cancel()+ assert not task.done()+ allow_end.set()+ assert await asyncio.to_thread(end_finished.wait, 5)+ with pytest.raises(asyncio.CancelledError):+ await task+ assert calls == ["begin", "body", "end"]+ finally:+ allow_end.set()+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)++@pytest.mark.asyncioasync def test_viking_vector_index_backend_upsert_many_persists_batch(tmp_path):collection_name = "bulk_upsert_test"@@ -266,3 +385,31 @@ async def test_vikingdb_manager_proxy_upsert_many_forwards_bound_context():assert await proxy.upsert_many(records) == ["rec-1", "rec-2"]assert captured == {"data_list": records, "ctx": ctx}+++async def test_vikingdb_manager_proxy_bulk_ingest_forwards_bound_context():+ from contextlib import asynccontextmanager++ from openviking.storage.vikingdb_manager import VikingDBManagerProxy++ ctx = SimpleNamespace(account_id="acc1")+ calls = []++ class _Manager:+ collection_name = "context"+ mode = "local"++ @asynccontextmanager+ async def bulk_ingest(self, *, ctx):+ calls.append(("begin", ctx))+ try:+ yield+ finally:+ calls.append(("end", ctx))++ proxy = VikingDBManagerProxy(_Manager(), ctx)+ async with proxy.bulk_ingest():+ calls.append(("body", ctx))++ assert calls == [("begin", ctx), ("body", ctx), ("end", ctx)]diff --git a/tests/vectordb/test_cuvs_collection.py b/tests/vectordb/test_cuvs_collection.pyindex 96919ba53..9b6536fd4 100644--- a/tests/vectordb/test_cuvs_collection.py+++ b/tests/vectordb/test_cuvs_collection.py@@ -430,6 +430,479 @@ def test_auto_cuvs_falls_back_then_retries_when_memory_is_available(monkeypatch)collection.close()+def test_auto_cuvs_background_memory_fallback_does_not_busy_loop(monkeypatch):+ class CountingMemoryRuntime(MemoryAwareFakeCuVSRuntime):+ def __init__(self):+ super().__init__("inner_product", free_memory_bytes=31)+ self.memory_info_count = 0+ self.memory_checked = threading.Event()++ def memory_info(self):+ self.memory_info_count += 1+ self.memory_checked.set()+ return super().memory_info()++ runtime = CountingMemoryRuntime()+ monkeypatch.setattr(+ cuvs_index,+ "_CuVSRuntime",+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,+ )+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "auto_cuvs_memory_backoff",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ config={+ "dense_search": {+ "backend": "auto_cuvs",+ "algorithm": "brute_force",+ "filter_cache_size": 0,+ "auto_memory_reserve_mb": 0,+ "auto_memory_safety_factor": 1.0,+ "auto_background_rebuild": True,+ "auto_rebuild_debounce_ms": 0,+ }+ },+ )+ try:+ index = collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ assert index.wait_for_background_rebuild(timeout=5)++ collection.upsert_data(+ [+ {"id": "first", "vector": [1.0, 0.0, 0.0, 0.0]},+ {"id": "second", "vector": [0.0, 1.0, 0.0, 0.0]},+ ]+ )+ assert runtime.memory_checked.wait(timeout=5)+ assert index._dense_rebuild_completed.wait(timeout=5)+ with index._dense_rebuild_state_lock:+ assert index._dense_rebuild_memory_blocked+ index._dense_rebuild_memory_retry_not_before = float("inf")+ memory_checks_after_failure = runtime.memory_info_count++ assert not index.wait_for_background_rebuild(timeout=0.1)+ assert runtime.memory_info_count == memory_checks_after_failure+ assert runtime.build_count == 0++ runtime.free_memory_bytes = 32+ with index._dense_rebuild_state_lock:+ index._dense_rebuild_memory_retry_not_before = 0.0+ result = collection.search_by_vector(+ "default",+ dense_vector=[1.0, 0.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["first"]+ assert index.wait_for_background_rebuild(timeout=5)+ result = collection.search_by_vector(+ "default",+ dense_vector=[1.0, 0.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["first"]+ assert runtime.build_count == 1+ assert runtime.search_count == 1+ finally:+ collection.close()+++def test_auto_cuvs_bulk_ingest_defers_nested_rebuild_and_routes_native(monkeypatch):+ class BuildStartedRuntime(MemoryAwareFakeCuVSRuntime):+ def __init__(self):+ super().__init__("inner_product", free_memory_bytes=1 << 20)+ self.build_started = threading.Event()++ def build(self, dataset):+ self.build_started.set()+ return super().build(dataset)++ runtime = BuildStartedRuntime()+ monkeypatch.setattr(+ cuvs_index,+ "_CuVSRuntime",+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,+ )+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "auto_cuvs_bulk_ingest",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ config={+ "dense_search": {+ "backend": "auto_cuvs",+ "algorithm": "brute_force",+ "filter_cache_size": 0,+ "auto_memory_reserve_mb": 0,+ "auto_memory_safety_factor": 1.0,+ "auto_background_rebuild": True,+ "auto_rebuild_debounce_ms": 100,+ }+ },+ )+ try:+ index = collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ assert index.wait_for_background_rebuild(timeout=5)+ assert runtime.build_count == 0++ collection.begin_bulk_ingest()+ try:+ collection.upsert_data([{"id": "first", "vector": [1.0, 0.0, 0.0, 0.0]}])+ # The scope remains open for longer than the normal debounce, but+ # no partial GPU index should be published.+ assert not runtime.build_started.wait(timeout=0.25)++ collection.begin_bulk_ingest()+ try:+ collection.upsert_data([{"id": "second", "vector": [0.0, 1.0, 0.0, 0.0]}])+ assert not runtime.build_started.wait(timeout=0.25)+ finally:+ collection.end_bulk_ingest()++ # Ending the inner scope must not resume rebuilds early. Dirty+ # reads stay correct by using the continuously updated native index.+ assert not runtime.build_started.wait(timeout=0.25)+ inner_collection = collection._Collection__collection+ inner_collection._rebuild_index("default", index)+ replacement_index = collection.get_index("default")+ assert replacement_index is not index+ assert index._dense_rebuild_thread is None+ assert replacement_index._dense_rebuild_thread is not None+ assert not runtime.build_started.wait(timeout=0.25)+ result = collection.search_by_vector(+ "default",+ dense_vector=[0.0, 1.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["second"]+ assert runtime.search_count == 0+ finally:+ collection.end_bulk_ingest()++ assert runtime.build_started.wait(timeout=1)+ assert replacement_index.wait_for_background_rebuild(timeout=5)+ assert runtime.build_count == 1+ finally:+ collection.close()+++def test_index_install_keeps_post_publication_worker_failures_out_of_commit_result(monkeypatch):+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "replacement_worker_failure",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ }+ )+ inner_collection = collection._Collection__collection+ old_index = object()+ replacement_index = object()+ calls = []+ inner_collection.indexes.set("default", old_index)++ def failing_stop(index):+ calls.append(("stop", index))+ raise RuntimeError("injected stop failure")++ def tracked_start(index):+ calls.append(("start", index))++ monkeypatch.setattr(inner_collection, "_stop_index_background_rebuild", failing_stop)+ monkeypatch.setattr(inner_collection, "_start_index_background_rebuild", tracked_start)+ try:+ expected_generation = inner_collection._index_mutation_barrier.snapshot_generation()+ assert inner_collection._install_index(+ "default",+ replacement_index,+ expected_generation=expected_generation,+ expected_replaced=old_index,+ start_background_rebuild=True,+ )+ assert inner_collection.indexes.get("default") is replacement_index+ assert calls == [("stop", old_index), ("start", replacement_index)]+ finally:+ # Plain objects intentionally stand in for index lifecycle hooks above.+ inner_collection.indexes.clear()+ collection.close()+++def test_auto_cuvs_stop_before_deferred_worker_start_is_durable(monkeypatch):+ runtime = MemoryAwareFakeCuVSRuntime("inner_product", free_memory_bytes=1 << 20)+ monkeypatch.setattr(+ cuvs_index,+ "_CuVSRuntime",+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,+ )+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "auto_cuvs_deferred_worker_retirement",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ config={+ "dense_search": {+ "backend": "auto_cuvs",+ "algorithm": "brute_force",+ "auto_memory_reserve_mb": 0,+ "auto_memory_safety_factor": 1.0,+ "auto_background_rebuild": True,+ }+ },+ )+ inner_collection = collection._Collection__collection+ index = inner_collection._new_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ [],+ defer_dense_rebuild_start=True,+ )+ try:+ assert index._dense_rebuild_thread is None+ assert not index._dense_rebuild_stop.is_set()++ inner_collection._stop_index_background_rebuild(index)+ inner_collection._start_index_background_rebuild(index)++ assert index._dense_rebuild_stop.is_set()+ assert index._dense_rebuild_thread is None+ assert runtime.build_count == 0+ finally:+ index.close()+ collection.close()+++def test_auto_cuvs_rebuild_discards_snapshot_stale_after_concurrent_write(monkeypatch):+ runtime = MemoryAwareFakeCuVSRuntime("inner_product", free_memory_bytes=1 << 20)+ monkeypatch.setattr(+ cuvs_index,+ "_CuVSRuntime",+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,+ )+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "auto_cuvs_stale_collection_rebuild",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ config={+ "dense_search": {+ "backend": "auto_cuvs",+ "algorithm": "brute_force",+ "filter_cache_size": 0,+ "auto_memory_reserve_mb": 0,+ "auto_memory_safety_factor": 1.0,+ "auto_background_rebuild": True,+ "auto_rebuild_debounce_ms": 10,+ }+ },+ )+ rebuild_thread = None+ writer_thread = None+ allow_rebuild = threading.Event()+ allow_index_update = threading.Event()+ try:+ original_index = collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ assert original_index.wait_for_background_rebuild(timeout=5)++ collection.begin_bulk_ingest()+ try:+ collection.upsert_data([{"id": "before", "vector": [1.0, 0.0, 0.0, 0.0]}])+ inner_collection = collection._Collection__collection+ original_new_index = inner_collection._new_index+ original_install_index = inner_collection._install_index+ original_upsert_data = original_index.upsert_data+ snapshot_captured = threading.Event()+ store_committed = threading.Event()+ publication_attempted = threading.Event()+ writer_errors = []++ def delayed_new_index(*args, **kwargs):+ snapshot_captured.set()+ assert allow_rebuild.wait(timeout=5)+ return original_new_index(*args, **kwargs)++ def delayed_index_update(delta_list):+ store_committed.set()+ assert allow_index_update.wait(timeout=5)+ return original_upsert_data(delta_list)++ def tracked_install(*args, **kwargs):+ publication_attempted.set()+ return original_install_index(*args, **kwargs)++ def write_after_snapshot():+ try:+ collection.upsert_data([{"id": "kept", "vector": [0.0, 1.0, 0.0, 0.0]}])+ except BaseException as exc:+ writer_errors.append(exc)++ monkeypatch.setattr(inner_collection, "_new_index", delayed_new_index)+ monkeypatch.setattr(inner_collection, "_install_index", tracked_install)+ monkeypatch.setattr(original_index, "upsert_data", delayed_index_update)+ rebuild_thread = threading.Thread(+ target=inner_collection._rebuild_index,+ args=("default", original_index),+ )+ rebuild_thread.start()+ assert snapshot_captured.wait(timeout=5)++ # This batch commits after the replacement's Store snapshot. The+ # old index receives it; the stale replacement must not retire that+ # only searchable copy.+ writer_thread = threading.Thread(target=write_after_snapshot)+ writer_thread.start()+ assert store_committed.wait(timeout=5)+ allow_rebuild.set()+ assert publication_attempted.wait(timeout=5)+ assert rebuild_thread.is_alive()+ assert collection.get_index("default") is original_index++ allow_index_update.set()+ writer_thread.join(timeout=5)+ rebuild_thread.join(timeout=5)+ assert not writer_thread.is_alive()+ assert not rebuild_thread.is_alive()+ assert writer_errors == []+ assert collection.get_index("default") is original_index++ fetched = collection.fetch_data(["before", "kept"])+ assert [item.id for item in fetched.items] == ["before", "kept"]+ result = collection.search_by_vector(+ "default",+ dense_vector=[0.0, 1.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["kept"]+ assert runtime.search_count == 0+ finally:+ allow_rebuild.set()+ allow_index_update.set()+ if writer_thread is not None:+ writer_thread.join(timeout=5)+ if rebuild_thread is not None:+ rebuild_thread.join(timeout=5)+ collection.end_bulk_ingest()++ assert original_index.wait_for_background_rebuild(timeout=5)+ result = collection.search_by_vector(+ "default",+ dense_vector=[0.0, 1.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["kept"]+ assert runtime.search_count == 1+ finally:+ collection.close()+++def test_auto_cuvs_bulk_ingest_delete_all_replacement_stays_suspended(monkeypatch):+ class BuildStartedRuntime(MemoryAwareFakeCuVSRuntime):+ def __init__(self):+ super().__init__("inner_product", free_memory_bytes=1 << 20)+ self.build_started = threading.Event()++ def build(self, dataset):+ self.build_started.set()+ return super().build(dataset)++ runtime = BuildStartedRuntime()+ monkeypatch.setattr(+ cuvs_index,+ "_CuVSRuntime",+ lambda _algorithm, _metric, _build_params, _search_params, _dtype: runtime,+ )+ collection = get_or_create_local_collection(+ meta_data={+ "CollectionName": "auto_cuvs_bulk_ingest_delete_all",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ ],+ },+ config={+ "dense_search": {+ "backend": "auto_cuvs",+ "algorithm": "brute_force",+ "filter_cache_size": 0,+ "auto_memory_reserve_mb": 0,+ "auto_memory_safety_factor": 1.0,+ "auto_background_rebuild": True,+ "auto_rebuild_debounce_ms": 100,+ }+ },+ )+ try:+ original_index = collection.create_index(+ "default",+ {+ "IndexName": "default",+ "VectorIndex": {"IndexType": "flat", "Distance": "cosine"},+ },+ )+ assert original_index.wait_for_background_rebuild(timeout=5)++ collection.begin_bulk_ingest()+ try:+ collection.upsert_data([{"id": "removed", "vector": [1.0, 0.0, 0.0, 0.0]}])+ collection.delete_all_data()+ replacement_index = collection.get_index("default")+ assert replacement_index is not original_index++ collection.upsert_data([{"id": "kept", "vector": [0.0, 1.0, 0.0, 0.0]}])+ # delete_all_data replaces the index while the scope is active.+ # The replacement must inherit suspension for longer than debounce.+ assert not runtime.build_started.wait(timeout=0.25)+ result = collection.search_by_vector(+ "default",+ dense_vector=[0.0, 1.0, 0.0, 0.0],+ limit=1,+ )+ assert [item.id for item in result.data] == ["kept"]+ assert runtime.search_count == 0+ finally:+ collection.end_bulk_ingest()++ assert runtime.build_started.wait(timeout=1)+ assert replacement_index.wait_for_background_rebuild(timeout=5)+ assert runtime.build_count == 1+ finally:+ collection.close()++def test_auto_cuvs_selective_first_query_skips_gpu_build(monkeypatch):runtimes = []dense_search_calls = 0diff --git a/tests/vectordb/test_cuvs_index.py b/tests/vectordb/test_cuvs_index.pyindex 0cf878639..a67930fa1 100644--- a/tests/vectordb/test_cuvs_index.py+++ b/tests/vectordb/test_cuvs_index.py@@ -957,6 +957,44 @@ def test_auto_cuvs_retries_after_gpu_memory_becomes_available():assert admitted.memory_free_bytes == 16+def test_auto_cuvs_checks_memory_before_materializing_host_dataset():+ runtime = FakeCuVSRuntime()+ runtime.free_memory_bytes = 15+ index = CuVSDenseIndex(+ dimension=2,+ distance="ip",+ normalize_vectors=False,+ field_types={},+ config={+ "algorithm": "brute_force",+ "filter_cache_size": 0,+ "auto_memory_reserve_mb": 0,+ "auto_memory_safety_factor": 1.0,+ },+ runtime=runtime,+ auto_memory=True,+ )+ index.add_candidates([candidate(1, [1.0, 0.0]), candidate(2, [0.0, 1.0])])+ vector_accesses = []++ class _TrackedRecord:+ fields = {}++ @property+ def vector(self):+ vector_accesses.append(True)+ raise AssertionError("host dataset was materialized before memory admission")++ with index._lock:+ index._records = {label: _TrackedRecord() for label in index._records}++ with pytest.raises(CuVSMemoryBudgetError, match="estimated GPU peak"):+ index.prepare_rebuild()++ assert vector_accesses == []+ assert runtime.build_count == 0++def test_auto_cuvs_converts_gpu_allocation_failure_to_native_fallback_signal():class OutOfMemoryRuntime(FakeCuVSRuntime):def build(self, _dataset):