Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
#!/usr/bin/env python3
"""Benchmark OpenViking collection-level native and cuVS vector search.
Unlike the index microbenchmark, this harness goes through CollectionAdapter
and therefore includes filter compilation, label mapping, record lookup, and
result normalization. It also measures the lazy rebuild paid by the first
query after upsert, delete, and process restart.
"""
from __future__ import annotations
import argparse
import gc
import json
import shutil
import statistics
import sys
import tempfile
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Sequence
import numpy as np
BENCHMARK_DIR = Path(__file__).resolve().parent
if str(BENCHMARK_DIR) not in sys.path:
sys.path.insert(0, str(BENCHMARK_DIR))
from run_index_benchmark import ( # noqa: E402
current_rss_bytes,
percentile,
prepare_dataset,
runtime_metadata,
)
from openviking.storage.vectordb_adapters.local_adapter import ( # noqa: E402
CuVSCollectionAdapter,
LocalCollectionAdapter,
)
SUPPORTED_BACKENDS = ("native", "cuvs_brute_force", "auto_cuvs")
@dataclass(frozen=True)
class FilterScenario:
name: str
filter: dict[str, Any] | None
distribution: str
selectivity: float
def parse_positive_int_list(value: str, option: str) -> list[int]:
try:
parsed = [int(item.strip()) for item in value.split(",") if item.strip()]
except ValueError as exc:
raise argparse.ArgumentTypeError(f"{option} must contain integers") from exc
if not parsed or any(item <= 0 for item in parsed):
raise argparse.ArgumentTypeError(f"{option} must contain positive integers")
if len(parsed) != len(set(parsed)):
raise argparse.ArgumentTypeError(f"{option} cannot contain duplicates")
return parsed
def filter_scenarios() -> list[FilterScenario]:
scenarios = [FilterScenario("unfiltered", None, "none", 1.0)]
for distribution, field in (
("uniform", "uniform_bucket"),
("clustered", "cluster_bucket"),
):
for label, upper_bound, selectivity in (
("10pct", 100, 0.10),
("1pct", 10, 0.01),
("0_1pct", 1, 0.001),
):
scenarios.append(
FilterScenario(
name=f"{distribution}_{label}",
filter={"op": "range", "field": field, "lt": upper_bound},
distribution=distribution,
selectivity=selectivity,
)
)
for label, prefix, selectivity in (
("10pct", "/docs/g0", 0.10),
("1pct", "/docs/g0/h0", 0.01),
("0_1pct", "/docs/g0/h0/i0", 0.001),
):
scenarios.append(
FilterScenario(
name=f"path_{label}",
filter={
"op": "must",
"field": "uri",
"conds": [prefix],
"para": "-d=-1",
},
distribution="path",
selectivity=selectivity,
)
)
return scenarios
def prebuild_selective_scenario(vector_count: int, native_threshold: int) -> FilterScenario | None:
"""Return a filter that should route native before the first GPU build."""
eligible_count = (vector_count + 999) // 1000
if native_threshold <= 0 or eligible_count > native_threshold:
return None
return FilterScenario(
name="prebuild_selective_0_1pct",
filter={"op": "must", "field": "uniform_bucket", "conds": [0]},
distribution="uniform",
selectivity=eligible_count / vector_count,
)
def scalar_fields(row_index: int, vector_count: int) -> tuple[int, int]:
uniform_bucket = row_index % 1000
cluster_bucket = min(999, (row_index * 1000) // vector_count)
return uniform_bucket, cluster_bucket
def make_record(
row_index: int,
vector: Sequence[float],
vector_count: int,
) -> dict[str, Any]:
uniform_bucket, cluster_bucket = scalar_fields(row_index, vector_count)
return {
"id": row_index + 1,
"vector": list(vector),
"uniform_bucket": uniform_bucket,
"cluster_bucket": cluster_bucket,
"uri": (
f"/docs/g{row_index % 10}/h{(row_index // 10) % 10}/"
f"i{(row_index // 100) % 10}/item-{row_index}"
),
}
def make_adapter(
backend: str,
project_path: Path,
collection_name: str,
*,
filter_cache_size: int,
auto_filter_native_threshold: int,
auto_path_filter_native_threshold: int,
):
if backend == "native":
return LocalCollectionAdapter(
collection_name=collection_name,
project_path=str(project_path),
index_name="default",
)
if backend == "cuvs_brute_force":
return CuVSCollectionAdapter(
collection_name=collection_name,
project_path=str(project_path),
index_name="default",
cuvs_config={
"algorithm": "brute_force",
"fallback_to_native": False,
"filter_cache_size": filter_cache_size,
},
)
if backend == "auto_cuvs":
return LocalCollectionAdapter(
collection_name=collection_name,
project_path=str(project_path),
index_name="default",
collection_config={
"dense_search": {
"backend": "auto_cuvs",
"algorithm": "brute_force",
"fallback_to_native": True,
"filter_cache_size": filter_cache_size,
"auto_filter_native_threshold": auto_filter_native_threshold,
"auto_path_filter_native_threshold": auto_path_filter_native_threshold,
}
},
)
raise ValueError(f"Unsupported backend: {backend}")
def collection_schema(collection_name: str, dimension: int) -> dict[str, Any]:
return {
"CollectionName": collection_name,
"Fields": [
{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
{"FieldName": "vector", "FieldType": "vector", "Dim": dimension},
{"FieldName": "uniform_bucket", "FieldType": "int64"},
{"FieldName": "cluster_bucket", "FieldType": "int64"},
{"FieldName": "uri", "FieldType": "path"},
],
"ScalarIndex": ["uniform_bucket", "cluster_bucket", "uri"],
}
def gpu_memory_used_bytes() -> int | None:
try:
import cupy as cp
free, total = cp.cuda.runtime.memGetInfo()
return int(total - free)
except Exception:
return None
def summarize_latency(latencies_ms: Sequence[float], query_count: int) -> dict[str, Any]:
total_seconds = sum(latencies_ms) / 1000.0
return {
"query_count": query_count,
"total_seconds": total_seconds,
"qps": query_count / total_seconds if total_seconds else 0.0,
"latency_ms": {
"mean": statistics.fmean(latencies_ms),
"p50": percentile(latencies_ms, 0.50),
"p95": percentile(latencies_ms, 0.95),
"p99": percentile(latencies_ms, 0.99),
},
"raw_latency_ms": list(latencies_ms),
}
def result_ids(records: Sequence[dict[str, Any]]) -> list[int]:
return [int(record["id"]) for record in records]
def run_search_scenario(
adapter: Any,
queries: np.ndarray,
*,
scenario: FilterScenario,
k: int,
warmup_queries: int,
) -> dict[str, Any]:
query_lists = [row.tolist() for row in queries]
first_started = time.perf_counter()
first_records = adapter.query(
query_vector=query_lists[0],
filter=scenario.filter,
limit=k,
output_fields=["id"],
)
first_query_ms = (time.perf_counter() - first_started) * 1000.0
for index in range(warmup_queries):
adapter.query(
query_vector=query_lists[index % len(query_lists)],
filter=scenario.filter,
limit=k,
output_fields=["id"],
)
latencies_ms: list[float] = []
neighbors: list[list[int]] = []
for query in query_lists:
started = time.perf_counter()
records = adapter.query(
query_vector=query,
filter=scenario.filter,
limit=k,
output_fields=["id"],
)
latencies_ms.append((time.perf_counter() - started) * 1000.0)
neighbors.append(result_ids(records))
return {
"name": scenario.name,
"filter": scenario.filter,
"distribution": scenario.distribution,
"target_selectivity": scenario.selectivity,
"first_query_ms": first_query_ms,
"first_result_count": len(first_records),
"neighbors": neighbors,
"search": summarize_latency(latencies_ms, len(query_lists)),
}
def run_single_query_scenario(
adapter: Any,
query: Sequence[float],
*,
scenario: FilterScenario,
k: int,
) -> dict[str, Any]:
gpu_before = gpu_memory_used_bytes()
started = time.perf_counter()
records = adapter.query(
query_vector=list(query),
filter=scenario.filter,
limit=k,
output_fields=["id"],
)
latency_ms = (time.perf_counter() - started) * 1000.0
gpu_after = gpu_memory_used_bytes()
return {
"name": scenario.name,
"filter": scenario.filter,
"distribution": scenario.distribution,
"target_selectivity": scenario.selectivity,
"latency_ms": latency_ms,
"result_count": len(records),
"gpu_before_bytes": gpu_before,
"gpu_after_bytes": gpu_after,
"gpu_delta_bytes": (
gpu_after - gpu_before if gpu_before is not None and gpu_after is not None else None
),
}
def recall_at_k(
actual: Sequence[Sequence[int]], expected: Sequence[Sequence[int]], k: int
) -> float:
if len(actual) != len(expected):
raise ValueError("Actual and expected query counts differ")
per_query: list[float] = []
for actual_row, expected_row in zip(actual, expected, strict=True):
denominator = min(k, len(expected_row))
if denominator == 0:
per_query.append(1.0 if not actual_row else 0.0)
continue
overlap = len(set(actual_row[:k]).intersection(expected_row[:k]))
per_query.append(overlap / denominator)
return statistics.fmean(per_query) if per_query else 1.0
def batched_upsert(adapter: Any, records: Sequence[dict[str, Any]], batch_size: int) -> float:
api_seconds = 0.0
for start in range(0, len(records), batch_size):
started = time.perf_counter()
adapter.upsert(list(records[start : start + batch_size]))
api_seconds += time.perf_counter() - started
return api_seconds
def ingest_dataset(
adapter: Any,
dataset: np.ndarray,
*,
batch_size: int,
) -> dict[str, Any]:
started = time.perf_counter()
api_seconds = 0.0
for start in range(0, dataset.shape[0], batch_size):
stop = min(start + batch_size, dataset.shape[0])
records = [
make_record(index, dataset[index].tolist(), dataset.shape[0])
for index in range(start, stop)
]
api_seconds += batched_upsert(adapter, records, batch_size)
total_seconds = time.perf_counter() - started
return {
"api_seconds": api_seconds,
"total_seconds": total_seconds,
"records_per_second": dataset.shape[0] / total_seconds if total_seconds else 0.0,
}
def mutation_records(
dataset: np.ndarray,
queries: np.ndarray,
count: int,
) -> list[dict[str, Any]]:
records = []
for index in range(count):
# Reuse normalized query vectors so the update changes dense ranking.
vector = queries[index % queries.shape[0]].tolist()
records.append(make_record(index, vector, dataset.shape[0]))
return records
def time_query(adapter: Any, query: Sequence[float], k: int) -> tuple[float, list[int]]:
started = time.perf_counter()
records = adapter.query(query_vector=list(query), limit=k, output_fields=["id"])
return (time.perf_counter() - started) * 1000.0, result_ids(records)
def run_mutation_lifecycle(
adapter: Any,
dataset: np.ndarray,
queries: np.ndarray,
*,
mutation_sizes: Sequence[int],
delete_count: int,
batch_size: int,
k: int,
) -> dict[str, Any]:
updates = []
for count in mutation_sizes:
if count > dataset.shape[0]:
continue
records = mutation_records(dataset, queries, count)
wall_started = time.perf_counter()
api_seconds = batched_upsert(adapter, records, batch_size)
write_seconds = time.perf_counter() - wall_started
next_query_ms, next_ids = time_query(adapter, queries[0], k)
warm_query_ms, _ = time_query(adapter, queries[0], k)
updates.append(
{
"count": count,
"write_api_seconds": api_seconds,
"write_wall_seconds": write_seconds,
"next_query_ms": next_query_ms,
"warm_query_ms": warm_query_ms,
"next_result_count": len(next_ids),
}
)
del records
gc.collect()
actual_delete_count = min(delete_count, dataset.shape[0])
delete_ids = list(range(1, actual_delete_count + 1))
delete_started = time.perf_counter()
deleted = adapter.delete(ids=delete_ids)
delete_seconds = time.perf_counter() - delete_started
delete_next_query_ms, delete_next_ids = time_query(adapter, queries[0], k)
delete_warm_query_ms, _ = time_query(adapter, queries[0], k)
return {
"updates": updates,
"delete": {
"count": deleted,
"write_seconds": delete_seconds,
"next_query_ms": delete_next_query_ms,
"warm_query_ms": delete_warm_query_ms,
"next_result_count": len(delete_next_ids),
},
}
def run_backend(
backend: str,
dataset: np.ndarray,
queries: np.ndarray,
*,
run_root: Path,
ingest_batch_size: int,
warmup_queries: int,
k: int,
mutation_sizes: Sequence[int],
delete_count: int,
filter_cache_size: int,
auto_filter_native_threshold: int,
auto_path_filter_native_threshold: int,
) -> dict[str, Any]:
collection_name = f"collection_benchmark_{backend}"
project_path = run_root / backend
adapter = make_adapter(
backend,
project_path,
collection_name,
filter_cache_size=filter_cache_size,
auto_filter_native_threshold=auto_filter_native_threshold,
auto_path_filter_native_threshold=auto_path_filter_native_threshold,
)
reopened = None
rss_before = current_rss_bytes()
gpu_before = gpu_memory_used_bytes()
try:
created = adapter.create_collection(
collection_name,
collection_schema(collection_name, dataset.shape[1]),
distance="cosine",
sparse_weight=0.0,
index_name="default",
)
if not created:
raise RuntimeError(f"Collection already exists for backend {backend}")
ingest = ingest_dataset(adapter, dataset, batch_size=ingest_batch_size)
record_count = adapter.count()
if record_count != dataset.shape[0]:
raise RuntimeError(
f"Collection count mismatch for {backend}: {record_count} != {dataset.shape[0]}"
)
rss_after_ingest = current_rss_bytes()
gpu_after_ingest = gpu_memory_used_bytes()
prebuild_selective_query = None
if backend == "auto_cuvs":
scenario = prebuild_selective_scenario(dataset.shape[0], auto_filter_native_threshold)
if scenario is not None:
prebuild_selective_query = run_single_query_scenario(
adapter,
queries[0],
scenario=scenario,
k=k,
)
searches = [
run_search_scenario(
adapter,
queries,
scenario=scenario,
k=k,
warmup_queries=warmup_queries,
)
for scenario in filter_scenarios()
]
rss_after_search = current_rss_bytes()
gpu_after_search = gpu_memory_used_bytes()
lifecycle = run_mutation_lifecycle(
adapter,
dataset,
queries,
mutation_sizes=mutation_sizes,
delete_count=delete_count,
batch_size=ingest_batch_size,
k=k,
)
close_started = time.perf_counter()
adapter.close()
close_seconds = time.perf_counter() - close_started
adapter = None
construct_started = time.perf_counter()
reopened = make_adapter(
backend,
project_path,
collection_name,
filter_cache_size=filter_cache_size,
auto_filter_native_threshold=auto_filter_native_threshold,
auto_path_filter_native_threshold=auto_path_filter_native_threshold,
)
construct_seconds = time.perf_counter() - construct_started
reopen_query_ms, reopen_ids = time_query(reopened, queries[0], k)
reopen_warm_query_ms, _ = time_query(reopened, queries[0], k)
lifecycle["restart"] = {
"close_seconds": close_seconds,
"adapter_construct_seconds": construct_seconds,
"first_query_ms": reopen_query_ms,
"warm_query_ms": reopen_warm_query_ms,
"first_result_count": len(reopen_ids),
}
return {
"backend": backend,
"ingest": ingest,
"record_count": record_count,
"rss_before_bytes": rss_before,
"rss_after_ingest_bytes": rss_after_ingest,
"rss_after_search_bytes": rss_after_search,
"rss_ingest_delta_bytes": (
rss_after_ingest - rss_before
if rss_before is not None and rss_after_ingest is not None
else None
),
"gpu_before_bytes": gpu_before,
"gpu_after_ingest_bytes": gpu_after_ingest,
"gpu_after_search_bytes": gpu_after_search,
"gpu_search_delta_bytes": (
gpu_after_search - gpu_before
if gpu_before is not None and gpu_after_search is not None
else None
),
"prebuild_selective_query": prebuild_selective_query,
"searches": searches,
"lifecycle": lifecycle,
}
finally:
if adapter is not None:
adapter.close()
if reopened is not None:
reopened.close()
gc.collect()
def attach_recall(results: Sequence[dict[str, Any]], k: int) -> None:
by_backend = {result["backend"]: result for result in results}
reference = by_backend.get("cuvs_brute_force")
if reference is None:
return
reference_searches = {item["name"]: item for item in reference["searches"]}
for result in results:
for search in result["searches"]:
expected = reference_searches[search["name"]]["neighbors"]
search["recall_at_k"] = recall_at_k(search["neighbors"], expected, k)
search["ground_truth_backend"] = "cuvs_brute_force"
def strip_neighbors(results: Iterable[dict[str, Any]]) -> None:
for result in results:
for search in result["searches"]:
search.pop("neighbors", None)
def print_summary(results: Sequence[dict[str, Any]]) -> None:
print("\nbackend scenario p50_ms p95_ms qps recall")
print("---------------------- -------------------- ------- ------- -------- -------")
for result in results:
for search in result["searches"]:
latency = search["search"]["latency_ms"]
print(
f"{result['backend']:<22} {search['name']:<20} "
f"{latency['p50']:>7.3f} {latency['p95']:>7.3f} "
f"{search['search']['qps']:>8.1f} "
f"{search.get('recall_at_k', 1.0):>7.4f}"
)
print("\nbackend mutation write_s next_query_ms warm_query_ms")
print("---------------------- --------- ------- ------------- -------------")
for result in results:
for update in result["lifecycle"]["updates"]:
print(
f"{result['backend']:<22} {update['count']:>9} "
f"{update['write_wall_seconds']:>7.3f} "
f"{update['next_query_ms']:>13.3f} {update['warm_query_ms']:>13.3f}"
)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--backends", default="native,cuvs_brute_force")
parser.add_argument("--vector-count", type=int, default=10_000)
parser.add_argument("--dimension", type=int, default=128)
parser.add_argument("--query-count", type=int, default=20)
parser.add_argument("--k", type=int, default=10)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--warmup-queries", type=int, default=3)
parser.add_argument("--ingest-batch-size", type=int, default=256)
parser.add_argument(
"--mutation-sizes",
type=lambda value: parse_positive_int_list(value, "--mutation-sizes"),
default=[1, 100, 1_000, 10_000],
)
parser.add_argument("--delete-count", type=int, default=100)
parser.add_argument("--filter-cache-size", type=int, default=16)
parser.add_argument("--auto-filter-native-threshold", type=int, default=2000)
parser.add_argument("--auto-path-filter-native-threshold", type=int, default=200)
parser.add_argument("--data-root", type=Path, default=Path("/tmp/openviking-cuvs-benchmark"))
parser.add_argument("--output", type=Path)
parser.add_argument("--keep-collections", action="store_true")
return parser
def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> list[str]:
for name in (
"vector_count",
"dimension",
"query_count",
"k",
"warmup_queries",
"ingest_batch_size",
"delete_count",
):
if getattr(args, name) <= 0:
parser.error(f"--{name.replace('_', '-')} must be positive")
if args.k > args.vector_count:
parser.error("--k cannot exceed --vector-count")
if args.filter_cache_size < 0:
parser.error("--filter-cache-size cannot be negative")
if args.auto_filter_native_threshold < 0:
parser.error("--auto-filter-native-threshold cannot be negative")
if args.auto_path_filter_native_threshold < 0:
parser.error("--auto-path-filter-native-threshold cannot be negative")
backends = [item.strip() for item in args.backends.split(",") if item.strip()]
unknown = sorted(set(backends).difference(SUPPORTED_BACKENDS))
if unknown:
parser.error(f"Unsupported backends: {', '.join(unknown)}")
if len(backends) != len(set(backends)):
parser.error("--backends cannot contain duplicates")
return backends
def main() -> None:
parser = build_parser()
args = parser.parse_args()
backends = validate_args(parser, args)
backends.sort(key=SUPPORTED_BACKENDS.index)
files = prepare_dataset(
args.data_root,
vector_count=args.vector_count,
dimension=args.dimension,
query_count=args.query_count,
metric="cosine",
seed=args.seed,
generation_chunk_size=16_384,
force=False,
)
dataset = np.load(files.dataset, mmap_mode="r", allow_pickle=False)
queries = np.load(files.queries, mmap_mode="r", allow_pickle=False)
collection_root = args.data_root / "collection-runs"
collection_root.mkdir(parents=True, exist_ok=True)
run_root = Path(tempfile.mkdtemp(prefix="run-", dir=collection_root))
results: list[dict[str, Any]] = []
try:
for backend in backends:
print(f"Running collection benchmark for {backend} ...", flush=True)
results.append(
run_backend(
backend,
dataset,
queries,
run_root=run_root,
ingest_batch_size=args.ingest_batch_size,
warmup_queries=args.warmup_queries,
k=args.k,
mutation_sizes=args.mutation_sizes,
delete_count=args.delete_count,
filter_cache_size=args.filter_cache_size,
auto_filter_native_threshold=args.auto_filter_native_threshold,
auto_path_filter_native_threshold=(args.auto_path_filter_native_threshold),
)
)
attach_recall(results, args.k)
strip_neighbors(results)
output_document = {
"format_version": 1,
"runtime": runtime_metadata(),
"dataset": {
"kind": "random",
"vector_count": int(dataset.shape[0]),
"dimension": int(dataset.shape[1]),
"query_count": int(queries.shape[0]),
"metric": "cosine",
"seed": args.seed,
"reused": files.reused,
"generated_seconds": files.generated_seconds,
},
"parameters": {
"k": args.k,
"warmup_queries": args.warmup_queries,
"ingest_batch_size": args.ingest_batch_size,
"mutation_sizes": args.mutation_sizes,
"delete_count": args.delete_count,
"filter_cache_size": args.filter_cache_size,
"auto_filter_native_threshold": args.auto_filter_native_threshold,
"auto_path_filter_native_threshold": (args.auto_path_filter_native_threshold),
"filter_scenarios": [scenario.name for scenario in filter_scenarios()],
},
"results": results,
}
output = args.output
if output is None:
result_dir = args.data_root / "results"
result_dir.mkdir(parents=True, exist_ok=True)
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
output = result_dir / (
f"collection-n{dataset.shape[0]}-d{dataset.shape[1]}-"
f"q{queries.shape[0]}-{stamp}.json"
)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(output_document, indent=2, sort_keys=True) + "\n")
print_summary(results)
print(f"\nWrote {output}")
finally:
if not args.keep_collections:
shutil.rmtree(run_root, ignore_errors=True)
if __name__ == "__main__":
main()