Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

22.3 KB
Raw Blame
#!/usr/bin/env python3
"""Benchmark OpenViking's async vector-service facade under concurrency.

This is the pre-embedding service layer: requests go through
VikingVectorIndexBackend and its asyncio.to_thread adapter, but use precomputed
query vectors so embedding and HTTP do not obscure vector-search scheduling.
"""

from __future__ import annotations

import argparse
import asyncio
import gc
import json
import os
import shutil
import statistics
import sys
import tempfile
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Sequence

import numpy as np

BENCHMARK_DIR = Path(__file__).resolve().parent
if str(BENCHMARK_DIR) not in sys.path:
    sys.path.insert(0, str(BENCHMARK_DIR))

from run_collection_benchmark import parse_positive_int_list  # noqa: E402
from run_index_benchmark import percentile, prepare_dataset, runtime_metadata  # noqa: E402

from openviking.server.identity import RequestContext, Role  # noqa: E402
from openviking.storage.viking_vector_index_backend import (  # noqa: E402
    VikingVectorIndexBackend,
)
from openviking_cli.session.user_id import UserIdentifier  # noqa: E402
from openviking_cli.utils.config.vectordb_config import (  # noqa: E402
    CuVSConfig,
    VectorDBBackendConfig,
)

SUPPORTED_BACKENDS = ("native", "cuvs_brute_force", "auto_cuvs_background")
ACCOUNT_ID = "benchmark"


def collection_schema(collection_name: str, dimension: int) -> dict[str, Any]:
    return {
        "CollectionName": collection_name,
        "Fields": [
            {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
            {"FieldName": "vector", "FieldType": "vector", "Dim": dimension},
            {"FieldName": "account_id", "FieldType": "string"},
            {"FieldName": "uniform_bucket", "FieldType": "int64"},
            {"FieldName": "row_number", "FieldType": "int64"},
        ],
        "ScalarIndex": ["account_id", "uniform_bucket", "row_number"],
    }


def make_record(row_index: int, vector: Sequence[float]) -> dict[str, Any]:
    return {
        "id": f"record-{row_index}",
        "vector": list(vector),
        "account_id": ACCOUNT_ID,
        "uniform_bucket": row_index % 1000,
        "row_number": row_index,
    }


def make_config(
    backend: str,
    *,
    project_path: Path,
    collection_name: str,
    dimension: int,
    filter_cache_size: int,
    auto_rebuild_debounce_ms: int,
) -> VectorDBBackendConfig:
    common = {
        "name": collection_name,
        "path": str(project_path),
        "index_name": "default",
        "distance_metric": "cosine",
        "dimension": dimension,
        "sparse_weight": 0.0,
    }
    if backend == "native":
        return VectorDBBackendConfig(backend="local", **common)
    if backend == "cuvs_brute_force":
        return VectorDBBackendConfig(
            backend="cuvs",
            cuvs=CuVSConfig(
                algorithm="brute_force",
                fallback_to_native=False,
                filter_cache_size=filter_cache_size,
            ),
            **common,
        )
    if backend == "auto_cuvs_background":
        return VectorDBBackendConfig(
            backend="local",
            cuvs=CuVSConfig(
                algorithm="brute_force",
                auto_enable=True,
                auto_background_rebuild=True,
                auto_rebuild_debounce_ms=auto_rebuild_debounce_ms,
                filter_cache_size=filter_cache_size,
            ),
            **common,
        )
    raise ValueError(f"Unsupported backend: {backend}")


async def wait_for_background_rebuild(manager: VikingVectorIndexBackend, timeout: float) -> float:
    index = manager._shared_adapter.get_collection().get_index("default")
    wait = getattr(index, "wait_for_background_rebuild", None)
    if wait is None:
        raise RuntimeError("auto background benchmark could not access the local index worker")
    started = time.perf_counter()
    completed = await asyncio.to_thread(wait, timeout)
    elapsed = time.perf_counter() - started
    if not completed:
        raise RuntimeError(f"auto background rebuild did not complete within {timeout}s")
    return elapsed


def concurrent_summary(
    latencies_ms: Sequence[float],
    *,
    wall_seconds: float,
    request_count: int,
    success_count: int,
    error_count: int,
) -> dict[str, Any]:
    return {
        "request_count": request_count,
        "success_count": success_count,
        "error_count": error_count,
        "wall_seconds": wall_seconds,
        "qps": success_count / wall_seconds if wall_seconds else 0.0,
        "latency_ms": {
            "mean": statistics.fmean(latencies_ms) if latencies_ms else 0.0,
            "p50": percentile(latencies_ms, 0.50),
            "p95": percentile(latencies_ms, 0.95),
            "p99": percentile(latencies_ms, 0.99),
            "max": max(latencies_ms, default=0.0),
        },
        "raw_latency_ms": list(latencies_ms),
    }


async def run_request_set(
    manager: Any,
    ctx: RequestContext,
    queries: np.ndarray,
    *,
    concurrency: int,
    request_count: int,
    k: int,
    filter_factory: Callable[[int], dict[str, Any] | None],
) -> dict[str, Any]:
    semaphore = asyncio.Semaphore(concurrency)
    start_event = asyncio.Event()
    query_lists = [row.tolist() for row in queries]

    async def one_request(request_index: int) -> tuple[float, str | None]:
        await start_event.wait()
        async with semaphore:
            started = time.perf_counter()
            try:
                records = await manager.query(
                    query_vector=query_lists[request_index % len(query_lists)],
                    filter=filter_factory(request_index),
                    limit=k,
                    output_fields=["id"],
                    ctx=ctx,
                )
                latency_ms = (time.perf_counter() - started) * 1000.0
                if len(records) != k:
                    return latency_ms, f"expected {k} results, received {len(records)}"
                return latency_ms, None
            except Exception as exc:
                latency_ms = (time.perf_counter() - started) * 1000.0
                return latency_ms, f"{type(exc).__name__}: {exc}"

    tasks = [asyncio.create_task(one_request(index)) for index in range(request_count)]
    await asyncio.sleep(0)
    wall_started = time.perf_counter()
    start_event.set()
    outcomes = await asyncio.gather(*tasks)
    wall_seconds = time.perf_counter() - wall_started
    latencies = [latency for latency, _ in outcomes]
    errors = [error for _, error in outcomes if error is not None]
    result = concurrent_summary(
        latencies,
        wall_seconds=wall_seconds,
        request_count=request_count,
        success_count=request_count - len(errors),
        error_count=len(errors),
    )
    result["errors"] = errors[:10]
    return result


def uniform_filter(selectivity: float) -> dict[str, Any]:
    upper_bound = max(1, int(1000 * selectivity))
    return {"op": "range", "field": "uniform_bucket", "lt": upper_bound}


def unique_filter(request_index: int, vector_count: int, salt: int) -> dict[str, Any]:
    width = max(10, min(100, vector_count))
    span = max(1, vector_count - width + 1)
    start = ((request_index + salt) * 997) % span
    return {
        "op": "range",
        "field": "row_number",
        "gte": start,
        "lt": start + width,
    }


async def ingest_dataset(
    manager: VikingVectorIndexBackend,
    dataset: np.ndarray,
    *,
    batch_size: int,
) -> dict[str, Any]:
    # Benchmark setup uses the shared adapter's batch API. Timed requests below
    # go through the public async service facade.
    adapter = manager._shared_adapter
    started = time.perf_counter()
    for start in range(0, dataset.shape[0], batch_size):
        stop = min(start + batch_size, dataset.shape[0])
        records = [make_record(index, dataset[index].tolist()) for index in range(start, stop)]
        await asyncio.to_thread(adapter.upsert, records)
    total_seconds = time.perf_counter() - started
    return {
        "total_seconds": total_seconds,
        "records_per_second": dataset.shape[0] / total_seconds if total_seconds else 0.0,
    }


async def warm_query(
    manager: VikingVectorIndexBackend,
    ctx: RequestContext,
    query: Sequence[float],
    *,
    filter: dict[str, Any] | None,
    k: int,
) -> float:
    started = time.perf_counter()
    records = await manager.query(
        query_vector=list(query),
        filter=filter,
        limit=k,
        output_fields=["id"],
        ctx=ctx,
    )
    if len(records) != k:
        raise RuntimeError(f"Warm query returned {len(records)} records, expected {k}")
    return (time.perf_counter() - started) * 1000.0


async def run_cached_scenario(
    manager: VikingVectorIndexBackend,
    ctx: RequestContext,
    queries: np.ndarray,
    *,
    name: str,
    filter: dict[str, Any] | None,
    concurrency_levels: Sequence[int],
    request_count: int,
    k: int,
) -> dict[str, Any]:
    initial_filter_ms = await warm_query(manager, ctx, queries[0], filter=filter, k=k)
    results = []
    for concurrency in concurrency_levels:
        await warm_query(manager, ctx, queries[0], filter=filter, k=k)
        result = await run_request_set(
            manager,
            ctx,
            queries,
            concurrency=concurrency,
            request_count=request_count,
            k=k,
            filter_factory=lambda _index, value=filter: value,
        )
        result["concurrency"] = concurrency
        results.append(result)
    return {
        "name": name,
        "filter_mode": "cached",
        "filter": filter,
        "initial_filter_query_ms": initial_filter_ms,
        "results": results,
    }


async def run_unique_filter_scenario(
    manager: VikingVectorIndexBackend,
    ctx: RequestContext,
    queries: np.ndarray,
    *,
    vector_count: int,
    concurrency_levels: Sequence[int],
    request_count: int,
    k: int,
) -> dict[str, Any]:
    results = []
    for matrix_index, concurrency in enumerate(concurrency_levels):
        salt = (matrix_index + 1) * 10_000
        result = await run_request_set(
            manager,
            ctx,
            queries,
            concurrency=concurrency,
            request_count=request_count,
            k=k,
            filter_factory=lambda index, s=salt: unique_filter(index, vector_count, s),
        )
        result["concurrency"] = concurrency
        results.append(result)
    return {
        "name": "unique_filter",
        "filter_mode": "new_per_request",
        "candidate_window": max(10, min(100, vector_count)),
        "results": results,
    }


async def run_post_mutation_scenario(
    manager: VikingVectorIndexBackend,
    ctx: RequestContext,
    dataset: np.ndarray,
    queries: np.ndarray,
    *,
    concurrency_levels: Sequence[int],
    k: int,
    wait_for_background: bool,
    background_timeout: float,
) -> dict[str, Any]:
    results = []
    for matrix_index, concurrency in enumerate(concurrency_levels):
        background_wait_seconds = 0.0
        if wait_for_background:
            background_wait_seconds = await wait_for_background_rebuild(
                manager, background_timeout
            )
        row_index = matrix_index % dataset.shape[0]
        record = make_record(row_index, queries[matrix_index % len(queries)].tolist())
        write_started = time.perf_counter()
        record_id = await manager.upsert(record, ctx=ctx)
        write_ms = (time.perf_counter() - write_started) * 1000.0
        if not record_id:
            raise RuntimeError("Post-mutation upsert failed")
        result = await run_request_set(
            manager,
            ctx,
            queries,
            concurrency=concurrency,
            request_count=concurrency,
            k=k,
            filter_factory=lambda _index: None,
        )
        result["concurrency"] = concurrency
        result["write_ms"] = write_ms
        result["pre_write_background_wait_seconds"] = background_wait_seconds
        results.append(result)
    return {
        "name": "post_mutation_burst",
        "filter_mode": "tenant_filter_after_cache_invalidation",
        "results": results,
    }


async def run_backend(
    backend: str,
    dataset: np.ndarray,
    queries: np.ndarray,
    *,
    run_root: Path,
    ingest_batch_size: int,
    filter_cache_size: int,
    auto_rebuild_debounce_ms: int,
    background_timeout: float,
    concurrency_levels: Sequence[int],
    cached_request_count: int,
    unique_request_count: int,
    k: int,
) -> dict[str, Any]:
    collection_name = f"service_benchmark_{backend}"
    config = make_config(
        backend,
        project_path=run_root / backend,
        collection_name=collection_name,
        dimension=dataset.shape[1],
        filter_cache_size=filter_cache_size,
        auto_rebuild_debounce_ms=auto_rebuild_debounce_ms,
    )
    manager = VikingVectorIndexBackend(config)
    ctx = RequestContext(user=UserIdentifier(ACCOUNT_ID, "user"), role=Role.USER)
    try:
        created = await manager.create_collection(
            collection_name,
            collection_schema(collection_name, dataset.shape[1]),
        )
        if not created:
            raise RuntimeError(f"Collection already exists for backend {backend}")
        ingest = await ingest_dataset(manager, dataset, batch_size=ingest_batch_size)
        record_count = await manager.count(ctx=ctx)
        if record_count != dataset.shape[0]:
            raise RuntimeError(
                f"Collection count mismatch for {backend}: {record_count} != {dataset.shape[0]}"
            )

        initial_background_wait_seconds = 0.0
        background_enabled = backend == "auto_cuvs_background"
        if background_enabled:
            initial_background_wait_seconds = await wait_for_background_rebuild(
                manager, background_timeout
            )

        scenarios = [
            await run_cached_scenario(
                manager,
                ctx,
                queries,
                name="tenant_cached",
                filter=None,
                concurrency_levels=concurrency_levels,
                request_count=cached_request_count,
                k=k,
            ),
            await run_cached_scenario(
                manager,
                ctx,
                queries,
                name="tenant_uniform_10pct_cached",
                filter=uniform_filter(0.10),
                concurrency_levels=concurrency_levels,
                request_count=cached_request_count,
                k=k,
            ),
            await run_unique_filter_scenario(
                manager,
                ctx,
                queries,
                vector_count=dataset.shape[0],
                concurrency_levels=concurrency_levels,
                request_count=unique_request_count,
                k=k,
            ),
            await run_post_mutation_scenario(
                manager,
                ctx,
                dataset,
                queries,
                concurrency_levels=concurrency_levels,
                k=k,
                wait_for_background=background_enabled,
                background_timeout=background_timeout,
            ),
        ]
        return {
            "backend": backend,
            "ingest": ingest,
            "record_count": record_count,
            "initial_background_wait_seconds": initial_background_wait_seconds,
            "scenarios": scenarios,
        }
    finally:
        await manager.close()
        gc.collect()


def print_summary(results: Sequence[dict[str, Any]]) -> None:
    print(
        "\nbackend                 scenario                         conc     p50_ms     p99_ms       qps  errors"
    )
    print(
        "----------------------  -------------------------------  ----  ----------  ----------  --------  ------"
    )
    for backend_result in results:
        for scenario in backend_result["scenarios"]:
            for result in scenario["results"]:
                latency = result["latency_ms"]
                print(
                    f"{backend_result['backend']:<22}  {scenario['name']:<31}  "
                    f"{result['concurrency']:>4}  {latency['p50']:>10.3f}  "
                    f"{latency['p99']:>10.3f}  {result['qps']:>8.1f}  "
                    f"{result['error_count']:>6}"
                )


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--backends", default="native,cuvs_brute_force")
    parser.add_argument("--vector-count", type=int, default=10_000)
    parser.add_argument("--dimension", type=int, default=128)
    parser.add_argument("--query-count", type=int, default=64)
    parser.add_argument("--k", type=int, default=10)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--ingest-batch-size", type=int, default=256)
    parser.add_argument("--filter-cache-size", type=int, default=16)
    parser.add_argument("--auto-rebuild-debounce-ms", type=int, default=500)
    parser.add_argument("--background-timeout", type=float, default=120.0)
    parser.add_argument(
        "--concurrency",
        type=lambda value: parse_positive_int_list(value, "--concurrency"),
        default=[1, 4, 16, 32, 64],
    )
    parser.add_argument("--cached-request-count", type=int, default=200)
    parser.add_argument("--unique-request-count", type=int, default=32)
    parser.add_argument("--data-root", type=Path, default=Path("/tmp/openviking-cuvs-benchmark"))
    parser.add_argument("--output", type=Path)
    parser.add_argument("--keep-collections", action="store_true")
    return parser


def validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> list[str]:
    for name in (
        "vector_count",
        "dimension",
        "query_count",
        "k",
        "ingest_batch_size",
        "cached_request_count",
        "unique_request_count",
    ):
        if getattr(args, name) <= 0:
            parser.error(f"--{name.replace('_', '-')} must be positive")
    if args.k > args.vector_count:
        parser.error("--k cannot exceed --vector-count")
    if args.filter_cache_size < 0:
        parser.error("--filter-cache-size cannot be negative")
    if args.auto_rebuild_debounce_ms < 0:
        parser.error("--auto-rebuild-debounce-ms cannot be negative")
    if args.background_timeout <= 0:
        parser.error("--background-timeout must be positive")
    backends = [item.strip() for item in args.backends.split(",") if item.strip()]
    if not backends:
        parser.error("--backends cannot be empty")
    unknown = sorted(set(backends).difference(SUPPORTED_BACKENDS))
    if unknown:
        parser.error(f"Unsupported backends: {', '.join(unknown)}")
    if len(backends) != len(set(backends)):
        parser.error("--backends cannot contain duplicates")
    return backends


async def async_main(args: argparse.Namespace, backends: Sequence[str]) -> dict[str, Any]:
    files = prepare_dataset(
        args.data_root,
        vector_count=args.vector_count,
        dimension=args.dimension,
        query_count=args.query_count,
        metric="cosine",
        seed=args.seed,
        generation_chunk_size=16_384,
        force=False,
    )
    dataset = np.load(files.dataset, mmap_mode="r", allow_pickle=False)
    queries = np.load(files.queries, mmap_mode="r", allow_pickle=False)
    collection_root = args.data_root / "service-runs"
    collection_root.mkdir(parents=True, exist_ok=True)
    run_root = Path(tempfile.mkdtemp(prefix="run-", dir=collection_root))
    results = []
    try:
        for backend in sorted(backends, key=SUPPORTED_BACKENDS.index):
            print(f"Running service concurrency benchmark for {backend} ...", flush=True)
            results.append(
                await run_backend(
                    backend,
                    dataset,
                    queries,
                    run_root=run_root,
                    ingest_batch_size=args.ingest_batch_size,
                    filter_cache_size=args.filter_cache_size,
                    auto_rebuild_debounce_ms=args.auto_rebuild_debounce_ms,
                    background_timeout=args.background_timeout,
                    concurrency_levels=args.concurrency,
                    cached_request_count=args.cached_request_count,
                    unique_request_count=args.unique_request_count,
                    k=args.k,
                )
            )
        return {
            "format_version": 1,
            "runtime": runtime_metadata(),
            "dataset": {
                "kind": "random",
                "vector_count": int(dataset.shape[0]),
                "dimension": int(dataset.shape[1]),
                "query_count": int(queries.shape[0]),
                "metric": "cosine",
                "seed": args.seed,
                "reused": files.reused,
                "generated_seconds": files.generated_seconds,
            },
            "parameters": {
                "k": args.k,
                "ingest_batch_size": args.ingest_batch_size,
                "filter_cache_size": args.filter_cache_size,
                "auto_rebuild_debounce_ms": args.auto_rebuild_debounce_ms,
                "background_timeout": args.background_timeout,
                "concurrency": args.concurrency,
                "cached_request_count": args.cached_request_count,
                "unique_request_count": args.unique_request_count,
                "asyncio_default_thread_pool_limit": min(32, (os.cpu_count() or 1) + 4),
            },
            "results": results,
        }
    finally:
        if not args.keep_collections:
            shutil.rmtree(run_root, ignore_errors=True)


def main() -> None:
    parser = build_parser()
    args = parser.parse_args()
    backends = validate_args(parser, args)
    output_document = asyncio.run(async_main(args, backends))
    output = args.output
    if output is None:
        result_dir = args.data_root / "results"
        result_dir.mkdir(parents=True, exist_ok=True)
        stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
        dataset = output_document["dataset"]
        output = result_dir / (
            f"service-n{dataset['vector_count']}-d{dataset['dimension']}-"
            f"q{dataset['query_count']}-{stamp}.json"
        )
    output.parent.mkdir(parents=True, exist_ok=True)
    output.write_text(json.dumps(output_document, indent=2, sort_keys=True) + "\n")
    print_summary(output_document["results"])
    print(f"\nWrote {output}")


if __name__ == "__main__":
    main()