Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
#!/usr/bin/env python3
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""OpenViking vector backend performance benchmark.
This benchmark targets OpenViking's VikingVectorIndexBackend boundary. It uses
the real OV context schema, URI scope filters, tenant context, and backend calls
while still accepting precomputed vectors from synthetic or dir-vector data.
"""
from __future__ import annotations
import argparse
import asyncio
import csv
import json
import math
import os
import platform
import random
import shutil
import statistics
import struct
import subprocess
import sys
import time
import uuid
from dataclasses import asdict, dataclass, field, replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable, Iterable, Iterator, Optional, Sequence
from benchmark.vectordb_perf.async_utils import map_bounded_as_completed
from openviking.server.identity import RequestContext, Role
from openviking.storage.collection_schemas import CollectionSchemas
from openviking.storage.expr import PathScope
from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config import OpenVikingConfigSingleton
BENCH_ACCOUNT_ID = "bench_account"
BENCH_USER_ID = "bench_user"
DIR_VECTOR_DATASET_URL = "https://github.com/KurtPatrickHere/dir-vector-dataset"
DEFAULT_DIR_VECTOR_DATASETS = ("wiki", "arxiv")
PROGRESS_INTERVAL_SECONDS = 5.0
DIR_VECTOR_QUERY_SCOPES = ("dataset", "derived_gt_lca_v1")
DERIVED_INDEX_READY_TIMEOUT_SECONDS = 600.0
PROFILE_DEFAULTS: dict[str, dict[str, Any]] = {
"smoke": {
"rows": 128,
"queries": 8,
"dim": 32,
"batch_size": 32,
"concurrency": 1,
"top_k": 10,
"path_depth": 4,
"path_fanout": 4,
"filter_selectivity": 0.25,
},
"standard": {
"rows": 10000,
"queries": 100,
"dim": 1024,
"batch_size": 500,
"concurrency": 4,
"top_k": 20,
"path_depth": 6,
"path_fanout": 8,
"filter_selectivity": 0.05,
},
"stress": {
"rows": 100000,
"queries": 500,
"dim": 1024,
"batch_size": 1000,
"concurrency": 8,
"top_k": 100,
"path_depth": 8,
"path_fanout": 10,
"filter_selectivity": 0.01,
},
}
DIR_VECTOR_FILES = {
"wiki": {
"corpus": "dbpedia_dir_2m_corpus.jsonl",
"corpus_paths": "dbpedia_dir_2m_corpus_paths.json",
"vectors": "dbpedia_dir_2m_corpus_vectors.fvecs",
"queries": "dbpedia_dir_2m_query.jsonl",
"query_vectors": "dbpedia_dir_2m_query_vectors.fvecs",
"ground_truth": "dbpedia_dir_2m_groundtruth.tsv",
},
"arxiv": {
"corpus": "arxiv_corpus_metadata.json",
"vectors": "arxiv_corpus_vectors.fvecs",
"queries": "arxiv_query_constraint.json",
"query_vectors": "arxiv_query_vectors.fvecs",
"ground_truth": "arxiv_ground_truth.txt",
},
"arxiv_category": {
"corpus": "arxiv_corpus_metadata.json",
"vectors": "arxiv_corpus_vectors.fvecs",
"queries": "arxiv_category_query_constraint.json",
"query_vectors": "arxiv_category_query_vectors.fvecs",
"ground_truth": "arxiv_category_ground_truth.txt",
},
}
@dataclass
class BenchOptions:
config: Optional[str]
output_dir: Path
run_id: str
profile: str
mode: str
workload: str
dataset_root: Optional[Path]
dataset: str
full: bool
rows: int
queries: int
dim: int
batch_size: int
concurrency: int
top_k: int
path_depth: int
path_fanout: int
filter_selectivity: float
seed: int
distance: Optional[str]
drop_at_end: bool
dir_vector_query_scope: str
@dataclass
class QueryCase:
query_id: str
vector: list[float]
filter_path: str
ground_truth_ids: list[str] = field(default_factory=list)
@dataclass(frozen=True)
class GroundTruthItem:
corpus_id: str
score: float = 1.0
@dataclass
class Workload:
name: str
dim: int
records: Callable[[], Iterator[dict[str, Any]]]
queries: list[QueryCase]
expected_rows: Optional[int]
directory_queries: Optional[list[QueryCase]] = None
path_diagnostics: dict[str, Any] = field(default_factory=dict)
@dataclass
class Event:
phase: str
operation: str
started_at: str
ended_at: str
latency_ms: float
success: bool
count: int = 0
error: Optional[str] = None
extra: dict[str, Any] = field(default_factory=dict)
@dataclass
class RunResult:
run_id: str
collection_name: str
output_dir: str
events: list[Event]
validation_errors: list[str]
quality: dict[str, Any]
workload: dict[str, Any]
environment: dict[str, Any]
kept_collection: bool
def directory_queries(workload: Workload) -> list[QueryCase]:
return workload.queries if workload.directory_queries is None else workload.directory_queries
def utc_now() -> str:
return datetime.now(UTC).isoformat()
def progress(message: str) -> None:
print(f"[vectordb_perf] {message}", file=sys.stderr, flush=True)
def progress_count(
phase: str,
done: int,
total: Optional[int],
started_at: float,
last_reported_at: float,
*,
force: bool = False,
) -> float:
now = time.perf_counter()
if not force and last_reported_at and now - last_reported_at < PROGRESS_INTERVAL_SECONDS:
return last_reported_at
elapsed = max(now - started_at, 1e-9)
rate = done / elapsed
if total:
progress(f"{phase}: {done}/{total} ({done / total * 100.0:.1f}%), {rate:.1f}/s")
else:
progress(f"{phase}: {done}, {rate:.1f}/s")
return now
def timed_event(
phase: str,
operation: str,
fn: Callable[[], Any],
*,
count: int = 0,
extra: Optional[dict[str, Any]] = None,
) -> tuple[Event, Any]:
started = utc_now()
start = time.perf_counter()
try:
result = fn()
success = True
error = None
except Exception as exc: # benchmark should record the failing operation
result = None
success = False
error = f"{type(exc).__name__}: {exc}"
ended = utc_now()
latency_ms = (time.perf_counter() - start) * 1000.0
return (
Event(
phase=phase,
operation=operation,
started_at=started,
ended_at=ended,
latency_ms=latency_ms,
success=success,
count=count,
error=error,
extra=extra or {},
),
result,
)
async def async_timed_event(
phase: str,
operation: str,
fn: Callable[[], Any],
*,
count: int = 0,
extra: Optional[dict[str, Any]] = None,
) -> tuple[Event, Any]:
started = utc_now()
start = time.perf_counter()
try:
result = await fn()
success = True
error = None
except Exception as exc: # benchmark should record the failing operation
result = None
success = False
error = f"{type(exc).__name__}: {exc}"
ended = utc_now()
latency_ms = (time.perf_counter() - start) * 1000.0
return (
Event(
phase=phase,
operation=operation,
started_at=started,
ended_at=ended,
latency_ms=latency_ms,
success=success,
count=count,
error=error,
extra=extra or {},
),
result,
)
def read_fvecs_dim(path: Path) -> int:
with path.open("rb") as handle:
raw = handle.read(4)
if len(raw) != 4:
raise ValueError(f"empty fvecs file: {path}")
return struct.unpack("<i", raw)[0]
def count_fvecs(path: Path) -> int:
dim = read_fvecs_dim(path)
if dim <= 0:
raise ValueError(f"invalid fvecs dimension {dim} in {path}")
stride = 4 + dim * 4
size = path.stat().st_size
if size % stride != 0:
raise ValueError(f"fvecs file size is not aligned with dim={dim}: {path}")
return size // stride
def iter_fvecs(path: Path, limit: Optional[int] = None) -> Iterator[list[float]]:
with path.open("rb") as handle:
count = 0
while limit is None or count < limit:
raw_dim = handle.read(4)
if not raw_dim:
return
if len(raw_dim) != 4:
raise ValueError(f"truncated fvecs dimension in {path}")
dim = struct.unpack("<i", raw_dim)[0]
if dim <= 0:
raise ValueError(f"invalid fvecs dimension {dim} in {path}")
raw_vec = handle.read(dim * 4)
if len(raw_vec) != dim * 4:
raise ValueError(f"truncated fvecs vector in {path}")
yield list(struct.unpack(f"<{dim}f", raw_vec))
count += 1
def iter_json_array_records(
path: Path,
limit: Optional[int] = None,
*,
chunk_size: int = 1024 * 1024,
) -> Iterator[dict[str, Any]]:
"""Stream a top-level JSON array without materializing the whole file."""
if chunk_size <= 0:
raise ValueError("chunk_size must be positive")
if limit is not None and limit <= 0:
return
decoder = json.JSONDecoder()
with path.open("r", encoding="utf-8") as handle:
buffer = ""
position = 0
eof = False
started = False
expect_value = True
after_comma = False
yielded = 0
def refill() -> None:
nonlocal buffer, position, eof
if position:
buffer = buffer[position:]
position = 0
chunk = handle.read(chunk_size)
if chunk:
buffer += chunk
else:
eof = True
while True:
if position >= len(buffer) and not eof:
refill()
while position < len(buffer) and buffer[position].isspace():
position += 1
if position >= len(buffer):
if eof:
raise ValueError(f"truncated JSON array in {path.name}")
refill()
continue
if not started:
if buffer[position] != "[":
raise ValueError(f"expected a top-level JSON array in {path.name}")
position += 1
started = True
continue
if not expect_value:
if buffer[position] == ",":
position += 1
expect_value = True
after_comma = True
continue
if buffer[position] == "]":
position += 1
while True:
while position < len(buffer) and buffer[position].isspace():
position += 1
if position < len(buffer):
raise ValueError(f"unexpected content after JSON array in {path.name}")
if eof:
return
refill()
raise ValueError(f"expected ',' or ']' in {path.name}")
if buffer[position] == "]":
if after_comma:
raise ValueError(f"trailing comma in JSON array in {path.name}")
position += 1
while True:
while position < len(buffer) and buffer[position].isspace():
position += 1
if position < len(buffer):
raise ValueError(f"unexpected content after JSON array in {path.name}")
if eof:
return
refill()
try:
item, end = decoder.raw_decode(buffer, position)
except json.JSONDecodeError as exc:
if eof:
raise ValueError(f"invalid JSON array in {path.name}: {exc}") from exc
refill()
continue
position = end
expect_value = False
after_comma = False
if not isinstance(item, dict):
raise ValueError(f"expected JSON objects in {path.name}")
yield item
yielded += 1
if limit is not None and yielded >= limit:
return
def iter_json_records(path: Path, limit: Optional[int] = None) -> Iterator[dict[str, Any]]:
if path.suffix == ".jsonl":
with path.open("r", encoding="utf-8") as handle:
for index, line in enumerate(handle):
if limit is not None and index >= limit:
return
stripped = line.strip()
if stripped:
item = json.loads(stripped)
if isinstance(item, dict):
yield item
return
with path.open("r", encoding="utf-8") as handle:
first = ""
while not first:
char = handle.read(1)
if not char:
raise ValueError(f"empty JSON file: {path}")
if not char.isspace():
first = char
if first == "[":
yield from iter_json_array_records(path, limit)
return
data = json.loads(path.read_text(encoding="utf-8"))
if isinstance(data, dict):
iterable: Iterable[Any] = data.values()
elif isinstance(data, list):
iterable = data
else:
raise ValueError(f"unsupported JSON corpus shape in {path}")
for index, item in enumerate(iterable):
if limit is not None and index >= limit:
return
if isinstance(item, dict):
yield item
def pick_value(record: dict[str, Any], candidates: Sequence[str]) -> Any:
for key in candidates:
value = record.get(key)
if value not in (None, ""):
return value
return None
def normalize_path(path: Any) -> str:
if isinstance(path, list):
if not path:
return "/"
return ";".join(normalize_path(item) for item in path if item not in (None, ""))
if not isinstance(path, str):
return "/"
stripped = path.strip()
if not stripped:
return "/"
if stripped.startswith("viking://"):
stripped = "/" + stripped[len("viking://") :].strip("/")
elif not stripped.startswith("/"):
stripped = "/" + stripped
return stripped.rstrip("/") if len(stripped) > 1 else stripped
GENERIC_PATH_FIELDS = (
"dir_path",
"dir_paths",
"path",
"paths",
"directory",
"directories",
"category_path",
"category_paths",
"date_path",
)
def extract_declared_path(
record: dict[str, Any], fields: Sequence[str] = GENERIC_PATH_FIELDS
) -> Optional[str]:
"""Return only explicitly declared path metadata, never arbitrary prose."""
value = pick_value(record, fields)
if value is not None:
return normalize_path(value)
for container_name in ("constraints", "constraint", "metadata"):
nested = record.get(container_name)
if isinstance(nested, dict):
value = pick_value(nested, fields)
if value is not None:
return normalize_path(value)
elif container_name == "constraint" and isinstance(nested, (str, list)):
return normalize_path(nested)
return None
def extract_path(record: dict[str, Any]) -> str:
return extract_declared_path(record) or "/"
def extract_legacy_path(record: dict[str, Any]) -> str:
"""Keep the pre-existing best-effort behavior for non-Wiki datasets."""
declared = extract_declared_path(record)
if declared is not None:
return declared
for value in record.values():
if isinstance(value, str) and "/" in value:
return normalize_path(value)
if isinstance(value, list):
path_items = [item for item in value if isinstance(item, str) and "/" in item]
if path_items:
return normalize_path(path_items)
return "/"
def record_id(record: dict[str, Any], fallback: int) -> str:
value = pick_value(
record,
(
"id",
"_id",
"entity_id",
"paper_id",
"doc_id",
"item_id",
"source_id",
"qid",
"query_id",
),
)
return str(value) if value is not None else str(fallback)
def path_prefix(path: str, segments: int) -> str:
parts = [part for part in path.strip("/").split("/") if part]
if not parts:
return "/"
selected = parts[: max(1, min(segments, len(parts)))]
return "/" + "/".join(selected)
def path_matches(path_value: Any, prefix: str) -> bool:
paths = str(path_value or "").split(";")
normalized_prefix = normalize_path(prefix)
for item in paths:
path = normalize_path(item)
if path == normalized_prefix or path.startswith(normalized_prefix.rstrip("/") + "/"):
return True
return False
def path_parts(path: str) -> tuple[str, ...]:
return tuple(part for part in normalize_path(path).strip("/").split("/") if part)
def common_path_prefix(paths: Sequence[str]) -> str:
if not paths:
return "/"
prefix = list(path_parts(paths[0]))
for path in paths[1:]:
parts = path_parts(path)
index = 0
while index < min(len(prefix), len(parts)) and prefix[index] == parts[index]:
index += 1
del prefix[index:]
if not prefix:
return "/"
return "/" + "/".join(prefix) if prefix else "/"
def iter_wiki_corpus_paths(path: Path, limit: Optional[int] = None) -> Iterator[tuple[str, str]]:
for index, item in enumerate(iter_json_array_records(path, limit)):
item_id = record_id(item, index)
if item_id != str(index):
raise ValueError(
f"Wiki corpus path mapping is not row-aligned at row {index}: id={item_id}"
)
raw_path = item.get("path")
normalized = normalize_path(raw_path)
if normalized == "/":
raise ValueError(f"Wiki corpus path mapping has an empty root path at row {index}")
yield item_id, normalized
def collect_wiki_corpus_paths(path: Path, corpus_ids: set[str]) -> dict[str, str]:
if not corpus_ids:
return {}
found: dict[str, str] = {}
for item_id, item_path in iter_wiki_corpus_paths(path):
if item_id in corpus_ids:
found[item_id] = item_path
if len(found) == len(corpus_ids):
break
missing = corpus_ids - found.keys()
if missing:
preview = ", ".join(sorted(missing)[:5])
raise ValueError(
f"Wiki corpus path mapping is missing {len(missing)} ground-truth ids: {preview}"
)
return found
def derived_gt_lca_scope(items: Sequence[GroundTruthItem], corpus_paths: dict[str, str]) -> str:
if not items:
return "/"
highest_score = max(item.score for item in items)
relevant_paths = [corpus_paths[item.corpus_id] for item in items if item.score == highest_score]
return common_path_prefix(relevant_paths)
def synthetic_vector(rng: random.Random, dim: int) -> list[float]:
return [rng.uniform(-1.0, 1.0) for _ in range(dim)]
def synthetic_path(index: int, *, depth: int, fanout: int) -> str:
fanout = max(1, fanout)
parts = []
value = index
for level in range(max(1, depth)):
parts.append(f"d{level}_{value % fanout}")
value //= fanout
return "/" + "/".join(parts)
def safe_uri_segment(value: Any) -> str:
text = str(value).strip().replace("/", "_")
return "".join(ch if ch.isalnum() or ch in ("_", "-", ".") else "_" for ch in text) or "item"
def resource_uri(dataset: str, path: str, item_id: Optional[Any] = None) -> str:
normalized = normalize_path(str(path or "/").split(";")[0])
suffix = normalized.strip("/")
base = f"viking://resources/bench/{safe_uri_segment(dataset)}"
if suffix:
base = f"{base}/{suffix}"
if item_id is not None:
base = f"{base}/{safe_uri_segment(item_id)}"
return base
def deterministic_time(index: int) -> str:
return f"2026-01-01T00:{(index // 60) % 60:02d}:{index % 60:02d}Z"
def ov_context_record(
*,
record_id: str,
vector: list[float],
uri: str,
index: int,
category: str = "",
content: str = "",
) -> dict[str, Any]:
timestamp = deterministic_time(index)
label = category or "resource"
return {
"id": record_id,
"uri": uri,
"type": "file",
"context_type": "resource",
"vector": vector,
"created_at": timestamp,
"updated_at": timestamp,
"active_count": 0,
"level": 2,
"name": safe_uri_segment(record_id),
"description": label,
"tags": label,
"search_tags": [label],
"abstract": f"source_id:{record_id} category:{label}",
"content": content or f"benchmark content for {record_id} in {label}",
"account_id": BENCH_ACCOUNT_ID,
"owner_user_id": BENCH_USER_ID,
}
def filter_segments(*, selectivity: float, fanout: int, depth: int) -> int:
selectivity = min(1.0, max(1e-9, selectivity))
fanout = max(2, fanout)
segments = math.ceil(math.log(selectivity) / math.log(1.0 / fanout))
return max(1, min(depth, segments))
def build_synthetic_workload(options: BenchOptions) -> Workload:
rows = max(1, options.rows)
dim = max(4, options.dim)
segment_count = filter_segments(
selectivity=options.filter_selectivity,
fanout=options.path_fanout,
depth=options.path_depth,
)
def records() -> Iterator[dict[str, Any]]:
rng = random.Random(options.seed)
for index in range(rows):
record_path = synthetic_path(
index, depth=options.path_depth, fanout=options.path_fanout
)
rid = f"syn-{index}"
yield ov_context_record(
record_id=rid,
vector=synthetic_vector(rng, dim),
uri=resource_uri("synthetic", record_path, rid),
index=index,
category=f"cat_{index % 16}",
)
query_rng = random.Random(options.seed)
vectors = [synthetic_vector(query_rng, dim) for _ in range(rows)]
query_count = max(1, min(options.queries, rows))
cases: list[QueryCase] = []
for qindex in range(query_count):
record_index = (qindex * max(1, rows // query_count)) % rows
full_path = synthetic_path(
record_index, depth=options.path_depth, fanout=options.path_fanout
)
cases.append(
QueryCase(
query_id=f"syn-q{qindex}",
vector=vectors[record_index],
filter_path=resource_uri("synthetic", path_prefix(full_path, segment_count)),
ground_truth_ids=[f"syn-{record_index}"],
)
)
return Workload(
name="synthetic",
dim=dim,
records=records,
queries=cases,
expected_rows=rows,
)
def load_scored_ground_truth(path: Path) -> dict[str, list[GroundTruthItem]]:
result: dict[str, list[GroundTruthItem]] = {}
with path.open("r", encoding="utf-8") as handle:
for index, line in enumerate(handle):
stripped = line.strip()
if not stripped:
continue
if "\t" in stripped:
parts = stripped.split("\t")
if parts[0].lower() in {"query-id", "query_id"}:
continue
if len(parts) >= 3 and is_number(parts[2]):
result.setdefault(str(parts[0]), []).append(
GroundTruthItem(str(parts[1]), float(parts[2]))
)
continue
if len(parts) >= 2:
result[str(parts[0])] = [GroundTruthItem(item) for item in parts[1:] if item]
continue
else:
parts = stripped.replace(",", " ").split()
if len(parts) < 2:
continue
# Plain-text ground-truth files omit query ids; line number is the query id.
result[str(index)] = [GroundTruthItem(item) for item in parts if item]
return result
def load_ground_truth(path: Path) -> dict[str, list[str]]:
return {
query_id: [item.corpus_id for item in items]
for query_id, items in load_scored_ground_truth(path).items()
}
def is_number(value: str) -> bool:
try:
float(value)
return True
except ValueError:
return False
def load_query_records(path: Path, limit: Optional[int]) -> list[dict[str, Any]]:
if path.suffix == ".jsonl":
return list(iter_json_records(path, limit))
data = json.loads(path.read_text(encoding="utf-8"))
if isinstance(data, list):
return [item for item in data[:limit] if isinstance(item, dict)] if limit else data
if isinstance(data, dict):
items = []
for key, value in data.items():
if limit is not None and len(items) >= limit:
break
if isinstance(value, dict):
row = dict(value)
else:
row = {"constraint": value}
row.setdefault("query_id", key)
items.append(row)
return items
raise ValueError(f"unsupported query JSON shape in {path}")
def build_dir_vector_workload(options: BenchOptions) -> Workload:
paths = dir_vector_paths(options.dataset_root, options.dataset)
dim = read_fvecs_dim(paths["vectors"])
total_rows = count_fvecs(paths["vectors"])
total_queries = count_fvecs(paths["query_vectors"])
row_limit = total_rows if options.full else min(options.rows, total_rows)
query_limit = total_queries if options.full else min(options.queries, total_queries)
sample_metadata = None if options.full else list(iter_json_records(paths["corpus"], row_limit))
sampled_ids = (
None
if sample_metadata is None
else {record_id(meta, index) for index, meta in enumerate(sample_metadata)}
)
query_records = load_query_records(paths["queries"], query_limit if options.full else None)
if options.full and len(query_records) != total_queries:
raise ValueError(
f"query metadata/vector count mismatch: {len(query_records)} vs {total_queries}"
)
query_vectors = list(iter_fvecs(paths["query_vectors"], len(query_records)))
if len(query_vectors) != len(query_records):
raise ValueError(
f"query metadata/vector count mismatch: {len(query_records)} vs {len(query_vectors)}"
)
scored_ground_truth = load_scored_ground_truth(paths["ground_truth"])
effective_query_scope = (
options.dir_vector_query_scope if options.dataset == "wiki" else "dataset"
)
query_ground_truth = [
scored_ground_truth.get(record_id(query, index))
or scored_ground_truth.get(str(index))
or []
for index, query in enumerate(query_records)
]
missing_wiki_scopes = [
record_id(query, index)
for index, query in enumerate(query_records)
if extract_declared_path(query) is None
]
if (
options.dataset == "wiki"
and effective_query_scope == "dataset"
and options.mode != "write-only"
and missing_wiki_scopes
):
raise ValueError(
f"wiki query metadata has no directory constraint for "
f"{len(missing_wiki_scopes)} queries; refusing a root-only Directory benchmark. "
"Use a corrected dataset, or explicitly opt into the diagnostic "
"--dir-vector-query-scope derived_gt_lca_v1 mode for Wiki."
)
wiki_gt_paths: dict[str, str] = {}
if options.dataset == "wiki" and options.mode != "write-only":
ground_truth_ids = {item.corpus_id for items in query_ground_truth for item in items}
wiki_gt_paths = collect_wiki_corpus_paths(paths["corpus_paths"], ground_truth_ids)
def records() -> Iterator[dict[str, Any]]:
vector_iter = iter_fvecs(paths["vectors"], row_limit)
metadata_iter = (
iter(sample_metadata)
if sample_metadata is not None
else iter_json_records(paths["corpus"], row_limit)
)
if options.dataset == "wiki":
mapping_iter = iter_wiki_corpus_paths(
paths["corpus_paths"], None if options.full else row_limit
)
combined: Iterable[tuple[dict[str, Any], list[float], tuple[str, str]]] = zip(
metadata_iter, vector_iter, mapping_iter, strict=True
)
else:
combined = (
(
meta,
vector,
(
record_id(meta, index),
extract_legacy_path(meta),
),
)
for index, (meta, vector) in enumerate(zip(metadata_iter, vector_iter, strict=True))
)
for index, (meta, vector, (mapped_id, mapped_path)) in enumerate(combined):
rid = record_id(meta, index)
if options.dataset == "wiki" and (rid != mapped_id or rid != str(index)):
raise ValueError(
"Wiki corpus metadata/path/vector rows are not aligned at "
f"row {index}: metadata_id={rid}, path_id={mapped_id}"
)
if options.dataset == "wiki" and mapped_path == "/":
raise ValueError(f"dir-vector corpus row {index} has no declared directory path")
category = str(pick_value(meta, ("category", "primary_category", "type")) or "")
title = str(pick_value(meta, ("title", "name", "label")) or rid)
yield ov_context_record(
record_id=rid,
vector=vector,
uri=resource_uri(options.dataset, mapped_path, rid),
index=index,
category=category,
content=title,
)
base_covered: list[QueryCase] = []
base_fallback: list[QueryCase] = []
directory_covered: list[QueryCase] = []
directory_fallback: list[QueryCase] = []
missing_dataset_scopes: list[str] = []
root_derived_scopes = 0
for index, (query_meta, vector) in enumerate(zip(query_records, query_vectors, strict=True)):
qid = record_id(query_meta, index)
scored_gt = query_ground_truth[index]
full_gt = [item.corpus_id for item in scored_gt]
gt = full_gt
if sampled_ids is not None:
gt = [item for item in gt if item in sampled_ids]
declared_scope = (
extract_declared_path(query_meta)
if options.dataset == "wiki"
else extract_legacy_path(query_meta)
)
directory_scope = declared_scope or "/"
if effective_query_scope == "derived_gt_lca_v1" and options.mode != "write-only":
directory_scope = derived_gt_lca_scope(scored_gt, wiki_gt_paths)
elif options.dataset == "wiki" and directory_scope == "/":
missing_dataset_scopes.append(qid)
base_case = QueryCase(
query_id=qid,
vector=vector,
filter_path=resource_uri(options.dataset, directory_scope),
ground_truth_ids=gt,
)
if gt or sampled_ids is None:
base_covered.append(base_case)
else:
base_fallback.append(base_case)
if options.dataset == "wiki" and directory_scope == "/":
if effective_query_scope == "derived_gt_lca_v1":
root_derived_scopes += 1
continue
directory_gt = (
[item for item in gt if path_matches(wiki_gt_paths[item], directory_scope)]
if options.dataset == "wiki"
else gt
)
directory_case = QueryCase(
query_id=qid,
vector=vector,
filter_path=resource_uri(options.dataset, directory_scope),
ground_truth_ids=directory_gt,
)
if directory_gt or sampled_ids is None:
directory_covered.append(directory_case)
else:
directory_fallback.append(directory_case)
def select_cases(covered: list[QueryCase], fallback: list[QueryCase]) -> list[QueryCase]:
selected = covered[:query_limit]
if len(selected) < query_limit:
selected.extend(fallback[: query_limit - len(selected)])
return selected
cases = select_cases(base_covered, base_fallback)
directory_cases = select_cases(directory_covered, directory_fallback)
if not cases:
raise ValueError("dir-vector query set is empty")
if options.mode != "write-only" and missing_dataset_scopes:
raise ValueError(
f"{options.dataset} query metadata has no directory constraint for "
f"{len(missing_dataset_scopes)} queries; refusing a root-only Directory benchmark. "
"Use a corrected dataset, or explicitly opt into the diagnostic "
"--dir-vector-query-scope derived_gt_lca_v1 mode for Wiki."
)
if options.mode != "write-only" and not directory_cases:
raise ValueError("dir-vector Directory query set is empty after path validation")
scope_depths = [len(path_parts(case.filter_path)) - 3 for case in directory_cases]
scope_depths = [max(0, depth) for depth in scope_depths]
path_diagnostics = {
"corpus_path_source": paths.get("corpus_paths", paths["corpus"]).name,
"query_scope_source": effective_query_scope,
"base_query_count": len(cases),
"directory_query_count": len(directory_cases),
"unique_directory_scopes": len({case.filter_path for case in directory_cases}),
"root_derived_scopes_omitted": root_derived_scopes,
"directory_scope_depth_min": min(scope_depths) if scope_depths else None,
"directory_scope_depth_median": statistics.median(scope_depths) if scope_depths else None,
"directory_scope_depth_max": max(scope_depths) if scope_depths else None,
}
return Workload(
name=f"dir-vector:{options.dataset}",
dim=dim,
records=records,
queries=cases,
expected_rows=row_limit,
directory_queries=directory_cases,
path_diagnostics=path_diagnostics,
)
def dir_vector_paths(dataset_root: Optional[Path], dataset: str) -> dict[str, Path]:
if dataset_root is None:
raise ValueError("--dataset-root is required for dir-vector workload")
dataset_files = DIR_VECTOR_FILES[dataset]
expected = ", ".join(dataset_files.values())
if not dataset_root.exists():
raise FileNotFoundError(
f"--dataset-root does not exist: {dataset_root}\n"
f"Download dir-vector data first: {DIR_VECTOR_DATASET_URL}\n"
f"Expected --dataset {dataset} files in that directory: {expected}"
)
paths = {name: dataset_root / filename for name, filename in dataset_files.items()}
missing = [str(path) for path in paths.values() if not path.exists()]
if missing:
raise FileNotFoundError(
"dir-vector dataset files missing:\n"
+ "\n".join(f"- {path}" for path in missing)
+ f"\nDownload dir-vector data first: {DIR_VECTOR_DATASET_URL}"
+ f"\nExpected --dataset {dataset} files: {expected}"
)
return paths
def build_schema(collection_name: str, dim: int) -> dict[str, Any]:
return CollectionSchemas.context_collection(collection_name, dim)
def load_backend_config(options: BenchOptions, collection_name: Optional[str] = None):
OpenVikingConfigSingleton.reset_instance()
config = OpenVikingConfigSingleton.initialize(config_path=options.config)
vectordb = config.storage.vectordb.model_copy(deep=True)
if collection_name is not None:
vectordb.name = collection_name
return vectordb
def make_collection_name(config_name: str, run_id: str) -> str:
base = "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in (config_name or "context"))
if not base or not base[0].isalpha():
base = f"bench_{base}"
suffix = "".join(ch if ch.isalnum() else "_" for ch in run_id)
return f"{base}_bench_{suffix}"[:120]
def batched(iterable: Iterable[dict[str, Any]], size: int) -> Iterator[list[dict[str, Any]]]:
batch: list[dict[str, Any]] = []
for item in iterable:
batch.append(item)
if len(batch) >= size:
yield batch
batch = []
if batch:
yield batch
def sample_record_ids(workload: Workload, limit: int) -> list[str]:
ids: list[str] = []
for record in workload.records():
value = record.get("id")
if value is not None:
ids.append(str(value))
if len(ids) >= limit:
break
return ids
async def run_search_phase(
*,
backend: VikingVectorIndexBackend,
ctx: RequestContext,
phase: str,
queries: list[QueryCase],
top_k: int,
concurrency: int,
filtered: bool,
) -> tuple[list[Event], list[str], dict[str, Any]]:
validation_errors: list[str] = []
quality_hits = 0
quality_total = 0
async def one_query(query: QueryCase) -> tuple[Event, list[dict[str, Any]]]:
target_directories = [query.filter_path] if filtered else None
return await async_timed_event(
phase,
"targeted_search_in_tenant" if filtered else "search_in_tenant",
lambda: backend.search_in_tenant(
ctx=ctx,
query_vector=query.vector,
context_type="resource",
target_directories=target_directories,
level=[2],
limit=top_k,
),
extra={
"query_id": query.query_id,
"filter_path": query.filter_path if filtered else "",
},
)
events: list[Event] = []
results: list[tuple[QueryCase, list[dict[str, Any]]]] = []
workers = max(1, concurrency)
progress(f"{phase}: start queries={len(queries)} concurrency={workers} top_k={top_k}")
started_at = time.perf_counter()
last_reported_at = 0.0
done = 0
if workers == 1:
for query in queries:
event, rows = await one_query(query)
events.append(event)
results.append((query, rows or []))
done += 1
last_reported_at = progress_count(
phase,
done,
len(queries),
started_at,
last_reported_at,
force=done == len(queries),
)
else:
async def tagged_query(query: QueryCase):
event, rows = await one_query(query)
return query, event, rows
async for query, event, rows in map_bounded_as_completed(queries, tagged_query, workers):
events.append(event)
results.append((query, rows or []))
done += 1
last_reported_at = progress_count(
phase,
done,
len(queries),
started_at,
last_reported_at,
force=done == len(queries),
)
for query, rows in results:
if not rows:
validation_errors.append(f"{phase}: query {query.query_id} returned no results")
continue
if filtered:
leaked = [
row.get("id")
for row in rows
if not path_matches(row.get("uri", ""), query.filter_path)
]
if leaked:
validation_errors.append(
f"{phase}: query {query.query_id} returned rows outside {query.filter_path}: {leaked[:5]}"
)
if query.ground_truth_ids:
quality_total += 1
if result_hits_ground_truth(rows, query.ground_truth_ids):
quality_hits += 1
quality = {
"queries_with_ground_truth": quality_total,
"hit_rate_at_k": (quality_hits / quality_total) if quality_total else None,
"hits": quality_hits,
}
return events, validation_errors, quality
async def wait_for_auto_derived_index(
backend: VikingVectorIndexBackend,
*,
timeout: float = DERIVED_INDEX_READY_TIMEOUT_SECONDS,
) -> bool:
"""Wait for Auto's background GPU snapshot before timed search phases."""
def wait() -> bool:
adapter = backend._shared_adapter
collection = adapter.get_collection()
index = collection.get_index(adapter.index_name)
if index is None:
raise RuntimeError(f"benchmark index not found: {adapter.index_name}")
wait_for_rebuild = getattr(index, "wait_for_background_rebuild", None)
if not callable(wait_for_rebuild):
raise RuntimeError("Auto backend does not expose background rebuild readiness")
return bool(wait_for_rebuild(timeout=timeout))
ready = await asyncio.to_thread(wait)
if not ready:
raise TimeoutError(f"Auto derived index was not ready within {timeout:.0f}s")
return True
def result_hits_ground_truth(rows: list[dict[str, Any]], ground_truth_ids: list[str]) -> bool:
returned_ids = {str(row.get("id")) for row in rows if row.get("id") is not None}
return any(str(item) in returned_ids for item in ground_truth_ids)
def benchmark_context() -> RequestContext:
return RequestContext(user=UserIdentifier(BENCH_ACCOUNT_ID, BENCH_USER_ID), role=Role.USER)
async def upsert_records(
backend: VikingVectorIndexBackend, ctx: RequestContext, records: list[dict[str, Any]]
) -> list[str]:
ids = await backend.upsert_many(records, ctx=ctx)
if len(ids) != len(records):
raise RuntimeError(f"bulk upsert returned {len(ids)} ids for {len(records)} records")
return ids
def run_benchmark(options: BenchOptions) -> RunResult:
return asyncio.run(run_benchmark_async(options))
def selected_dir_vector_datasets(dataset: str) -> tuple[str, ...]:
return DEFAULT_DIR_VECTOR_DATASETS if dataset == "all" else (dataset,)
def run_benchmark_suite(options: BenchOptions) -> list[tuple[BenchOptions, RunResult]]:
if options.workload != "dir-vector" or options.dataset != "all":
return [(options, run_benchmark(options))]
datasets = selected_dir_vector_datasets(options.dataset)
for dataset in datasets:
dir_vector_paths(options.dataset_root, dataset)
runs = []
for dataset in datasets:
child = replace(
options,
dataset=dataset,
run_id=f"{options.run_id}_{dataset}",
output_dir=options.output_dir / dataset,
)
runs.append((child, run_benchmark(child)))
return runs
async def run_benchmark_async(options: BenchOptions) -> RunResult:
workload = (
build_synthetic_workload(options)
if options.workload == "synthetic"
else build_dir_vector_workload(options)
)
output_dir = options.output_dir
output_dir.mkdir(parents=True, exist_ok=True)
# Load once to discover the configured base name, then reload with the run-specific name.
base_config = load_backend_config(options)
collection_name = make_collection_name(base_config.name or "context", options.run_id)
vectordb_config = load_backend_config(options, collection_name)
if options.distance:
vectordb_config.distance_metric = options.distance
vectordb_config.dimension = workload.dim
backend = VikingVectorIndexBackend(config=vectordb_config)
ctx = benchmark_context()
progress(
"start "
f"run_id={options.run_id} workload={workload.name} mode={options.mode} "
f"records={workload.expected_rows} base_queries={len(workload.queries)} "
f"directory_queries={len(directory_queries(workload))} "
f"dim={workload.dim}"
)
progress(f"collection={collection_name}")
events: list[Event] = []
validation_errors: list[str] = []
quality: dict[str, Any] = {}
kept_collection = True
inserted = 0
fetch_ids: list[str] = []
if options.mode != "read-only":
progress("setup: create collection")
event, created = await async_timed_event(
"setup",
"create_collection",
lambda: backend.create_collection(
collection_name,
build_schema(collection_name, workload.dim),
),
)
events.append(event)
if not event.success:
validation_errors.append(event.error or "create_collection failed")
await close_backend(backend)
return finalize_result(
options, workload, collection_name, events, validation_errors, quality, True
)
if created is False:
validation_errors.append(f"collection already exists: {collection_name}")
await close_backend(backend)
return finalize_result(
options, workload, collection_name, events, validation_errors, quality, True
)
progress("setup: collection ready")
batch_size = max(1, options.batch_size)
progress(f"ingest: start batch_size={batch_size}")
ingest_started_at = time.perf_counter()
last_ingest_reported_at = 0.0
async with backend.bulk_ingest(ctx=ctx):
for batch in batched(workload.records(), batch_size):
if len(fetch_ids) < 20:
fetch_ids.extend(str(row["id"]) for row in batch[: 20 - len(fetch_ids)])
event, ids = await async_timed_event(
"ingest",
"upsert_rows_batch",
lambda batch=batch: upsert_records(backend, ctx, batch),
count=len(batch),
)
events.append(event)
if not event.success:
validation_errors.append(event.error or "upsert_rows_batch failed")
break
inserted += len(ids or [])
last_ingest_reported_at = progress_count(
"ingest",
inserted,
workload.expected_rows,
ingest_started_at,
last_ingest_reported_at,
force=inserted == workload.expected_rows,
)
if options.mode != "read-only" and inserted == 0:
validation_errors.append("no records inserted")
await close_backend(backend)
return finalize_result(
options,
workload,
collection_name,
events,
validation_errors,
quality,
kept_collection=True,
)
if options.mode == "write-only":
if options.drop_at_end:
progress("cleanup: drop collection")
event, dropped = await async_timed_event(
"cleanup", "drop_collection", lambda: backend.drop_collection()
)
events.append(event)
kept_collection = not bool(dropped)
if not event.success:
validation_errors.append(event.error or "drop_collection failed")
await close_backend(backend)
return finalize_result(
options,
workload,
collection_name,
events,
validation_errors,
quality,
kept_collection=kept_collection,
)
cuvs_config = getattr(vectordb_config, "cuvs", None)
auto_background_enabled = bool(
cuvs_config is not None
and getattr(cuvs_config, "auto_enable", False)
and getattr(cuvs_config, "auto_background_rebuild", False)
)
if auto_background_enabled:
progress("prepare: wait for Auto derived index")
event, _ = await async_timed_event(
"prepare",
"wait_for_auto_derived_index",
lambda: wait_for_auto_derived_index(backend),
)
events.append(event)
if not event.success:
validation_errors.append(event.error or "Auto derived index readiness failed")
await close_backend(backend)
return finalize_result(
options,
workload,
collection_name,
events,
validation_errors,
quality,
kept_collection=True,
)
if options.mode == "read-only":
fetch_ids = sample_record_ids(workload, 20)
progress("validate: count/get")
event, count_result = await async_timed_event(
"validate", "count_all", lambda: backend.count(ctx=ctx)
)
events.append(event)
if event.success and workload.expected_rows is not None and int(count_result or 0) < inserted:
validation_errors.append(f"count_all={count_result} smaller than inserted={inserted}")
elif not event.success:
validation_errors.append(event.error or "count_all failed")
event, fetched = await async_timed_event(
"validate", "fetch_by_ids", lambda: backend.get(fetch_ids, ctx=ctx), count=len(fetch_ids)
)
events.append(event)
if not event.success:
validation_errors.append(event.error or "fetch_by_ids failed")
elif len(fetched or []) != len(fetch_ids):
validation_errors.append(
f"fetch_by_ids returned {len(fetched or [])}, expected {len(fetch_ids)}"
)
selected_queries = workload.queries
selected_directory_queries = directory_queries(workload)
search_events, errors, vector_quality = await run_search_phase(
backend=backend,
ctx=ctx,
phase="vector_search",
queries=selected_queries,
top_k=options.top_k,
concurrency=options.concurrency,
filtered=False,
)
events.extend(search_events)
validation_errors.extend(errors)
quality["vector_search"] = vector_quality
filtered_events, errors, filtered_quality = await run_search_phase(
backend=backend,
ctx=ctx,
phase="filtered_vector_search",
queries=selected_directory_queries,
top_k=options.top_k,
concurrency=options.concurrency,
filtered=True,
)
events.extend(filtered_events)
validation_errors.extend(errors)
quality["filtered_vector_search"] = filtered_quality
first_filter = selected_directory_queries[0].filter_path
event, _ = await async_timed_event(
"validate",
"count_filtered",
lambda: backend.count(PathScope("uri", first_filter, depth=-1), ctx=ctx),
extra={"filter_path": first_filter},
)
events.append(event)
if not event.success:
validation_errors.append(event.error or "count_filtered failed")
if options.drop_at_end:
progress("cleanup: drop collection")
event, dropped = await async_timed_event(
"cleanup", "drop_collection", lambda: backend.drop_collection()
)
events.append(event)
kept_collection = not bool(dropped)
if not event.success:
validation_errors.append(event.error or "drop_collection failed")
await close_backend(backend)
return finalize_result(
options,
workload,
collection_name,
events,
validation_errors,
quality,
kept_collection=kept_collection,
)
async def close_backend(backend: VikingVectorIndexBackend) -> None:
try:
await backend.close()
except Exception:
pass
def finalize_result(
options: BenchOptions,
workload: Workload,
collection_name: str,
events: list[Event],
validation_errors: list[str],
quality: dict[str, Any],
kept_collection: bool,
) -> RunResult:
return RunResult(
run_id=options.run_id,
collection_name=collection_name,
output_dir=str(options.output_dir),
events=events,
validation_errors=validation_errors,
quality=quality,
workload={
"name": workload.name,
"mode": options.mode,
"workload": options.workload,
"dataset": options.dataset if options.workload == "dir-vector" else "synthetic",
"record_count": workload.expected_rows,
"query_count": len(workload.queries),
"directory_query_count": len(directory_queries(workload)),
"vector_dim": workload.dim,
"top_k": options.top_k,
"full": options.full,
"profile": options.profile,
"path_diagnostics": workload.path_diagnostics,
},
environment=collect_environment(),
kept_collection=kept_collection,
)
def percentile(values: Sequence[float], pct: float) -> float:
if not values:
return 0.0
sorted_values = sorted(values)
index = min(len(sorted_values) - 1, max(0, math.ceil((pct / 100.0) * len(sorted_values)) - 1))
return sorted_values[index]
def phase_summary_rows(events: list[Event]) -> list[dict[str, Any]]:
grouped: dict[tuple[str, str], list[Event]] = {}
for event in events:
grouped.setdefault((event.phase, event.operation), []).append(event)
rows: list[dict[str, Any]] = []
for (phase, operation), items in sorted(grouped.items()):
latencies = [item.latency_ms for item in items]
wall_ms = phase_wall_ms(items)
count = sum(item.count or 1 for item in items)
rows.append(
{
"phase": phase,
"operation": operation,
"events": len(items),
"count": count,
"wall_ms": round(wall_ms, 3),
"success": sum(1 for item in items if item.success),
"errors": sum(1 for item in items if not item.success),
"avg_ms": round(sum(latencies) / len(latencies), 3),
"p50_ms": round(percentile(latencies, 50), 3),
"p95_ms": round(percentile(latencies, 95), 3),
"p99_ms": round(percentile(latencies, 99), 3),
"max_ms": round(max(latencies), 3),
"throughput_per_sec": round((count / wall_ms) * 1000.0, 3) if wall_ms > 0 else 0.0,
}
)
return rows
def phase_wall_ms(events: list[Event]) -> float:
starts: list[datetime] = []
ends: list[datetime] = []
for event in events:
try:
starts.append(datetime.fromisoformat(event.started_at))
ends.append(datetime.fromisoformat(event.ended_at))
except ValueError:
continue
if starts and ends:
return max(0.0, (max(ends) - min(starts)).total_seconds() * 1000.0)
return sum(event.latency_ms for event in events)
def collect_environment() -> dict[str, Any]:
env: dict[str, Any] = {
"observed_at": utc_now(),
"platform": platform.platform(),
"machine": platform.machine(),
"python": sys.version,
"pid": os.getpid(),
"cpu_count": os.cpu_count(),
"memory_bytes": total_memory_bytes(),
"cgroup": read_cgroup_limits(),
"gpu": nvidia_smi(),
"remote_resources_verified": False,
}
return env
def total_memory_bytes() -> Optional[int]:
try:
pages = os.sysconf("SC_PHYS_PAGES")
page_size = os.sysconf("SC_PAGE_SIZE")
return int(pages * page_size)
except (AttributeError, OSError, ValueError):
return None
def read_int_file(path: Path) -> Optional[int | str]:
try:
raw = path.read_text(encoding="utf-8").strip()
except OSError:
return None
if raw == "max":
return "max"
try:
return int(raw.split()[0])
except (ValueError, IndexError):
return raw
def read_cgroup_limits() -> dict[str, Any]:
return {
"cpu_max": read_int_file(Path("/sys/fs/cgroup/cpu.max")),
"memory_max": read_int_file(Path("/sys/fs/cgroup/memory.max")),
"memory_current": read_int_file(Path("/sys/fs/cgroup/memory.current")),
}
def nvidia_smi() -> list[dict[str, str]]:
if shutil.which("nvidia-smi") is None:
return []
try:
result = subprocess.run(
[
"nvidia-smi",
"--query-gpu=name,memory.total,memory.used,utilization.gpu",
"--format=csv,noheader,nounits",
],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
timeout=3,
)
except (OSError, subprocess.TimeoutExpired):
return []
if result.returncode != 0:
return []
rows = []
for line in result.stdout.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) >= 4:
rows.append(
{
"name": parts[0],
"memory_total_mb": parts[1],
"memory_used_mb": parts[2],
"utilization_gpu_percent": parts[3],
}
)
return rows
def write_outputs(result: RunResult, options: BenchOptions) -> None:
output_dir = Path(result.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
summary_rows = phase_summary_rows(result.events)
write_json(output_dir / "environment.json", result.environment)
config_json = {
**asdict(options),
"output_dir": str(options.output_dir),
"dataset_root": str(options.dataset_root) if options.dataset_root else None,
}
write_json(output_dir / "run_config.json", config_json)
write_json(
output_dir / "run_summary.json",
{
"run_id": result.run_id,
"collection_name": result.collection_name,
"kept_collection": result.kept_collection,
"validation_errors": result.validation_errors,
"workload": result.workload,
"quality": result.quality,
"phase_summary": summary_rows,
},
)
write_jsonl(output_dir / "events.jsonl", [asdict(event) for event in result.events])
write_csv(output_dir / "phase_summary.csv", summary_rows)
write_text(output_dir / "summary_zh.md", build_markdown_report(result, summary_rows, options))
def write_suite_outputs(output_dir: Path, runs: list[tuple[BenchOptions, RunResult]]) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
run_summaries = {
run_options.dataset: phase_summary_rows(result.events) for run_options, result in runs
}
rows = [
{
"dataset": run_options.dataset,
"mode": result.workload.get("mode"),
"records": result.workload.get("record_count"),
"inserted": metric_value(run_summaries[run_options.dataset], "ingest", "count"),
"queries": result.workload.get("query_count"),
"directory_queries": result.workload.get("directory_query_count"),
"dim": result.workload.get("vector_dim"),
"top_k": result.workload.get("top_k"),
"vector_qps": metric_value(
run_summaries[run_options.dataset], "vector_search", "throughput_per_sec"
),
"vector_recall": quality_rate(result.quality, "vector_search"),
"filtered_qps": metric_value(
run_summaries[run_options.dataset], "filtered_vector_search", "throughput_per_sec"
),
"filtered_recall": quality_rate(result.quality, "filtered_vector_search"),
"recall_scope": recall_scope(result),
"summary": str(Path(result.output_dir) / "summary_zh.md"),
"collection": result.collection_name,
"errors": len(result.validation_errors),
"kept_collection": result.kept_collection,
}
for run_options, result in runs
]
write_json(
output_dir / "run_summary.json",
{
"datasets": rows,
"validation_errors": {
run_options.dataset: result.validation_errors for run_options, result in runs
},
},
)
lines = [
"# OpenViking Vector Backend 真实数据汇总",
"",
"本次 `dir-vector` 真实数据 benchmark 覆盖 `wiki` 和 `arxiv` 两个数据集。",
"",
markdown_table(rows),
"",
]
for run_options, result in runs:
lines.extend(["", f"## {run_options.dataset} 明细", ""])
detail = (
build_markdown_report(result, run_summaries[run_options.dataset], run_options)
.strip()
.splitlines()
)
for line in detail[1:]:
lines.append("#" + line if line.startswith("#") else line)
write_text(output_dir / "summary_zh.md", "\n".join(lines))
def metric_row(summary_rows: list[dict[str, Any]], phase: str) -> dict[str, Any]:
for row in summary_rows:
if row.get("phase") == phase:
return row
return {}
def metric_value(summary_rows: list[dict[str, Any]], phase: str, key: str) -> Any:
value = metric_row(summary_rows, phase).get(key)
return "-" if value is None else value
def quality_rate(quality: dict[str, Any], phase: str) -> str:
value = quality.get(phase, {}).get("hit_rate_at_k")
if value is None:
return "-"
return f"{float(value) * 100.0:.2f}%"
def recall_scope(result: RunResult, phase: str = "filtered_vector_search") -> str:
query_scope_source = result.workload.get("path_diagnostics", {}).get("query_scope_source")
if phase == "filtered_vector_search" and query_scope_source and query_scope_source != "dataset":
return str(query_scope_source)
if result.workload.get("workload") == "dir-vector" and not result.workload.get("full"):
return "sampled_subset"
return "official_full"
def workload_rows(result: RunResult, summary_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
workload = result.workload
return [
{
"workload": workload.get("name"),
"mode": workload.get("mode"),
"dataset": workload.get("dataset"),
"records": workload.get("record_count"),
"inserted": metric_value(summary_rows, "ingest", "count"),
"queries": workload.get("query_count"),
"directory_queries": workload.get("directory_query_count"),
"dim": workload.get("vector_dim"),
"top_k": workload.get("top_k"),
"full": workload.get("full"),
}
]
def recall_qps_rows(
result: RunResult, summary_rows: list[dict[str, Any]], options: BenchOptions
) -> list[dict[str, Any]]:
recall_header = f"gt_recall@{options.top_k}"
return [
{
"phase": "vector_search",
"qps": metric_value(summary_rows, "vector_search", "throughput_per_sec"),
"avg_ms": metric_value(summary_rows, "vector_search", "avg_ms"),
"p95_ms": metric_value(summary_rows, "vector_search", "p95_ms"),
"scope": recall_scope(result, "vector_search"),
recall_header: quality_rate(result.quality, "vector_search"),
},
{
"phase": "filtered_vector_search",
"qps": metric_value(summary_rows, "filtered_vector_search", "throughput_per_sec"),
"avg_ms": metric_value(summary_rows, "filtered_vector_search", "avg_ms"),
"p95_ms": metric_value(summary_rows, "filtered_vector_search", "p95_ms"),
"scope": recall_scope(result, "filtered_vector_search"),
recall_header: quality_rate(result.quality, "filtered_vector_search"),
},
]
def build_markdown_report(
result: RunResult, summary_rows: list[dict[str, Any]], options: BenchOptions
) -> str:
lines = [
"# OpenViking Vector Backend 性能验收报告",
"",
f"- Run ID: `{result.run_id}`",
f"- Workload: `{options.workload}`",
f"- Mode: `{options.mode}`",
f"- Collection: `{result.collection_name}`",
f"- 功能错误: `{len(result.validation_errors)}`",
f"- Collection 保留: `{result.kept_collection}`",
"",
"## 结论",
"",
]
if result.validation_errors:
lines.append("功能验收未通过。性能慢只记录,以下功能错误会导致非零退出码:")
for error in result.validation_errors[:20]:
lines.append(f"- {error}")
else:
lines.append("功能验收通过。")
lines.extend(
[
"",
"## 数据规模",
"",
markdown_table(workload_rows(result, summary_rows)),
"",
"## Directory path 口径",
"",
f"- Corpus path source: `{result.workload.get('path_diagnostics', {}).get('corpus_path_source', 'generated')}`",
f"- Query scope source: `{result.workload.get('path_diagnostics', {}).get('query_scope_source', 'generated')}`",
f"- Directory queries: `{result.workload.get('directory_query_count')}`",
f"- Unique directory scopes: `{result.workload.get('path_diagnostics', {}).get('unique_directory_scopes', '-')}`",
f"- Root-derived scopes omitted: `{result.workload.get('path_diagnostics', {}).get('root_derived_scopes_omitted', 0)}`",
"",
"## 召回与 QPS",
"",
markdown_table(recall_qps_rows(result, summary_rows, options)),
"",
"gt_recall@K 按 query 的 ground truth 命中率统计;`sampled_subset` 只用于 smoke sanity,不代表官方全量 recall。`derived_gt_lca_v1` 使用 ground truth 派生目录,只是诊断 workload,也不代表发布数据集的官方 query constraint。",
"",
"## 性能汇总",
"",
markdown_table(summary_rows),
"",
"## 环境说明",
"",
f"- Platform: `{result.environment.get('platform')}`",
f"- CPU count: `{result.environment.get('cpu_count')}`",
f"- Memory bytes: `{result.environment.get('memory_bytes')}`",
f"- GPU: `{result.environment.get('gpu') or 'not detected'}`",
"- 远端/GPU 服务资源未由本 runner 验证;报告只记录 runner 本机观测信息。",
"",
"## 清理",
"",
]
)
if result.kept_collection:
lines.append(
"默认保留测试 collection。需要清理时,用相同配置连接后删除 "
f"`{result.collection_name}`。"
)
else:
lines.append("本次运行已尝试删除测试 collection。")
lines.extend(
[
"",
"## 输出文件",
"",
"- `run_summary.json`: 汇总数据",
"- `events.jsonl`: 每次操作明细",
"- `phase_summary.csv`: 阶段聚合",
"- `environment.json`: 本机环境观测",
]
)
return "\n".join(lines) + "\n"
def markdown_table(rows: list[dict[str, Any]]) -> str:
if not rows:
return "无数据。"
headers = list(rows[0].keys())
lines = [
"| " + " | ".join(headers) + " |",
"| " + " | ".join("---" for _ in headers) + " |",
]
for row in rows:
lines.append("| " + " | ".join(str(row.get(header, "")) for header in headers) + " |")
return "\n".join(lines)
def write_json(path: Path, value: Any) -> None:
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
def write_text(path: Path, value: str) -> None:
path.write_text(value, encoding="utf-8")
def write_jsonl(path: Path, values: Iterable[Any]) -> None:
with path.open("w", encoding="utf-8") as handle:
for value in values:
handle.write(json.dumps(value, ensure_ascii=False, default=str) + "\n")
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
if not rows:
path.write_text("", encoding="utf-8")
return
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys()))
writer.writeheader()
writer.writerows(rows)
def parse_args(argv: Optional[Sequence[str]] = None) -> BenchOptions:
parser = argparse.ArgumentParser(
description="Run OpenViking vector backend performance benchmark"
)
parser.add_argument("--config", help="Path to ov.conf. Defaults to OpenViking config lookup.")
parser.add_argument("--output-dir", type=Path)
parser.add_argument("--run-id")
parser.add_argument("--profile", choices=sorted(PROFILE_DEFAULTS), default="smoke")
parser.add_argument(
"--mode",
choices=["read-write", "write-only", "read-only"],
default="read-write",
help="read-write creates, writes, and reads; write-only skips reads; read-only skips create/upsert",
)
parser.add_argument("--workload", choices=["synthetic", "dir-vector"], default="synthetic")
parser.add_argument("--dataset-root", type=Path)
parser.add_argument(
"--dataset",
choices=["all", *sorted(DIR_VECTOR_FILES)],
default="all",
help="dir-vector dataset; default runs wiki and arxiv",
)
parser.add_argument("--full", action="store_true", help="Run full dir-vector corpus/query set")
parser.add_argument("--rows", type=int)
parser.add_argument("--queries", type=int)
parser.add_argument("--dim", type=int)
parser.add_argument("--batch-size", type=int)
parser.add_argument("--concurrency", type=int)
parser.add_argument("--top-k", type=int)
parser.add_argument("--path-depth", type=int)
parser.add_argument("--path-fanout", type=int)
parser.add_argument("--filter-selectivity", type=float)
parser.add_argument(
"--dir-vector-query-scope",
choices=DIR_VECTOR_QUERY_SCOPES,
default="dataset",
help=(
"Directory scope source for dir-vector queries. dataset requires published query "
"constraints; derived_gt_lca_v1 gives Wiki an explicit diagnostic scope derived "
"from the deepest common path of highest-score ground-truth items. Other datasets "
"continue to use their published constraints."
),
)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--distance", choices=["ip", "l2", "cosine"])
parser.add_argument("--drop-at-end", action="store_true", help="Drop benchmark collection")
args = parser.parse_args(argv)
if args.mode == "read-only" and args.drop_at_end:
raise ValueError("--drop-at-end is not allowed with --mode read-only")
defaults = PROFILE_DEFAULTS[args.profile]
run_id = (
args.run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "_" + uuid.uuid4().hex[:8]
)
output_dir = args.output_dir or Path("benchmark/results/vectordb_perf") / run_id
return BenchOptions(
config=args.config,
output_dir=output_dir,
run_id=run_id,
profile=args.profile,
mode=args.mode,
workload=args.workload,
dataset_root=args.dataset_root,
dataset=args.dataset,
full=args.full,
rows=args.rows if args.rows is not None else defaults["rows"],
queries=args.queries if args.queries is not None else defaults["queries"],
dim=args.dim if args.dim is not None else defaults["dim"],
batch_size=args.batch_size if args.batch_size is not None else defaults["batch_size"],
concurrency=args.concurrency if args.concurrency is not None else defaults["concurrency"],
top_k=args.top_k if args.top_k is not None else defaults["top_k"],
path_depth=args.path_depth if args.path_depth is not None else defaults["path_depth"],
path_fanout=args.path_fanout if args.path_fanout is not None else defaults["path_fanout"],
filter_selectivity=args.filter_selectivity
if args.filter_selectivity is not None
else defaults["filter_selectivity"],
seed=args.seed,
distance=args.distance,
drop_at_end=args.drop_at_end,
dir_vector_query_scope=args.dir_vector_query_scope,
)
def main(argv: Optional[Sequence[str]] = None) -> int:
try:
options = parse_args(argv)
runs = run_benchmark_suite(options)
except (FileNotFoundError, ValueError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
for run_options, result in runs:
write_outputs(result, run_options)
if len(runs) > 1:
write_suite_outputs(options.output_dir, runs)
print(f"summary: {options.output_dir / 'summary_zh.md'}")
else:
print(f"summary: {Path(runs[0][1].output_dir) / 'summary_zh.md'}")
return 1 if any(result.validation_errors for _, result in runs) else 0
if __name__ == "__main__":
raise SystemExit(main())