Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(vectordb): batch benchmark upserts (#3264)

Yuanqing ZHAO0080f94bdcb8
diff --git a/benchmark/vectordb_perf/README.md b/benchmark/vectordb_perf/README.md
index e45d0c7e1..913daabf4 100644
--- a/benchmark/vectordb_perf/README.md
+++ b/benchmark/vectordb_perf/README.md
@@ -4,7 +4,7 @@
它不直接调用 `CollectionAdapter`,而是走 `VikingVectorIndexBackend`:
- 建表使用 `CollectionSchemas.context_collection`
-- 写入使用 `VikingVectorIndexBackend.upsert(..., ctx=...)`
+- 写入使用 `VikingVectorIndexBackend.upsert_many(..., ctx=...)`
- 查询使用 `VikingVectorIndexBackend.search_in_tenant(...)`
- 目录范围使用 `target_directories`,内部会编译成 `PathScope("uri", ..., depth=-1)`
@@ -192,6 +192,13 @@ runner 会把配置里的 `name` 改成 `<name>_bench_<run-id>`,避免覆盖
功能错误会导致非零退出码;性能慢只记录到报告里。
+`ingest` 会把每个 `--batch-size` 分组通过一次 backend `upsert_many` 调用写入,
+而不是逐行调用 `upsert`。对本地后端,这会把每批记录合并到一次 store 持久化操作;
+store 与 vector index 之间不提供跨组件事务。远端 adapter 还可根据请求上限进一步拆批,
+因此一次 `upsert_many` 也不代表一个远端请求或跨子批事务。批量接口只提供完整记录
+upsert;需要保留未提供字段的 partial update 仍使用单条 `upsert`。当前 runner 是该接口
+在仓库内的首个调用方;本改动不自动改变现有 embedding、migration 或 ovpack 写入流程。
+
`--mode write-only` 只会出现 `setup` / `ingest` / 可选 `cleanup` 阶段。
`--mode read-only` 只会出现 `validate` / `vector_search` / `filtered_vector_search` 阶段。
运行中会向 stderr 输出进度,包括当前 collection、写入条数、查询条数和速率。
@@ -293,7 +300,7 @@ Volcengine VikingDB:
| `--rows` | synthetic 行数;dir-vector 抽样行数 |
| `--queries` | 查询数 |
| `--dim` | synthetic 向量维度 |
-| `--batch-size` | upsert 批大小 |
+| `--batch-size` | 每次 backend `upsert_many` 调用的记录数;远端 adapter 可按请求上限再拆分 |
| `--concurrency` | 查询并发 |
| `--top-k` | 检索返回条数 |
| `--distance` | `ip`、`l2`、`cosine` |
diff --git a/benchmark/vectordb_perf/run.py b/benchmark/vectordb_perf/run.py
index d7ae7aebd..a221da7c3 100644
--- a/benchmark/vectordb_perf/run.py
+++ b/benchmark/vectordb_perf/run.py
@@ -30,15 +30,13 @@ from pathlib import Path
from typing import Any, Callable, Iterable, Iterator, Optional, Sequence
from benchmark.vectordb_perf.async_utils import map_bounded_as_completed
-
-from openviking.storage.expr import PathScope
from openviking.server.identity import RequestContext, Role
from openviking.storage.collection_schemas import CollectionSchemas
+from openviking.storage.expr import PathScope
from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config import OpenVikingConfigSingleton
-
BENCH_ACCOUNT_ID = "bench_account"
BENCH_USER_ID = "bench_user"
DIR_VECTOR_DATASET_URL = "https://github.com/KurtPatrickHere/dir-vector-dataset"
@@ -524,7 +522,9 @@ def build_synthetic_workload(options: BenchOptions) -> Workload:
cases: list[QueryCase] = []
for qindex in range(query_count):
record_index = (qindex * max(1, rows // query_count)) % rows
- full_path = synthetic_path(record_index, depth=options.path_depth, fanout=options.path_fanout)
+ full_path = synthetic_path(
+ record_index, depth=options.path_depth, fanout=options.path_fanout
+ )
cases.append(
QueryCase(
query_id=f"syn-q{qindex}",
@@ -619,7 +619,7 @@ def build_dir_vector_workload(options: BenchOptions) -> Workload:
if sample_metadata is not None
else iter_json_records(paths["corpus"], row_limit)
)
- for index, (meta, vector) in enumerate(zip(metadata_iter, vector_iter)):
+ for index, (meta, vector) in enumerate(zip(metadata_iter, vector_iter, strict=False)):
rid = record_id(meta, index)
path = extract_path(meta)
category = str(pick_value(meta, ("category", "primary_category", "type")) or "")
@@ -638,7 +638,7 @@ def build_dir_vector_workload(options: BenchOptions) -> Workload:
ground_truth = load_ground_truth(paths["ground_truth"])
covered_cases: list[QueryCase] = []
fallback_cases: list[QueryCase] = []
- for index, (query_meta, vector) in enumerate(zip(query_records, query_vectors)):
+ for index, (query_meta, vector) in enumerate(zip(query_records, query_vectors, strict=False)):
qid = record_id(query_meta, index)
gt = [str(item) for item in (ground_truth.get(qid) or ground_truth.get(str(index)) or [])]
if sampled_ids is not None:
@@ -760,7 +760,10 @@ async def run_search_phase(
level=[2],
limit=top_k,
),
- extra={"query_id": query.query_id, "filter_path": query.filter_path if filtered else ""},
+ extra={
+ "query_id": query.query_id,
+ "filter_path": query.filter_path if filtered else "",
+ },
)
events: list[Event] = []
@@ -785,13 +788,12 @@ async def run_search_phase(
force=done == len(queries),
)
else:
+
async def tagged_query(query: QueryCase):
event, rows = await one_query(query)
return query, event, rows
- async for query, event, rows in map_bounded_as_completed(
- queries, tagged_query, workers
- ):
+ async for query, event, rows in map_bounded_as_completed(queries, tagged_query, workers):
events.append(event)
results.append((query, rows or []))
done += 1
@@ -843,11 +845,9 @@ def benchmark_context() -> RequestContext:
async def upsert_records(
backend: VikingVectorIndexBackend, ctx: RequestContext, records: list[dict[str, Any]]
) -> list[str]:
- ids: list[str] = []
- for record in records:
- inserted = await backend.upsert(record, ctx=ctx)
- if inserted:
- ids.append(inserted)
+ ids = await backend.upsert_many(records, ctx=ctx)
+ if len(ids) != len(records):
+ raise RuntimeError(f"bulk upsert returned {len(ids)} ids for {len(records)} records")
return ids
@@ -946,13 +946,13 @@ async def run_benchmark_async(options: BenchOptions) -> RunResult:
fetch_ids.extend(str(row["id"]) for row in batch[: 20 - len(fetch_ids)])
event, ids = await async_timed_event(
"ingest",
- "upsert_rows_serial",
+ "upsert_rows_batch",
lambda batch=batch: upsert_records(backend, ctx, batch),
count=len(batch),
)
events.append(event)
if not event.success:
- validation_errors.append(event.error or "upsert_rows_serial failed")
+ validation_errors.append(event.error or "upsert_rows_batch failed")
break
inserted += len(ids or [])
last_ingest_reported_at = progress_count(
@@ -1018,7 +1018,9 @@ async def run_benchmark_async(options: BenchOptions) -> RunResult:
if not event.success:
validation_errors.append(event.error or "fetch_by_ids failed")
elif len(fetched or []) != len(fetch_ids):
- validation_errors.append(f"fetch_by_ids returned {len(fetched or [])}, expected {len(fetch_ids)}")
+ validation_errors.append(
+ f"fetch_by_ids returned {len(fetched or [])}, expected {len(fetch_ids)}"
+ )
selected_queries = workload.queries
search_events, errors, vector_quality = await run_search_phase(
@@ -1152,9 +1154,7 @@ def phase_summary_rows(events: list[Event]) -> list[dict[str, Any]]:
"p95_ms": round(percentile(latencies, 95), 3),
"p99_ms": round(percentile(latencies, 99), 3),
"max_ms": round(max(latencies), 3),
- "throughput_per_sec": round((count / wall_ms) * 1000.0, 3)
- if wall_ms > 0
- else 0.0,
+ "throughput_per_sec": round((count / wall_ms) * 1000.0, 3) if wall_ms > 0 else 0.0,
}
)
return rows
@@ -1283,9 +1283,7 @@ def write_outputs(result: RunResult, options: BenchOptions) -> None:
write_text(output_dir / "summary_zh.md", build_markdown_report(result, summary_rows, options))
-def write_suite_outputs(
- output_dir: Path, runs: list[tuple[BenchOptions, RunResult]]
-) -> None:
+def write_suite_outputs(output_dir: Path, runs: list[tuple[BenchOptions, RunResult]]) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
run_summaries = {
run_options.dataset: phase_summary_rows(result.events) for run_options, result in runs
@@ -1299,7 +1297,9 @@ def write_suite_outputs(
"queries": result.workload.get("query_count"),
"dim": result.workload.get("vector_dim"),
"top_k": result.workload.get("top_k"),
- "vector_qps": metric_value(run_summaries[run_options.dataset], "vector_search", "throughput_per_sec"),
+ "vector_qps": metric_value(
+ run_summaries[run_options.dataset], "vector_search", "throughput_per_sec"
+ ),
"vector_recall": quality_rate(result.quality, "vector_search"),
"filtered_qps": metric_value(
run_summaries[run_options.dataset], "filtered_vector_search", "throughput_per_sec"
@@ -1332,9 +1332,11 @@ def write_suite_outputs(
]
for run_options, result in runs:
lines.extend(["", f"## {run_options.dataset} 明细", ""])
- detail = build_markdown_report(
- result, run_summaries[run_options.dataset], run_options
- ).strip().splitlines()
+ detail = (
+ build_markdown_report(result, run_summaries[run_options.dataset], run_options)
+ .strip()
+ .splitlines()
+ )
for line in detail[1:]:
lines.append("#" + line if line.startswith("#") else line)
write_text(output_dir / "summary_zh.md", "\n".join(lines))
@@ -1518,7 +1520,9 @@ def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
def parse_args(argv: Optional[Sequence[str]] = None) -> BenchOptions:
- parser = argparse.ArgumentParser(description="Run OpenViking vector backend performance benchmark")
+ parser = argparse.ArgumentParser(
+ description="Run OpenViking vector backend performance benchmark"
+ )
parser.add_argument("--config", help="Path to ov.conf. Defaults to OpenViking config lookup.")
parser.add_argument("--output-dir", type=Path)
parser.add_argument("--run-id")
@@ -1556,7 +1560,9 @@ def parse_args(argv: Optional[Sequence[str]] = None) -> BenchOptions:
raise ValueError("--drop-at-end is not allowed with --mode read-only")
defaults = PROFILE_DEFAULTS[args.profile]
- run_id = args.run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "_" + uuid.uuid4().hex[:8]
+ run_id = (
+ args.run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "_" + uuid.uuid4().hex[:8]
+ )
output_dir = args.output_dir or Path("benchmark/results/vectordb_perf") / run_id
return BenchOptions(
config=args.config,
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index da1b1fd66..1295009ff 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -179,6 +179,32 @@ class _SingleAccountBackend:
return result
+ def _prepare_upsert_payloads(self, data_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Prepare a batch in one worker-thread handoff."""
+ return [self._prepare_upsert_payload(data) for data in data_list]
+
+ def _bind_upsert_payload(self, data: Dict[str, Any]) -> Dict[str, Any]:
+ """Copy a record, enforce its bound account, and apply write defaults."""
+ payload = dict(data)
+ if self._bound_account_id:
+ account_id = payload.get("account_id")
+ if account_id and account_id != self._bound_account_id:
+ raise PermissionError(
+ "record account_id does not match the request context account_id"
+ )
+ payload["account_id"] = self._bound_account_id
+
+ context_type = payload.get("context_type")
+ if context_type and context_type not in VikingVectorIndexBackend.ALLOWED_CONTEXT_TYPES:
+ raise ValueError(
+ f"Invalid context_type: {context_type}. "
+ f"Must be one of {sorted(VikingVectorIndexBackend.ALLOWED_CONTEXT_TYPES)}"
+ )
+
+ if not payload.get("id"):
+ payload["id"] = str(uuid.uuid4())
+ return payload
+
@staticmethod
def _is_not_found_error(exc: Exception) -> bool:
message = str(exc).lower()
@@ -265,28 +291,12 @@ class _SingleAccountBackend:
# =========================================================================
async def upsert(self, data: Dict[str, Any], partial_update: bool = False) -> str:
- payload = dict(data)
- logger.debug(
- f"[_SingleAccountBackend.upsert] Input data.account_id={payload.get('account_id')}, bound_account_id={self._bound_account_id}"
- )
- if self._bound_account_id and not payload.get("account_id"):
- payload["account_id"] = self._bound_account_id
- logger.debug(
- f"[_SingleAccountBackend.upsert] Final payload.account_id={payload.get('account_id')}"
- )
-
- context_type = payload.get("context_type")
- if context_type and context_type not in VikingVectorIndexBackend.ALLOWED_CONTEXT_TYPES:
- logger.warning(
- "Invalid context_type: %s. Must be one of %s",
- context_type,
- sorted(VikingVectorIndexBackend.ALLOWED_CONTEXT_TYPES),
- )
+ try:
+ payload = self._bind_upsert_payload(data)
+ except (PermissionError, ValueError) as exc:
+ logger.warning("Rejecting upsert: %s", exc)
return ""
- if not payload.get("id"):
- payload["id"] = str(uuid.uuid4())
-
if partial_update:
try:
existing_records = await self._async_adapter.call("get", [payload["id"]])
@@ -309,6 +319,43 @@ class _SingleAccountBackend:
ids = await self._async_adapter.call("upsert", payload)
return ids[0] if ids else ""
+ async def upsert_many(self, data_list: List[Dict[str, Any]]) -> List[str]:
+ """Bulk full-record upsert through one adapter call.
+
+ The batch is validated before the adapter is invoked. Partial-update
+ semantics intentionally remain on :meth:`upsert`, where each existing
+ record is read and merged independently. Returned IDs preserve input
+ order; invalid batches raise before the adapter is invoked.
+ """
+ if not data_list:
+ return []
+
+ payloads: List[Dict[str, Any]] = []
+ seen_ids: set[str] = set()
+ for index, data in enumerate(data_list):
+ try:
+ payload = self._bind_upsert_payload(data)
+ except PermissionError as exc:
+ raise PermissionError(f"record at index {index}: {exc}") from exc
+ except ValueError as exc:
+ raise ValueError(f"record at index {index}: {exc}") from exc
+ record_id = str(payload["id"])
+ if record_id in seen_ids:
+ raise ValueError(f"duplicate record id at index {index}")
+ seen_ids.add(record_id)
+ payloads.append(payload)
+
+ payloads = await self._async_adapter.run(self._prepare_upsert_payloads, payloads)
+ ids = await self._async_adapter.call("upsert", payloads)
+ normalized_ids = [str(item) for item in (ids or []) if item is not None]
+ expected_ids = [str(payload["id"]) for payload in payloads]
+ if normalized_ids != expected_ids:
+ raise RuntimeError(
+ "bulk upsert adapter returned IDs that do not match the input count and order "
+ f"(expected {len(expected_ids)}, got {len(normalized_ids)})"
+ )
+ return normalized_ids
+
async def update(self, data: Dict[str, Any]) -> UpdateResult:
"""Strict update path. The target record must already exist."""
try:
@@ -776,15 +823,47 @@ class VikingVectorIndexBackend:
fields before issuing the final upsert.
"""
logger.debug(
- f"[VikingVectorIndexBackend.upsert] Called with ctx.account_id={ctx.account_id}, partial_update={partial_update}, data={data}"
+ "[VikingVectorIndexBackend.upsert] Called with ctx.account_id=%s, "
+ "partial_update=%s, data=%s",
+ ctx.account_id,
+ partial_update,
+ data,
)
backend = self._get_backend_for_context(ctx)
logger.debug(
- f"[VikingVectorIndexBackend.upsert] Using backend for account_id={ctx.account_id}"
+ "[VikingVectorIndexBackend.upsert] Using backend for account_id=%s",
+ ctx.account_id,
)
result = await backend.upsert(data, partial_update=partial_update)
logger.debug(
- f"[VikingVectorIndexBackend.upsert] Completed with partial_update={partial_update}, result={result}"
+ "[VikingVectorIndexBackend.upsert] Completed with partial_update=%s, result=%s",
+ partial_update,
+ result,
+ )
+ return result
+
+ async def upsert_many(
+ self, data_list: List[Dict[str, Any]], *, ctx: RequestContext
+ ) -> List[str]:
+ """Bulk full-record upsert.
+
+ All records are validated for the bound account before one adapter
+ invocation is made. Adapters may split that invocation into multiple
+ data-plane requests, so this API does not guarantee transaction
+ atomicity. Use :meth:`upsert` with ``partial_update=True`` when omitted
+ fields must be preserved from existing records.
+ """
+ logger.debug(
+ "[VikingVectorIndexBackend.upsert_many] Called with ctx.account_id=%s, count=%s",
+ ctx.account_id,
+ len(data_list),
+ )
+ backend = self._get_backend_for_context(ctx)
+ result = await backend.upsert_many(data_list)
+ logger.debug(
+ "[VikingVectorIndexBackend.upsert_many] Completed with count=%s, result_count=%s",
+ len(data_list),
+ len(result),
)
return result
diff --git a/openviking/storage/vikingdb_manager.py b/openviking/storage/vikingdb_manager.py
index 39c1247e9..3231f6c4f 100644
--- a/openviking/storage/vikingdb_manager.py
+++ b/openviking/storage/vikingdb_manager.py
@@ -294,6 +294,10 @@ class VikingDBManagerProxy:
"""
return await self._manager.upsert(data, ctx=self._ctx, partial_update=partial_update)
+ async def upsert_many(self, data_list: List[Dict[str, Any]]) -> List[str]:
+ """Bulk full-record upsert with the proxy's bound request context."""
+ return await self._manager.upsert_many(data_list, ctx=self._ctx)
+
async def get(self, ids: List[str]) -> List[Dict[str, Any]]:
return await self._manager.get(ids, ctx=self._ctx)
diff --git a/tests/benchmark/test_vectordb_perf_run.py b/tests/benchmark/test_vectordb_perf_run.py
new file mode 100644
index 000000000..fed1eb9e7
--- /dev/null
+++ b/tests/benchmark/test_vectordb_perf_run.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import pytest
+
+from benchmark.vectordb_perf.run import upsert_records
+
+
+async def test_upsert_records_uses_one_bulk_backend_call():
+ calls = []
+
+ class _Backend:
+ async def upsert_many(self, records, *, ctx):
+ calls.append((records, ctx))
+ return [record["id"] for record in records]
+
+ async def upsert(self, record, *, ctx): # pragma: no cover - should never run
+ raise AssertionError(f"unexpected serial upsert: {record}, {ctx}")
+
+ backend = _Backend()
+ ctx = object()
+ records = [{"id": "rec-1"}, {"id": "rec-2"}]
+
+ ids = await upsert_records(backend, ctx, records)
+
+ assert ids == ["rec-1", "rec-2"]
+ assert calls == [(records, ctx)]
+
+
+async def test_upsert_records_rejects_incomplete_bulk_result():
+ class _Backend:
+ async def upsert_many(self, records, *, ctx):
+ del records, ctx
+ return ["rec-1"]
+
+ with pytest.raises(RuntimeError, match="returned 1 ids for 2 records"):
+ await upsert_records(
+ _Backend(),
+ object(),
+ [{"id": "rec-1"}, {"id": "rec-2"}],
+ )
diff --git a/tests/storage/test_bulk_upsert.py b/tests/storage/test_bulk_upsert.py
new file mode 100644
index 000000000..80ffdba2a
--- /dev/null
+++ b/tests/storage/test_bulk_upsert.py
@@ -0,0 +1,268 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import uuid
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.viking_vector_index_backend import (
+ VikingVectorIndexBackend,
+ _SingleAccountBackend,
+)
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig
+
+
+def make_backend(adapter):
+ return _SingleAccountBackend(
+ config=VectorDBBackendConfig(backend="local", name="context", dimension=4),
+ bound_account_id="acc1",
+ shared_adapter=adapter,
+ )
+
+
+async def test_single_account_backend_upsert_many_uses_one_adapter_call():
+ calls = []
+
+ class _Collection:
+ def get_meta_data(self):
+ return {
+ "Fields": [
+ {"FieldName": "id"},
+ {"FieldName": "uri"},
+ {"FieldName": "abstract", "FieldType": "text"},
+ {"FieldName": "account_id"},
+ {"FieldName": "context_type"},
+ ]
+ }
+
+ class _Adapter:
+ mode = "local"
+ USE_CONTENT_FIELD = False
+
+ def get_collection(self):
+ return _Collection()
+
+ def upsert(self, data):
+ calls.append(data)
+ return [row["id"] for row in data]
+
+ backend = make_backend(_Adapter())
+ records = [
+ {
+ "id": "rec-1",
+ "uri": "viking://resources/one",
+ "abstract": "one",
+ "context_type": "resource",
+ "unknown": "ignored",
+ },
+ {
+ "uri": "viking://resources/two",
+ "context_type": "resource",
+ },
+ ]
+
+ ids = await backend.upsert_many(records)
+
+ assert len(calls) == 1
+ assert isinstance(calls[0], list)
+ assert ids == [row["id"] for row in calls[0]]
+ assert ids[0] == "rec-1"
+ uuid.UUID(ids[1])
+ assert calls[0] == [
+ {
+ "id": "rec-1",
+ "uri": "viking://resources/one",
+ "abstract": "one",
+ "account_id": "acc1",
+ "context_type": "resource",
+ },
+ {
+ "id": ids[1],
+ "uri": "viking://resources/two",
+ "abstract": "",
+ "account_id": "acc1",
+ "context_type": "resource",
+ },
+ ]
+ assert "id" not in records[1]
+ assert "account_id" not in records[0]
+ assert records[0]["unknown"] == "ignored"
+
+
+async def test_single_account_backend_upsert_many_empty_batch_skips_adapter():
+ class _Adapter:
+ mode = "local"
+ USE_CONTENT_FIELD = False
+
+ def upsert(self, data): # pragma: no cover - should never run
+ raise AssertionError(f"unexpected upsert: {data}")
+
+ assert await make_backend(_Adapter()).upsert_many([]) == []
+
+
+ ("records", "error_type", "match"),
+ [
+ (
+ [{"id": "rec-1", "context_type": "invalid"}],
+ ValueError,
+ "record at index 0: Invalid context_type",
+ ),
+ (
+ [{"id": "rec-1", "account_id": "another-account"}],
+ PermissionError,
+ "record at index 0: record account_id does not match",
+ ),
+ (
+ [{"id": "rec-1"}, {"id": "rec-1"}],
+ ValueError,
+ "duplicate record id at index 1",
+ ),
+ ],
+)
+async def test_single_account_backend_upsert_many_rejects_invalid_batch_before_write(
+ records, error_type, match
+):
+ class _Adapter:
+ mode = "local"
+ USE_CONTENT_FIELD = False
+
+ def upsert(self, data): # pragma: no cover - should never run
+ raise AssertionError(f"unexpected upsert: {data}")
+
+ with pytest.raises(error_type, match=match):
+ await make_backend(_Adapter()).upsert_many(records)
+
+
+async def test_single_account_backend_upsert_rejects_foreign_account_before_write():
+ class _Adapter:
+ mode = "local"
+ USE_CONTENT_FIELD = False
+
+ def upsert(self, data): # pragma: no cover - should never run
+ raise AssertionError(f"unexpected upsert: {data}")
+
+ result = await make_backend(_Adapter()).upsert({"id": "rec-1", "account_id": "another-account"})
+
+ assert result == ""
+
+
[email protected]("returned_ids", [["rec-1"], ["rec-2", "rec-1"]])
+async def test_single_account_backend_upsert_many_rejects_invalid_adapter_result(returned_ids):
+ class _Collection:
+ def get_meta_data(self):
+ return {
+ "Fields": [
+ {"FieldName": "id"},
+ {"FieldName": "account_id"},
+ ]
+ }
+
+ class _Adapter:
+ mode = "local"
+ USE_CONTENT_FIELD = False
+
+ def get_collection(self):
+ return _Collection()
+
+ def upsert(self, data):
+ del data
+ return returned_ids
+
+ with pytest.raises(RuntimeError, match="do not match the input count and order"):
+ await make_backend(_Adapter()).upsert_many([{"id": "rec-1"}, {"id": "rec-2"}])
+
+
+async def test_viking_vector_index_backend_upsert_many_delegates_once():
+ backend = object.__new__(VikingVectorIndexBackend)
+ ctx = SimpleNamespace(account_id="acc1")
+ records = [{"id": "rec-1"}, {"id": "rec-2"}]
+ calls = []
+
+ class _BoundBackend:
+ async def upsert_many(self, data_list):
+ calls.append(data_list)
+ return [row["id"] for row in data_list]
+
+ backend._get_backend_for_context = lambda _ctx: _BoundBackend()
+
+ assert await backend.upsert_many(records, ctx=ctx) == ["rec-1", "rec-2"]
+ assert calls == [records]
+
+
+async def test_viking_vector_index_backend_upsert_many_persists_batch(tmp_path):
+ collection_name = "bulk_upsert_test"
+ backend = VikingVectorIndexBackend(
+ VectorDBBackendConfig(
+ backend="local",
+ name=collection_name,
+ dimension=4,
+ path=str(tmp_path),
+ )
+ )
+ ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
+ schema = {
+ "CollectionName": collection_name,
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {"FieldName": "uri", "FieldType": "path"},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},
+ {"FieldName": "abstract", "FieldType": "string"},
+ {"FieldName": "account_id", "FieldType": "string"},
+ {"FieldName": "context_type", "FieldType": "string"},
+ ],
+ "ScalarIndex": ["uri", "account_id", "context_type"],
+ }
+ records = [
+ {
+ "id": f"rec-{index}",
+ "uri": f"viking://resources/{index}",
+ "vector": [1.0, 0.0, 0.0, float(index)],
+ "abstract": f"record {index}",
+ "context_type": "resource",
+ }
+ for index in range(2)
+ ]
+
+ try:
+ assert await backend.create_collection(collection_name, schema)
+ assert await backend.upsert_many(records, ctx=ctx) == ["rec-0", "rec-1"]
+ assert await backend.count(ctx=ctx) == 2
+ fetched = await backend.get(["rec-0", "rec-1"], ctx=ctx)
+ assert [row["id"] for row in fetched] == ["rec-0", "rec-1"]
+ assert all(row["account_id"] == "acc1" for row in fetched)
+ finally:
+ await backend.close()
+
+
+async def test_vikingdb_manager_proxy_upsert_many_forwards_bound_context():
+ from openviking.storage.vikingdb_manager import VikingDBManagerProxy
+
+ ctx = SimpleNamespace(account_id="acc1")
+ captured = {}
+
+ class _Manager:
+ collection_name = "context"
+ mode = "local"
+
+ async def upsert_many(self, data_list, *, ctx):
+ captured["data_list"] = data_list
+ captured["ctx"] = ctx
+ return [row["id"] for row in data_list]
+
+ proxy = VikingDBManagerProxy(_Manager(), ctx)
+ records = [{"id": "rec-1"}, {"id": "rec-2"}]
+
+ assert await proxy.upsert_many(records) == ["rec-1", "rec-2"]
+ assert captured == {"data_list": records, "ctx": ctx}