Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(vectordb): avoid hydrating unrequested vectors (#3375)

Yuanqing ZHAOe1cea0998b3e
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 0e9c61493..3f5e5fcb2 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -533,46 +533,64 @@ class LocalCollection(ICollection):
pk_list = label_list
fields_list = []
- if not output_fields:
- output_fields = list(self.meta.fields_dict.keys())
- if self.meta.primary_key or output_fields:
+ projected_fields = (
+ list(self.meta.fields_dict.keys()) if not output_fields else list(output_fields)
+ )
+ include_vector = bool(self.meta.vector_key) and self.meta.vector_key in projected_fields
+ if self.meta.primary_key or projected_fields:
if not self.store_mgr:
raise RuntimeError("Store manager is not initialized")
- cands_list = self.store_mgr.fetch_cands_data(label_list)
+
+ cands_vectors: Optional[List[List[float]]] = None
+ if include_vector:
+ cands_list = self.store_mgr.fetch_cands_data(label_list)
+ cands_fields_payloads = [
+ cand.fields if cand is not None else None for cand in cands_list
+ ]
+ cands_vectors = [cand.vector if cand is not None else [] for cand in cands_list]
+ else:
+ cands_fields_payloads = self.store_mgr.fetch_cands_fields(label_list)
valid_indices = []
- for i, cand in enumerate(cands_list):
- if cand is not None:
+ for i, fields_payload in enumerate(cands_fields_payloads):
+ if fields_payload is not None:
valid_indices.append(i)
else:
logger.warning(
f"Candidate data is None for label index {i} (label: {label_list[i] if i < len(label_list) else 'unknown'}), skipping."
)
- if len(valid_indices) < len(cands_list):
- cands_list = [cands_list[i] for i in valid_indices]
+ if len(valid_indices) < len(cands_fields_payloads):
+ cands_fields_payloads = [cands_fields_payloads[i] for i in valid_indices]
+ if cands_vectors is not None:
+ cands_vectors = [cands_vectors[i] for i in valid_indices]
pk_list = [pk_list[i] for i in valid_indices]
scores_list = [scores_list[i] for i in valid_indices]
# Parse each candidate's fields defensively: a single corrupted JSON
# string (e.g. truncated by the storage layer's uint16 length prefix)
- # must not fail the whole query. Skip the bad ones and keep cands_list,
- # pk_list and scores_list aligned, mirroring the None-skip above.
+ # must not fail the whole query. Skip the bad ones and keep payloads,
+ # vectors, pk_list and scores_list aligned, mirroring the None-skip above.
cands_fields = []
json_valid_indices = []
- for i, cand in enumerate(cands_list):
+ for i, fields_payload in enumerate(cands_fields_payloads):
try:
- cands_fields.append(json.loads(cand.fields))
+ parsed_fields = json.loads(fields_payload)
+ if not isinstance(parsed_fields, dict):
+ raise TypeError("candidate fields JSON must be an object")
+ cands_fields.append(parsed_fields)
json_valid_indices.append(i)
except (json.JSONDecodeError, TypeError) as e:
logger.warning(
- f"Failed to parse candidate fields as JSON (label={cand.label}, "
- f"fields_len={len(cand.fields) if cand.fields else 0}), skipping. "
+ f"Failed to parse candidate fields as JSON (label={pk_list[i]}, "
+ f"fields_len={len(fields_payload) if fields_payload else 0}), skipping. "
f"Error: {e}"
)
- if len(json_valid_indices) < len(cands_list):
- cands_list = [cands_list[i] for i in json_valid_indices]
+ if len(json_valid_indices) < len(cands_fields_payloads):
+ cands_fields_payloads = [cands_fields_payloads[i] for i in json_valid_indices]
+ if cands_vectors is not None:
+ cands_vectors = [cands_vectors[i] for i in json_valid_indices]
pk_list = [pk_list[i] for i in json_valid_indices]
scores_list = [scores_list[i] for i in json_valid_indices]
@@ -581,12 +599,12 @@ class LocalCollection(ICollection):
cands_field.get(self.meta.primary_key, "") for cands_field in cands_fields
]
fields_list = [
- {field: cands_field.get(field, None) for field in output_fields}
+ {field: cands_field.get(field, None) for field in projected_fields}
for cands_field in cands_fields
]
- if self.meta.vector_key:
- for i, cands in enumerate(cands_list):
- fields_list[i][self.meta.vector_key] = cands.vector
+ if include_vector and self.meta.vector_key and cands_vectors is not None:
+ for i, vector in enumerate(cands_vectors):
+ fields_list[i][self.meta.vector_key] = vector
search_result.data = [
SearchItemResult(id=pk, fields=fields, score=score)
diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.py
index 403306e7a..6a14d5f90 100644
--- a/openviking/storage/vectordb/store/store_manager.py
+++ b/openviking/storage/vectordb/store/store_manager.py
@@ -198,6 +198,27 @@ class StoreManager:
]
return cands_list
+ def fetch_cands_fields(self, label_list: List[int]) -> List[Optional[str]]:
+ """Fetch only the serialized scalar fields for candidate labels.
+
+ This avoids materializing dense and sparse vectors when a caller only
+ needs projected scalar fields. The returned list preserves the input
+ order and uses ``None`` for missing candidates, matching
+ :meth:`fetch_cands_data`.
+ """
+ bytes_list = self.storage.read(
+ [str(label) for label in label_list],
+ StoreManager.CandsTable,
+ )
+ return [
+ (
+ CandidateData.bytes_row.deserialize_field(bytes_data, "fields")
+ if bytes_data
+ else None
+ )
+ for bytes_data in bytes_list
+ ]
+
def get_all_cands_data(self) -> List[CandidateData]:
"""Get all candidate data from the store.
diff --git a/tests/vectordb/test_local_collection_projection.py b/tests/vectordb/test_local_collection_projection.py
new file mode 100644
index 000000000..0852ba5f9
--- /dev/null
+++ b/tests/vectordb/test_local_collection_projection.py
@@ -0,0 +1,192 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.storage.vectordb.collection.local_collection import LocalCollection
+from openviking.storage.vectordb.store.data import CandidateData
+from openviking.storage.vectordb.store.store_manager import StoreManager
+
+
+class _FakeIndex:
+ def __init__(self, labels, scores):
+ self.labels = labels
+ self.scores = scores
+
+ def search(self, *_args, **_kwargs):
+ return list(self.labels), list(self.scores)
+
+
+class _FakeIndexes:
+ def __init__(self, index):
+ self.index = index
+
+ def get(self, _name):
+ return self.index
+
+
+class _FakeStoreManager:
+ def __init__(self, candidates, fields_payloads=None):
+ self.candidates = candidates
+ self.fields_payloads = fields_payloads
+ self.calls = []
+
+ def fetch_cands_data(self, labels):
+ self.calls.append(("data", list(labels)))
+ return list(self.candidates)
+
+ def fetch_cands_fields(self, labels):
+ self.calls.append(("fields", list(labels)))
+ if self.fields_payloads is not None:
+ return list(self.fields_payloads)
+ return [
+ candidate.fields if candidate is not None else None for candidate in self.candidates
+ ]
+
+
+def _candidate(label, doc_id, uri, vector):
+ return CandidateData(
+ label=label,
+ vector=vector,
+ fields=json.dumps({"doc_id": doc_id, "uri": uri}),
+ )
+
+
+def _collection(store, labels=(11, 12), scores=(0.9, 0.8)):
+ collection = object.__new__(LocalCollection)
+ collection.indexes = _FakeIndexes(_FakeIndex(labels, scores))
+ collection.store_mgr = store
+ collection.meta = SimpleNamespace(
+ primary_key="doc_id",
+ vector_key="embedding",
+ fields_dict={"doc_id": {}, "embedding": {}, "uri": {}},
+ )
+ return collection
+
+
+def test_search_by_vector_projects_fields_without_decoding_vectors():
+ store = _FakeStoreManager(
+ [
+ _candidate(11, "first", "/docs/one", [1.0, 0.0]),
+ _candidate(12, "second", "/docs/two", [0.0, 1.0]),
+ ]
+ )
+
+ result = _collection(store).search_by_vector(
+ "default", dense_vector=[1.0, 0.0], output_fields=["uri"]
+ )
+
+ assert store.calls == [("fields", [11, 12])]
+ assert [(item.id, item.fields, item.score) for item in result.data] == [
+ ("first", {"uri": "/docs/one"}, 0.9),
+ ("second", {"uri": "/docs/two"}, 0.8),
+ ]
+
+
[email protected]("output_fields", [None, [], ["uri", "embedding"]])
+def test_search_by_vector_preserves_full_and_explicit_vector_hydration(output_fields):
+ store = _FakeStoreManager(
+ [
+ _candidate(11, "first", "/docs/one", [1.0, 0.0]),
+ _candidate(12, "second", "/docs/two", [0.0, 1.0]),
+ ]
+ )
+
+ result = _collection(store).search_by_vector(
+ "default", dense_vector=[1.0, 0.0], output_fields=output_fields
+ )
+
+ assert store.calls == [("data", [11, 12])]
+ assert [item.id for item in result.data] == ["first", "second"]
+ assert [item.fields["embedding"] for item in result.data] == [
+ [1.0, 0.0],
+ [0.0, 1.0],
+ ]
+ if output_fields:
+ assert result.data[0].fields == {
+ "uri": "/docs/one",
+ "embedding": [1.0, 0.0],
+ }
+ else:
+ assert result.data[0].fields == {
+ "doc_id": "first",
+ "embedding": [1.0, 0.0],
+ "uri": "/docs/one",
+ }
+
+
+def test_search_by_vector_supports_vector_only_projection():
+ store = _FakeStoreManager(
+ [
+ _candidate(11, "first", "/docs/one", [1.0, 0.0]),
+ _candidate(12, "second", "/docs/two", [0.0, 1.0]),
+ ]
+ )
+
+ result = _collection(store).search_by_vector(
+ "default",
+ dense_vector=[1.0, 0.0],
+ output_fields=["embedding"],
+ )
+
+ assert store.calls == [("data", [11, 12])]
+ assert [(item.id, item.fields) for item in result.data] == [
+ ("first", {"embedding": [1.0, 0.0]}),
+ ("second", {"embedding": [0.0, 1.0]}),
+ ]
+
+
+def test_projected_hydration_skips_missing_and_corrupt_rows_without_misalignment():
+ store = _FakeStoreManager(
+ candidates=[],
+ fields_payloads=[
+ json.dumps({"doc_id": "first", "uri": "/docs/one"}),
+ None,
+ '{"doc_id": "broken"',
+ "null",
+ json.dumps({"doc_id": "fourth", "uri": "/docs/four"}),
+ ],
+ )
+ collection = _collection(
+ store,
+ labels=(11, 12, 13, 14, 15),
+ scores=(0.9, 0.8, 0.7, 0.65, 0.6),
+ )
+
+ result = collection.search_by_vector("default", dense_vector=[1.0, 0.0], output_fields=["uri"])
+
+ assert [(item.id, item.fields, item.score) for item in result.data] == [
+ ("first", {"uri": "/docs/one"}, 0.9),
+ ("fourth", {"uri": "/docs/four"}, 0.6),
+ ]
+
+
+def test_store_manager_projects_fields_and_preserves_missing_positions(monkeypatch):
+ payloads = {
+ "11": _candidate(11, "first", "/docs/one", [1.0, 0.0]).serialize(),
+ "13": _candidate(13, "third", "/docs/three", [0.0, 1.0]).serialize(),
+ }
+
+ class _Storage:
+ def read(self, keys, table):
+ assert table == StoreManager.CandsTable
+ return [payloads.get(key, b"") for key in keys]
+
+ manager = StoreManager(_Storage())
+
+ def reject_full_decode(_payload):
+ raise AssertionError("projected fetch must not deserialize CandidateData")
+
+ monkeypatch.setattr(CandidateData, "from_bytes", staticmethod(reject_full_decode))
+
+ fields = manager.fetch_cands_fields([11, 12, 13, 11])
+
+ assert fields == [
+ json.dumps({"doc_id": "first", "uri": "/docs/one"}),
+ None,
+ json.dumps({"doc_id": "third", "uri": "/docs/three"}),
+ json.dumps({"doc_id": "first", "uri": "/docs/one"}),
+ ]