Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(vectordb): avoid hydrating unrequested vectors (#3375)
Yuanqing ZHAOe1cea0998b3e
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 0e9c61493..3f5e5fcb2 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -533,46 +533,64 @@ class LocalCollection(ICollection):pk_list = label_listfields_list = []- if not output_fields:- output_fields = list(self.meta.fields_dict.keys())- if self.meta.primary_key or output_fields:+ projected_fields = (+ list(self.meta.fields_dict.keys()) if not output_fields else list(output_fields)+ )+ include_vector = bool(self.meta.vector_key) and self.meta.vector_key in projected_fields+ if self.meta.primary_key or projected_fields:if not self.store_mgr:raise RuntimeError("Store manager is not initialized")- cands_list = self.store_mgr.fetch_cands_data(label_list)++ cands_vectors: Optional[List[List[float]]] = None+ if include_vector:+ cands_list = self.store_mgr.fetch_cands_data(label_list)+ cands_fields_payloads = [+ cand.fields if cand is not None else None for cand in cands_list+ ]+ cands_vectors = [cand.vector if cand is not None else [] for cand in cands_list]+ else:+ cands_fields_payloads = self.store_mgr.fetch_cands_fields(label_list)valid_indices = []- for i, cand in enumerate(cands_list):- if cand is not None:+ for i, fields_payload in enumerate(cands_fields_payloads):+ if fields_payload is not None:valid_indices.append(i)else:logger.warning(f"Candidate data is None for label index {i} (label: {label_list[i] if i < len(label_list) else 'unknown'}), skipping.")- if len(valid_indices) < len(cands_list):- cands_list = [cands_list[i] for i in valid_indices]+ if len(valid_indices) < len(cands_fields_payloads):+ cands_fields_payloads = [cands_fields_payloads[i] for i in valid_indices]+ if cands_vectors is not None:+ cands_vectors = [cands_vectors[i] for i in valid_indices]pk_list = [pk_list[i] for i in valid_indices]scores_list = [scores_list[i] for i in valid_indices]# Parse each candidate's fields defensively: a single corrupted JSON# string (e.g. truncated by the storage layer's uint16 length prefix)- # must not fail the whole query. Skip the bad ones and keep cands_list,- # pk_list and scores_list aligned, mirroring the None-skip above.+ # must not fail the whole query. Skip the bad ones and keep payloads,+ # vectors, pk_list and scores_list aligned, mirroring the None-skip above.cands_fields = []json_valid_indices = []- for i, cand in enumerate(cands_list):+ for i, fields_payload in enumerate(cands_fields_payloads):try:- cands_fields.append(json.loads(cand.fields))+ parsed_fields = json.loads(fields_payload)+ if not isinstance(parsed_fields, dict):+ raise TypeError("candidate fields JSON must be an object")+ cands_fields.append(parsed_fields)json_valid_indices.append(i)except (json.JSONDecodeError, TypeError) as e:logger.warning(- f"Failed to parse candidate fields as JSON (label={cand.label}, "- f"fields_len={len(cand.fields) if cand.fields else 0}), skipping. "+ f"Failed to parse candidate fields as JSON (label={pk_list[i]}, "+ f"fields_len={len(fields_payload) if fields_payload else 0}), skipping. "f"Error: {e}")- if len(json_valid_indices) < len(cands_list):- cands_list = [cands_list[i] for i in json_valid_indices]+ if len(json_valid_indices) < len(cands_fields_payloads):+ cands_fields_payloads = [cands_fields_payloads[i] for i in json_valid_indices]+ if cands_vectors is not None:+ cands_vectors = [cands_vectors[i] for i in json_valid_indices]pk_list = [pk_list[i] for i in json_valid_indices]scores_list = [scores_list[i] for i in json_valid_indices]@@ -581,12 +599,12 @@ class LocalCollection(ICollection):cands_field.get(self.meta.primary_key, "") for cands_field in cands_fields]fields_list = [- {field: cands_field.get(field, None) for field in output_fields}+ {field: cands_field.get(field, None) for field in projected_fields}for cands_field in cands_fields]- if self.meta.vector_key:- for i, cands in enumerate(cands_list):- fields_list[i][self.meta.vector_key] = cands.vector+ if include_vector and self.meta.vector_key and cands_vectors is not None:+ for i, vector in enumerate(cands_vectors):+ fields_list[i][self.meta.vector_key] = vectorsearch_result.data = [SearchItemResult(id=pk, fields=fields, score=score)diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.pyindex 403306e7a..6a14d5f90 100644--- a/openviking/storage/vectordb/store/store_manager.py+++ b/openviking/storage/vectordb/store/store_manager.py@@ -198,6 +198,27 @@ class StoreManager:]return cands_list+ def fetch_cands_fields(self, label_list: List[int]) -> List[Optional[str]]:+ """Fetch only the serialized scalar fields for candidate labels.++ This avoids materializing dense and sparse vectors when a caller only+ needs projected scalar fields. The returned list preserves the input+ order and uses ``None`` for missing candidates, matching+ :meth:`fetch_cands_data`.+ """+ bytes_list = self.storage.read(+ [str(label) for label in label_list],+ StoreManager.CandsTable,+ )+ return [+ (+ CandidateData.bytes_row.deserialize_field(bytes_data, "fields")+ if bytes_data+ else None+ )+ for bytes_data in bytes_list+ ]+def get_all_cands_data(self) -> List[CandidateData]:"""Get all candidate data from the store.diff --git a/tests/vectordb/test_local_collection_projection.py b/tests/vectordb/test_local_collection_projection.pynew file mode 100644index 000000000..0852ba5f9--- /dev/null+++ b/tests/vectordb/test_local_collection_projection.py@@ -0,0 +1,192 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import json+from types import SimpleNamespace++import pytest++from openviking.storage.vectordb.collection.local_collection import LocalCollection+from openviking.storage.vectordb.store.data import CandidateData+from openviking.storage.vectordb.store.store_manager import StoreManager+++class _FakeIndex:+ def __init__(self, labels, scores):+ self.labels = labels+ self.scores = scores++ def search(self, *_args, **_kwargs):+ return list(self.labels), list(self.scores)+++class _FakeIndexes:+ def __init__(self, index):+ self.index = index++ def get(self, _name):+ return self.index+++class _FakeStoreManager:+ def __init__(self, candidates, fields_payloads=None):+ self.candidates = candidates+ self.fields_payloads = fields_payloads+ self.calls = []++ def fetch_cands_data(self, labels):+ self.calls.append(("data", list(labels)))+ return list(self.candidates)++ def fetch_cands_fields(self, labels):+ self.calls.append(("fields", list(labels)))+ if self.fields_payloads is not None:+ return list(self.fields_payloads)+ return [+ candidate.fields if candidate is not None else None for candidate in self.candidates+ ]+++def _candidate(label, doc_id, uri, vector):+ return CandidateData(+ label=label,+ vector=vector,+ fields=json.dumps({"doc_id": doc_id, "uri": uri}),+ )+++def _collection(store, labels=(11, 12), scores=(0.9, 0.8)):+ collection = object.__new__(LocalCollection)+ collection.indexes = _FakeIndexes(_FakeIndex(labels, scores))+ collection.store_mgr = store+ collection.meta = SimpleNamespace(+ primary_key="doc_id",+ vector_key="embedding",+ fields_dict={"doc_id": {}, "embedding": {}, "uri": {}},+ )+ return collection+++def test_search_by_vector_projects_fields_without_decoding_vectors():+ store = _FakeStoreManager(+ [+ _candidate(11, "first", "/docs/one", [1.0, 0.0]),+ _candidate(12, "second", "/docs/two", [0.0, 1.0]),+ ]+ )++ result = _collection(store).search_by_vector(+ "default", dense_vector=[1.0, 0.0], output_fields=["uri"]+ )++ assert store.calls == [("fields", [11, 12])]+ assert [(item.id, item.fields, item.score) for item in result.data] == [+ ("first", {"uri": "/docs/one"}, 0.9),+ ("second", {"uri": "/docs/two"}, 0.8),+ ]++[email protected]("output_fields", [None, [], ["uri", "embedding"]])+def test_search_by_vector_preserves_full_and_explicit_vector_hydration(output_fields):+ store = _FakeStoreManager(+ [+ _candidate(11, "first", "/docs/one", [1.0, 0.0]),+ _candidate(12, "second", "/docs/two", [0.0, 1.0]),+ ]+ )++ result = _collection(store).search_by_vector(+ "default", dense_vector=[1.0, 0.0], output_fields=output_fields+ )++ assert store.calls == [("data", [11, 12])]+ assert [item.id for item in result.data] == ["first", "second"]+ assert [item.fields["embedding"] for item in result.data] == [+ [1.0, 0.0],+ [0.0, 1.0],+ ]+ if output_fields:+ assert result.data[0].fields == {+ "uri": "/docs/one",+ "embedding": [1.0, 0.0],+ }+ else:+ assert result.data[0].fields == {+ "doc_id": "first",+ "embedding": [1.0, 0.0],+ "uri": "/docs/one",+ }+++def test_search_by_vector_supports_vector_only_projection():+ store = _FakeStoreManager(+ [+ _candidate(11, "first", "/docs/one", [1.0, 0.0]),+ _candidate(12, "second", "/docs/two", [0.0, 1.0]),+ ]+ )++ result = _collection(store).search_by_vector(+ "default",+ dense_vector=[1.0, 0.0],+ output_fields=["embedding"],+ )++ assert store.calls == [("data", [11, 12])]+ assert [(item.id, item.fields) for item in result.data] == [+ ("first", {"embedding": [1.0, 0.0]}),+ ("second", {"embedding": [0.0, 1.0]}),+ ]+++def test_projected_hydration_skips_missing_and_corrupt_rows_without_misalignment():+ store = _FakeStoreManager(+ candidates=[],+ fields_payloads=[+ json.dumps({"doc_id": "first", "uri": "/docs/one"}),+ None,+ '{"doc_id": "broken"',+ "null",+ json.dumps({"doc_id": "fourth", "uri": "/docs/four"}),+ ],+ )+ collection = _collection(+ store,+ labels=(11, 12, 13, 14, 15),+ scores=(0.9, 0.8, 0.7, 0.65, 0.6),+ )++ result = collection.search_by_vector("default", dense_vector=[1.0, 0.0], output_fields=["uri"])++ assert [(item.id, item.fields, item.score) for item in result.data] == [+ ("first", {"uri": "/docs/one"}, 0.9),+ ("fourth", {"uri": "/docs/four"}, 0.6),+ ]+++def test_store_manager_projects_fields_and_preserves_missing_positions(monkeypatch):+ payloads = {+ "11": _candidate(11, "first", "/docs/one", [1.0, 0.0]).serialize(),+ "13": _candidate(13, "third", "/docs/three", [0.0, 1.0]).serialize(),+ }++ class _Storage:+ def read(self, keys, table):+ assert table == StoreManager.CandsTable+ return [payloads.get(key, b"") for key in keys]++ manager = StoreManager(_Storage())++ def reject_full_decode(_payload):+ raise AssertionError("projected fetch must not deserialize CandidateData")++ monkeypatch.setattr(CandidateData, "from_bytes", staticmethod(reject_full_decode))++ fields = manager.fetch_cands_fields([11, 12, 13, 11])++ assert fields == [+ json.dumps({"doc_id": "first", "uri": "/docs/one"}),+ None,+ json.dumps({"doc_id": "third", "uri": "/docs/three"}),+ json.dumps({"doc_id": "first", "uri": "/docs/one"}),+ ]