Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vectordb): skip candidates with corrupted JSON fields in search / 检索时跳过 fields 损坏的候选 (#2302)
Kcheneb5b46bed5fc
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 5d1797dea..ff7c0146a 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -313,7 +313,27 @@ class LocalCollection(ICollection):pk_list = [pk_list[i] for i in valid_indices]scores_list = [scores_list[i] for i in valid_indices]- cands_fields = [json.loads(cand.fields) for cand in cands_list]+ # Parse each candidate's fields defensively: a single corrupted JSON+ # string (e.g. truncated by the storage layer's uint16 length prefix)+ # must not fail the whole query. Skip the bad ones and keep cands_list,+ # pk_list and scores_list aligned, mirroring the None-skip above.+ cands_fields = []+ json_valid_indices = []+ for i, cand in enumerate(cands_list):+ try:+ cands_fields.append(json.loads(cand.fields))+ json_valid_indices.append(i)+ except (json.JSONDecodeError, TypeError) as e:+ logger.warning(+ f"Failed to parse candidate fields as JSON (label={cand.label}, "+ f"fields_len={len(cand.fields) if cand.fields else 0}), skipping. "+ f"Error: {e}"+ )++ if len(json_valid_indices) < len(cands_list):+ cands_list = [cands_list[i] for i in json_valid_indices]+ pk_list = [pk_list[i] for i in json_valid_indices]+ scores_list = [scores_list[i] for i in json_valid_indices]if self.meta.primary_key:pk_list = [diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 02fff3bca..8b80f2e1f 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -356,7 +356,7 @@ class _SingleAccountBackend:order_desc=order_desc,)except Exception as e:- logger.error("Error querying collection: %s", e)+ logger.error("Error querying collection: %s", e, exc_info=True)return []async def search(diff --git a/tests/vectordb/test_openviking_vectordb.py b/tests/vectordb/test_openviking_vectordb.pyindex a4de5cc42..dae80c7fc 100644--- a/tests/vectordb/test_openviking_vectordb.py+++ b/tests/vectordb/test_openviking_vectordb.py@@ -1,8 +1,10 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+import jsonimport shutilimport unittest+from unittest.mock import patchfrom openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection@@ -175,6 +177,76 @@ class TestOpenVikingVectorDB(unittest.TestCase):])+ def test_search_skips_candidate_with_corrupted_fields(self):+ """损坏 fields 的 candidate 应被跳过,查询不抛异常,且剩余结果 id/score/fields 对齐不错位。++ 复现根因:search_by_vector 对每条 candidate 无条件 json.loads,单条坏数据+ (写入路径 uint16 截断产生的不完整 JSON)会让整个查询崩溃。修复后应跳过坏项、+ 同步过滤 pk/score,保持各列表对齐。+ """+ collection = self._create_collection()++ # 4 条数据,向量第 0 位各不相同 -> 与 query 的距离唯一 -> top-k 与 score 确定+ docs = []+ for i in range(4):+ vec = [0.0] * 1024+ vec[0] = float(i + 1)+ docs.append(+ {+ "id": f"doc_{i}",+ "uri": f"viking://resources/t/doc_{i}.md",+ "type": "file",+ "context_type": "markdown",+ "vector": vec,+ "sparse_vector": {},+ "created_at": "2026-01-01T10:00:00.000001",+ "updated_at": "2026-01-01T12:00:00.000002",+ "active_count": i,+ "parent_uri": "viking://resources/t/",+ "is_leaf": True,+ "name": f"doc_{i}.md",+ "description": f"desc {i}",+ "tags": "t",+ "abstract": f"abs {i}",+ }+ )+ self.assertEqual(len(collection.upsert_data(docs).ids), len(docs))+ self._create_index(collection)++ query = [0.0] * 1024+ query[0] = 5.0 # l2 距离: doc_3=1, doc_2=2, doc_1=3, doc_0=4(唯一)++ # 基准:未污染时的 id -> score 映射+ baseline = collection.search_by_vector("idx_filters", dense_vector=query, limit=10)+ baseline_scores = {item.id: item.score for item in baseline.data}+ self.assertEqual(len(baseline_scores), 4)++ # 在坏数据真实入口 fetch_cands_data 处,把最近的 doc_3 的 fields 改成未闭合 JSON+ inner = collection._Collection__collection+ real_fetch = inner.store_mgr.fetch_cands_data+ corrupted = {}++ def corrupting_fetch(label_list):+ cands = real_fetch(label_list)+ for c in cands:+ if c is None:+ continue+ if json.loads(c.fields).get("id") == "doc_3":+ corrupted["hit"] = True+ c.fields = '{"id": "doc_3", "uri": "viking://resources/t/doc_3'+ return cands++ with patch.object(inner.store_mgr, "fetch_cands_data", side_effect=corrupting_fetch):+ result = collection.search_by_vector("idx_filters", dense_vector=query, limit=10)++ self.assertTrue(corrupted.get("hit"), "测试未能注入损坏数据")+ ids = sorted(item.id for item in result.data)+ self.assertEqual(ids, ["doc_0", "doc_1", "doc_2"], "坏项应被跳过,其余有效项全部返回")+ for item in result.data:+ # id 与 fields 内 id 自洽、score 与基准一致 —— 任一错位都会让断言失败+ self.assertEqual(item.id, item.fields.get("id"))+ self.assertEqual(item.score, baseline_scores[item.id])+def test_filters_update_delete_recall(self):collection = self._create_collection()self._insert_data(collection)