Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vectordb): skip candidates with corrupted JSON fields in search / 检索时跳过 fields 损坏的候选 (#2302)

Kcheneb5b46bed5fc
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 5d1797dea..ff7c0146a 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -313,7 +313,27 @@ class LocalCollection(ICollection):
pk_list = [pk_list[i] for i in valid_indices]
scores_list = [scores_list[i] for i in valid_indices]
- cands_fields = [json.loads(cand.fields) for cand in cands_list]
+ # Parse each candidate's fields defensively: a single corrupted JSON
+ # string (e.g. truncated by the storage layer's uint16 length prefix)
+ # must not fail the whole query. Skip the bad ones and keep cands_list,
+ # pk_list and scores_list aligned, mirroring the None-skip above.
+ cands_fields = []
+ json_valid_indices = []
+ for i, cand in enumerate(cands_list):
+ try:
+ cands_fields.append(json.loads(cand.fields))
+ json_valid_indices.append(i)
+ except (json.JSONDecodeError, TypeError) as e:
+ logger.warning(
+ f"Failed to parse candidate fields as JSON (label={cand.label}, "
+ f"fields_len={len(cand.fields) if cand.fields else 0}), skipping. "
+ f"Error: {e}"
+ )
+
+ if len(json_valid_indices) < len(cands_list):
+ cands_list = [cands_list[i] for i in json_valid_indices]
+ pk_list = [pk_list[i] for i in json_valid_indices]
+ scores_list = [scores_list[i] for i in json_valid_indices]
if self.meta.primary_key:
pk_list = [
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 02fff3bca..8b80f2e1f 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -356,7 +356,7 @@ class _SingleAccountBackend:
order_desc=order_desc,
)
except Exception as e:
- logger.error("Error querying collection: %s", e)
+ logger.error("Error querying collection: %s", e, exc_info=True)
return []
async def search(
diff --git a/tests/vectordb/test_openviking_vectordb.py b/tests/vectordb/test_openviking_vectordb.py
index a4de5cc42..dae80c7fc 100644
--- a/tests/vectordb/test_openviking_vectordb.py
+++ b/tests/vectordb/test_openviking_vectordb.py
@@ -1,8 +1,10 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+import json
import shutil
import unittest
+from unittest.mock import patch
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
@@ -175,6 +177,76 @@ class TestOpenVikingVectorDB(unittest.TestCase):
]
)
+ def test_search_skips_candidate_with_corrupted_fields(self):
+ """损坏 fields 的 candidate 应被跳过,查询不抛异常,且剩余结果 id/score/fields 对齐不错位。
+
+ 复现根因:search_by_vector 对每条 candidate 无条件 json.loads,单条坏数据
+ (写入路径 uint16 截断产生的不完整 JSON)会让整个查询崩溃。修复后应跳过坏项、
+ 同步过滤 pk/score,保持各列表对齐。
+ """
+ collection = self._create_collection()
+
+ # 4 条数据,向量第 0 位各不相同 -> 与 query 的距离唯一 -> top-k 与 score 确定
+ docs = []
+ for i in range(4):
+ vec = [0.0] * 1024
+ vec[0] = float(i + 1)
+ docs.append(
+ {
+ "id": f"doc_{i}",
+ "uri": f"viking://resources/t/doc_{i}.md",
+ "type": "file",
+ "context_type": "markdown",
+ "vector": vec,
+ "sparse_vector": {},
+ "created_at": "2026-01-01T10:00:00.000001",
+ "updated_at": "2026-01-01T12:00:00.000002",
+ "active_count": i,
+ "parent_uri": "viking://resources/t/",
+ "is_leaf": True,
+ "name": f"doc_{i}.md",
+ "description": f"desc {i}",
+ "tags": "t",
+ "abstract": f"abs {i}",
+ }
+ )
+ self.assertEqual(len(collection.upsert_data(docs).ids), len(docs))
+ self._create_index(collection)
+
+ query = [0.0] * 1024
+ query[0] = 5.0 # l2 距离: doc_3=1, doc_2=2, doc_1=3, doc_0=4(唯一)
+
+ # 基准:未污染时的 id -> score 映射
+ baseline = collection.search_by_vector("idx_filters", dense_vector=query, limit=10)
+ baseline_scores = {item.id: item.score for item in baseline.data}
+ self.assertEqual(len(baseline_scores), 4)
+
+ # 在坏数据真实入口 fetch_cands_data 处,把最近的 doc_3 的 fields 改成未闭合 JSON
+ inner = collection._Collection__collection
+ real_fetch = inner.store_mgr.fetch_cands_data
+ corrupted = {}
+
+ def corrupting_fetch(label_list):
+ cands = real_fetch(label_list)
+ for c in cands:
+ if c is None:
+ continue
+ if json.loads(c.fields).get("id") == "doc_3":
+ corrupted["hit"] = True
+ c.fields = '{"id": "doc_3", "uri": "viking://resources/t/doc_3'
+ return cands
+
+ with patch.object(inner.store_mgr, "fetch_cands_data", side_effect=corrupting_fetch):
+ result = collection.search_by_vector("idx_filters", dense_vector=query, limit=10)
+
+ self.assertTrue(corrupted.get("hit"), "测试未能注入损坏数据")
+ ids = sorted(item.id for item in result.data)
+ self.assertEqual(ids, ["doc_0", "doc_1", "doc_2"], "坏项应被跳过,其余有效项全部返回")
+ for item in result.data:
+ # id 与 fields 内 id 自洽、score 与基准一致 —— 任一错位都会让断言失败
+ self.assertEqual(item.id, item.fields.get("id"))
+ self.assertEqual(item.score, baseline_scores[item.id])
+
def test_filters_update_delete_recall(self):
collection = self._create_collection()
self._insert_data(collection)