Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(storage): sanitize vectordb unicode recovery (#2103)
Qin Haojie813db21c5a00
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex e6cb3802b..5d1797dea 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -41,6 +41,7 @@ from openviking.storage.vectordb.utils.constants import (from openviking.storage.vectordb.utils.data_processor import DataProcessorfrom openviking.storage.vectordb.utils.dict_utils import ThreadSafeDictManagerfrom openviking.storage.vectordb.utils.id_generator import generate_auto_id+from openviking.storage.vectordb.utils.json_safety import safe_json_dumpsfrom openviking.storage.vectordb.utils.path_safety import (resolve_storage_path,safe_join,@@ -568,7 +569,7 @@ class LocalCollection(ICollection):if sparse_dict and isinstance(sparse_dict, dict):cands_list[i].sparse_raw_terms = list(sparse_dict.keys())cands_list[i].sparse_values = list(sparse_dict.values())- cands_list[i].fields = json.dumps(data)+ cands_list[i].fields = safe_json_dumps(data, ensure_ascii=False)cands_list[i].expire_ns_ts = time.time_ns() + ttl * 1000000000 if ttl > 0 else 0if not self.store_mgr:@@ -1004,14 +1005,22 @@ class PersistCollection(LocalCollection):for data in delta_list:if data.type == OpType.PUT.value:if delete_list:- index.delete_data(delete_list)- _processed += len(delete_list)+ _processed += self._replay_recovery_records(+ index_name=index_name,+ index=index,+ records=delete_list,+ operation="delete",+ )delete_list = []upsert_list.append(data)elif data.type == OpType.DEL.value:if upsert_list:- index.upsert_data(upsert_list)- _processed += len(upsert_list)+ _processed += self._replay_recovery_records(+ index_name=index_name,+ index=index,+ records=upsert_list,+ operation="upsert",+ )upsert_list = []delete_list.append(data)now = time.time()@@ -1024,14 +1033,63 @@ class PersistCollection(LocalCollection):)_last_log = nowif upsert_list:- index.upsert_data(upsert_list)- _processed += len(upsert_list)+ _processed += self._replay_recovery_records(+ index_name=index_name,+ index=index,+ records=upsert_list,+ operation="upsert",+ )if delete_list:- index.delete_data(delete_list)- _processed += len(delete_list)+ _processed += self._replay_recovery_records(+ index_name=index_name,+ index=index,+ records=delete_list,+ operation="delete",+ )logger.info("Index '%s': replay complete (%d records)", index_name, _processed)self.indexes.set(index_name, index)+ def _replay_recovery_records(+ self,+ *,+ index_name: str,+ index: PersistentIndex,+ records: List[DeltaRecord],+ operation: str,+ ) -> int:+ if not records:+ return 0++ replay = index.upsert_data if operation == "upsert" else index.delete_data+ try:+ replay(records)+ return len(records)+ except Exception as exc:+ logger.warning(+ "Index '%s': failed to replay %d %s delta records as a batch: %s; "+ "retrying individually",+ index_name,+ len(records),+ operation,+ exc,+ )++ processed = 0+ for record in records:+ try:+ replay([record])+ processed += 1+ except Exception as exc:+ logger.warning(+ "Index '%s': skipping corrupt %s delta record label=%s type=%s: %s",+ index_name,+ operation,+ getattr(record, "label", None),+ getattr(record, "type", None),+ exc,+ )+ return processed+def _persist_all_indexes(self):"""Persist all indexes.diff --git a/openviking/storage/vectordb/utils/data_processor.py b/openviking/storage/vectordb/utils/data_processor.pyindex 7a75b1b1e..109eb0d40 100644--- a/openviking/storage/vectordb/utils/data_processor.py+++ b/openviking/storage/vectordb/utils/data_processor.py@@ -17,6 +17,10 @@ from pydantic import ()from openviking.storage.vectordb.utils.id_generator import generate_auto_id+from openviking.storage.vectordb.utils.json_safety import (+ safe_json_dumps,+ sanitize_unicode_for_json,+)def get_pydantic_type(field_type: str) -> Type:@@ -158,7 +162,7 @@ class DataProcessor:# Pydantic Validation (Type check, Defaults, Unknown fields, Custom format checks)# model_validate will raise ValidationError on failurevalidated_obj = self._validator_model.model_validate(data)- processed_data = validated_obj.model_dump()+ processed_data = sanitize_unicode_for_json(validated_obj.model_dump())return processed_data@@ -334,9 +338,9 @@ class DataProcessor:def convert_fields_for_index(self, fields_json: str) -> str:if not fields_json:return fields_json- data = json.loads(fields_json)+ data = sanitize_unicode_for_json(json.loads(fields_json))converted = self.convert_fields_dict_for_index(data)- return json.dumps(converted, ensure_ascii=False)+ return safe_json_dumps(converted, ensure_ascii=False)def _convert_time_range_node(self, node: Dict[str, Any], field_type: str) -> Dict[str, Any]:if field_type != "date_time":diff --git a/openviking/storage/vectordb/utils/json_safety.py b/openviking/storage/vectordb/utils/json_safety.pynew file mode 100644index 000000000..d8583e3ce--- /dev/null+++ b/openviking/storage/vectordb/utils/json_safety.py@@ -0,0 +1,36 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""JSON helpers that keep vectordb fields UTF-8 safe."""++from __future__ import annotations++import json+from typing import Any+++def sanitize_unicode_for_json(value: Any) -> Any:+ """Replace isolated UTF-16 surrogate code points before JSON storage.++ Python's JSON decoder accepts lone surrogates such as ``"\\ud800"`` and+ represents them as surrogate code points in ``str``. Those strings cannot be+ encoded as valid UTF-8 and can break lower-level index recovery. The UTF-16+ round trip keeps valid non-BMP characters intact while replacing isolated+ surrogates with U+FFFD.+ """+ if isinstance(value, str):+ return value.encode("utf-16", "surrogatepass").decode("utf-16", "replace")+ if isinstance(value, dict):+ return {+ sanitize_unicode_for_json(key): sanitize_unicode_for_json(item)+ for key, item in value.items()+ }+ if isinstance(value, list):+ return [sanitize_unicode_for_json(item) for item in value]+ if isinstance(value, tuple):+ return tuple(sanitize_unicode_for_json(item) for item in value)+ return value+++def safe_json_dumps(value: Any, **kwargs: Any) -> str:+ """Dump JSON after removing UTF-8-invalid surrogate code points."""+ return json.dumps(sanitize_unicode_for_json(value), **kwargs)diff --git a/tests/vectordb/test_data_processor.py b/tests/vectordb/test_data_processor.pyindex a0fb2367f..a95130c05 100644--- a/tests/vectordb/test_data_processor.py+++ b/tests/vectordb/test_data_processor.py@@ -1,6 +1,7 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+import jsonimport shutilimport unittestfrom datetime import datetime, timezone@@ -23,6 +24,7 @@ class TestDataProcessor(unittest.TestCase):"geo": {"FieldType": "geo_point"},"uri": {"FieldType": "path"},"tags": {"FieldType": "list<string>"},+ "abstract": {"FieldType": "string"},}self.processor = DataProcessor(self.fields_dict)@@ -106,6 +108,32 @@ class TestDataProcessor(unittest.TestCase):with self.assertRaises(ValidationError):self.processor.validate_and_process(data_invalid_dt)+ def test_validate_and_process_sanitizes_lone_surrogates(self):+ processed = self.processor.validate_and_process(+ {"abstract": "bad \ud800", "tags": ["ok \udc00"], "uri": "/tmp/test"}+ )++ self.assertEqual(processed["abstract"], "bad \ufffd")+ self.assertEqual(processed["tags"], ["ok \ufffd"])+ processed["abstract"].encode("utf-8")++ def test_convert_fields_for_index_sanitizes_lone_surrogates(self):+ fields_json = '{"abstract": "bad \\ud800", "tags": ["ok \\udc00"], "uri": "/tmp/test"}'++ converted_json = self.processor.convert_fields_for_index(fields_json)+ converted_json.encode("utf-8")+ converted = json.loads(converted_json)++ self.assertEqual(converted["abstract"], "bad \ufffd")+ self.assertEqual(converted["tags"], ["ok \ufffd"])++ def test_convert_fields_for_index_preserves_valid_emoji(self):+ fields_json = json.dumps({"abstract": "face \U0001f600", "uri": "/tmp/test"})++ converted = json.loads(self.processor.convert_fields_for_index(fields_json))++ self.assertEqual(converted["abstract"], "face \U0001f600")+if __name__ == "__main__":unittest.main()