Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(storage): sanitize vectordb unicode recovery (#2103)

Qin Haojie813db21c5a00
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index e6cb3802b..5d1797dea 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -41,6 +41,7 @@ from openviking.storage.vectordb.utils.constants import (
from openviking.storage.vectordb.utils.data_processor import DataProcessor
from openviking.storage.vectordb.utils.dict_utils import ThreadSafeDictManager
from openviking.storage.vectordb.utils.id_generator import generate_auto_id
+from openviking.storage.vectordb.utils.json_safety import safe_json_dumps
from openviking.storage.vectordb.utils.path_safety import (
resolve_storage_path,
safe_join,
@@ -568,7 +569,7 @@ class LocalCollection(ICollection):
if sparse_dict and isinstance(sparse_dict, dict):
cands_list[i].sparse_raw_terms = list(sparse_dict.keys())
cands_list[i].sparse_values = list(sparse_dict.values())
- cands_list[i].fields = json.dumps(data)
+ cands_list[i].fields = safe_json_dumps(data, ensure_ascii=False)
cands_list[i].expire_ns_ts = time.time_ns() + ttl * 1000000000 if ttl > 0 else 0
if not self.store_mgr:
@@ -1004,14 +1005,22 @@ class PersistCollection(LocalCollection):
for data in delta_list:
if data.type == OpType.PUT.value:
if delete_list:
- index.delete_data(delete_list)
- _processed += len(delete_list)
+ _processed += self._replay_recovery_records(
+ index_name=index_name,
+ index=index,
+ records=delete_list,
+ operation="delete",
+ )
delete_list = []
upsert_list.append(data)
elif data.type == OpType.DEL.value:
if upsert_list:
- index.upsert_data(upsert_list)
- _processed += len(upsert_list)
+ _processed += self._replay_recovery_records(
+ index_name=index_name,
+ index=index,
+ records=upsert_list,
+ operation="upsert",
+ )
upsert_list = []
delete_list.append(data)
now = time.time()
@@ -1024,14 +1033,63 @@ class PersistCollection(LocalCollection):
)
_last_log = now
if upsert_list:
- index.upsert_data(upsert_list)
- _processed += len(upsert_list)
+ _processed += self._replay_recovery_records(
+ index_name=index_name,
+ index=index,
+ records=upsert_list,
+ operation="upsert",
+ )
if delete_list:
- index.delete_data(delete_list)
- _processed += len(delete_list)
+ _processed += self._replay_recovery_records(
+ index_name=index_name,
+ index=index,
+ records=delete_list,
+ operation="delete",
+ )
logger.info("Index '%s': replay complete (%d records)", index_name, _processed)
self.indexes.set(index_name, index)
+ def _replay_recovery_records(
+ self,
+ *,
+ index_name: str,
+ index: PersistentIndex,
+ records: List[DeltaRecord],
+ operation: str,
+ ) -> int:
+ if not records:
+ return 0
+
+ replay = index.upsert_data if operation == "upsert" else index.delete_data
+ try:
+ replay(records)
+ return len(records)
+ except Exception as exc:
+ logger.warning(
+ "Index '%s': failed to replay %d %s delta records as a batch: %s; "
+ "retrying individually",
+ index_name,
+ len(records),
+ operation,
+ exc,
+ )
+
+ processed = 0
+ for record in records:
+ try:
+ replay([record])
+ processed += 1
+ except Exception as exc:
+ logger.warning(
+ "Index '%s': skipping corrupt %s delta record label=%s type=%s: %s",
+ index_name,
+ operation,
+ getattr(record, "label", None),
+ getattr(record, "type", None),
+ exc,
+ )
+ return processed
+
def _persist_all_indexes(self):
"""Persist all indexes.
diff --git a/openviking/storage/vectordb/utils/data_processor.py b/openviking/storage/vectordb/utils/data_processor.py
index 7a75b1b1e..109eb0d40 100644
--- a/openviking/storage/vectordb/utils/data_processor.py
+++ b/openviking/storage/vectordb/utils/data_processor.py
@@ -17,6 +17,10 @@ from pydantic import (
)
from openviking.storage.vectordb.utils.id_generator import generate_auto_id
+from openviking.storage.vectordb.utils.json_safety import (
+ safe_json_dumps,
+ sanitize_unicode_for_json,
+)
def get_pydantic_type(field_type: str) -> Type:
@@ -158,7 +162,7 @@ class DataProcessor:
# Pydantic Validation (Type check, Defaults, Unknown fields, Custom format checks)
# model_validate will raise ValidationError on failure
validated_obj = self._validator_model.model_validate(data)
- processed_data = validated_obj.model_dump()
+ processed_data = sanitize_unicode_for_json(validated_obj.model_dump())
return processed_data
@@ -334,9 +338,9 @@ class DataProcessor:
def convert_fields_for_index(self, fields_json: str) -> str:
if not fields_json:
return fields_json
- data = json.loads(fields_json)
+ data = sanitize_unicode_for_json(json.loads(fields_json))
converted = self.convert_fields_dict_for_index(data)
- return json.dumps(converted, ensure_ascii=False)
+ return safe_json_dumps(converted, ensure_ascii=False)
def _convert_time_range_node(self, node: Dict[str, Any], field_type: str) -> Dict[str, Any]:
if field_type != "date_time":
diff --git a/openviking/storage/vectordb/utils/json_safety.py b/openviking/storage/vectordb/utils/json_safety.py
new file mode 100644
index 000000000..d8583e3ce
--- /dev/null
+++ b/openviking/storage/vectordb/utils/json_safety.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""JSON helpers that keep vectordb fields UTF-8 safe."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def sanitize_unicode_for_json(value: Any) -> Any:
+ """Replace isolated UTF-16 surrogate code points before JSON storage.
+
+ Python's JSON decoder accepts lone surrogates such as ``"\\ud800"`` and
+ represents them as surrogate code points in ``str``. Those strings cannot be
+ encoded as valid UTF-8 and can break lower-level index recovery. The UTF-16
+ round trip keeps valid non-BMP characters intact while replacing isolated
+ surrogates with U+FFFD.
+ """
+ if isinstance(value, str):
+ return value.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
+ if isinstance(value, dict):
+ return {
+ sanitize_unicode_for_json(key): sanitize_unicode_for_json(item)
+ for key, item in value.items()
+ }
+ if isinstance(value, list):
+ return [sanitize_unicode_for_json(item) for item in value]
+ if isinstance(value, tuple):
+ return tuple(sanitize_unicode_for_json(item) for item in value)
+ return value
+
+
+def safe_json_dumps(value: Any, **kwargs: Any) -> str:
+ """Dump JSON after removing UTF-8-invalid surrogate code points."""
+ return json.dumps(sanitize_unicode_for_json(value), **kwargs)
diff --git a/tests/vectordb/test_data_processor.py b/tests/vectordb/test_data_processor.py
index a0fb2367f..a95130c05 100644
--- a/tests/vectordb/test_data_processor.py
+++ b/tests/vectordb/test_data_processor.py
@@ -1,6 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+import json
import shutil
import unittest
from datetime import datetime, timezone
@@ -23,6 +24,7 @@ class TestDataProcessor(unittest.TestCase):
"geo": {"FieldType": "geo_point"},
"uri": {"FieldType": "path"},
"tags": {"FieldType": "list<string>"},
+ "abstract": {"FieldType": "string"},
}
self.processor = DataProcessor(self.fields_dict)
@@ -106,6 +108,32 @@ class TestDataProcessor(unittest.TestCase):
with self.assertRaises(ValidationError):
self.processor.validate_and_process(data_invalid_dt)
+ def test_validate_and_process_sanitizes_lone_surrogates(self):
+ processed = self.processor.validate_and_process(
+ {"abstract": "bad \ud800", "tags": ["ok \udc00"], "uri": "/tmp/test"}
+ )
+
+ self.assertEqual(processed["abstract"], "bad \ufffd")
+ self.assertEqual(processed["tags"], ["ok \ufffd"])
+ processed["abstract"].encode("utf-8")
+
+ def test_convert_fields_for_index_sanitizes_lone_surrogates(self):
+ fields_json = '{"abstract": "bad \\ud800", "tags": ["ok \\udc00"], "uri": "/tmp/test"}'
+
+ converted_json = self.processor.convert_fields_for_index(fields_json)
+ converted_json.encode("utf-8")
+ converted = json.loads(converted_json)
+
+ self.assertEqual(converted["abstract"], "bad \ufffd")
+ self.assertEqual(converted["tags"], ["ok \ufffd"])
+
+ def test_convert_fields_for_index_preserves_valid_emoji(self):
+ fields_json = json.dumps({"abstract": "face \U0001f600", "uri": "/tmp/test"})
+
+ converted = json.loads(self.processor.convert_fields_for_index(fields_json))
+
+ self.assertEqual(converted["abstract"], "face \U0001f600")
+
if __name__ == "__main__":
unittest.main()