Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vectordb): validate snapshot payload in PersistentIndex newest-version selection (#5595) (#5784)

akushonkamen7928018bf9d9
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.py
index b3104e338..85cb03088 100644
--- a/openviking/storage/vectordb/index/local_index.py
+++ b/openviking/storage/vectordb/index/local_index.py
@@ -22,7 +22,7 @@ from openviking.storage.vectordb.index.cuvs_index import (
)
from openviking.storage.vectordb.index.index import IIndex
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
-from openviking.storage.vectordb.utils.constants import IndexFileMarkers
+from openviking.storage.vectordb.utils.constants import MANAGER_META_FILE, IndexFileMarkers
from openviking.storage.vectordb.utils.data_processor import DataProcessor
from openviking.storage.vectordb.utils.json_safety import safe_json_dumps
from openviking.storage.vectordb.utils.path_safety import (
@@ -1625,8 +1625,15 @@ class PersistentIndex(LocalIndex):
- It has a corresponding .write_done marker file
- The version directory exists
- The version number is a valid integer timestamp
-
- Invalid or incomplete versions (without .write_done) are ignored.
+ - It contains a non-empty manager_meta.json, the metadata file the
+ native loader requires when opening a snapshot
+
+ Invalid or incomplete versions are ignored. In particular, a torn
+ snapshot that carries a .write_done marker but no payload (e.g. left
+ behind by a crash after the marker was created but before the dumped
+ blocks reached disk) is skipped, so selection falls back to the
+ previous complete snapshot instead of returning a version the loader
+ would reject.
"""
if not os.path.exists(self.version_dir):
return 0
@@ -1647,6 +1654,13 @@ class PersistentIndex(LocalIndex):
if not marker_path.exists():
continue
+ # Must contain a non-empty manager metadata file. This is the
+ # file the native loader fail-fasts on, so a snapshot without it
+ # can never be loaded even though its .write_done marker exists.
+ manager_meta_path = version_path / MANAGER_META_FILE
+ if not manager_meta_path.is_file() or manager_meta_path.stat().st_size == 0:
+ continue
+
valid_versions.append(int(name))
if not valid_versions:
diff --git a/openviking/storage/vectordb/utils/constants.py b/openviking/storage/vectordb/utils/constants.py
index 583df6132..8efb9fbad 100644
--- a/openviking/storage/vectordb/utils/constants.py
+++ b/openviking/storage/vectordb/utils/constants.py
@@ -40,6 +40,13 @@ class IndexFileMarkers(str, Enum):
WRITE_DONE = ".write_done" # Index write complete marker file suffix
+# Metadata file the native engine requires inside every snapshot directory
+# (see kMetaFile in src/index/detail/index_manager_impl.cpp); the loader
+# fail-fasts when it is missing, so it is the minimum payload evidence of a
+# completed snapshot.
+MANAGER_META_FILE = "manager_meta.json"
+
+
# ==================== Scheduler related constants ====================
DEFAULT_TTL_CLEANUP_SECONDS = 0 # TTL expired data cleanup interval (seconds)
DEFAULT_INDEX_MAINTENANCE_SECONDS = 30 # Index maintenance task interval (seconds)
diff --git a/tests/vectordb/test_crash_recovery.py b/tests/vectordb/test_crash_recovery.py
index 463821e7c..80d040178 100644
--- a/tests/vectordb/test_crash_recovery.py
+++ b/tests/vectordb/test_crash_recovery.py
@@ -7,6 +7,7 @@ import sys
import time
import unittest
from contextlib import ExitStack
+from pathlib import Path
from unittest.mock import patch
from openviking.storage.vectordb import engine
@@ -15,6 +16,7 @@ from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
DB_PATH_CRASH = "./test_data/test_db_crash_recovery"
DB_PATH_ROBUST = "./test_data/test_db_robust_crash"
+DB_PATH_TORN = "./test_data/test_db_torn_snapshot"
LARGE_TEXT = "旧数据升级后的长文本" * 4000
@@ -127,6 +129,40 @@ def setup_robust_collection(path):
return col
+def setup_torn_snapshot_collection(path):
+ """Helper to setup collection config for torn-snapshot tests"""
+ meta_data = {
+ "CollectionName": "torn_snapshot_col",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},
+ {"FieldName": "data", "FieldType": "string"},
+ ],
+ }
+ col = get_or_create_local_collection(meta_data=meta_data, path=path)
+ if not col.has_index("idx_torn"):
+ col.create_index(
+ "idx_torn",
+ {"IndexName": "idx_torn", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},
+ )
+ return col
+
+
+def list_version_numbers(version_dir):
+ """Return the snapshot version numbers currently on disk, sorted."""
+ return sorted(
+ int(name)
+ for name in os.listdir(version_dir)
+ if name.isdigit() and (Path(version_dir) / name).is_dir()
+ )
+
+
+def make_torn_version(version_dir, version):
+ """Create a marker-complete but payload-less (torn) snapshot version."""
+ os.makedirs(str(Path(version_dir, str(version))))
+ Path(version_dir, f"{version}.write_done").touch()
+
+
def worker_cycle_1_write(path, event_ready):
"""
Cycle 1: Write 500 items (ID 0-499).
@@ -361,5 +397,85 @@ class TestCrashRecovery(unittest.TestCase):
print("\n[Main] Robust recovery test passed.")
+class TestTornSnapshotSelection(unittest.TestCase):
+ """A snapshot marked .write_done but missing its payload must never be selected.
+
+ A crash between the native dump and the payload blocks reaching disk can
+ leave behind a version directory that carries a .write_done marker but no
+ loadable payload. Selecting it as the newest version makes the next open
+ hard-fail on the native loader before any recovery path runs, even though
+ the previous complete snapshot is still on disk and loadable.
+ """
+
+ def setUp(self):
+ if os.path.exists(DB_PATH_TORN):
+ shutil.rmtree(DB_PATH_TORN)
+
+ def tearDown(self):
+ if os.path.exists(DB_PATH_TORN):
+ shutil.rmtree(DB_PATH_TORN)
+
+ def test_torn_newest_snapshot_is_skipped_on_reopen(self):
+ print("\n=== Test Torn Newest Snapshot Is Skipped On Reopen ===")
+
+ col = setup_torn_snapshot_collection(DB_PATH_TORN)
+ data = [{"id": i, "vector": [0.1] * 4, "data": f"torn_{i}"} for i in range(10)]
+ col.upsert_data(data)
+ version_dir = col.get_index("idx_torn").version_dir
+ col.close()
+
+ # After close() only the newest complete version remains on disk.
+ valid_versions = list_version_numbers(version_dir)
+ self.assertEqual(len(valid_versions), 1)
+ valid_version = valid_versions[0]
+ self.assertTrue(Path(version_dir, str(valid_version), "manager_meta.json").is_file())
+
+ # Crash artifact: a newer, marker-complete but payload-less version.
+ torn_version = valid_version + 1
+ make_torn_version(version_dir, torn_version)
+
+ # Reopening must succeed and fall back to the previous complete
+ # snapshot instead of hard-failing on the empty newest one.
+ reopened = setup_torn_snapshot_collection(DB_PATH_TORN)
+ reopened_index = reopened.get_index("idx_torn")
+ self.assertEqual(int(reopened_index.now_version), valid_version)
+ self.assertEqual(int(reopened_index.get_newest_version()), valid_version)
+
+ res = reopened.fetch_data(list(range(10)))
+ self.assertEqual(len(res.items), 10)
+ self.assertEqual(
+ {item.id for item in res.items}, set(range(10)), "All ids should be recovered"
+ )
+ search_res = reopened.search_by_vector("idx_torn", dense_vector=[0.1] * 4, limit=10)
+ self.assertEqual({item.id for item in search_res.data}, set(range(10)))
+
+ reopened.close()
+ print("[Main] Torn snapshot skipped, previous complete snapshot reloaded.")
+
+ def test_get_newest_version_ignores_missing_or_empty_meta(self):
+ print("\n=== Test get_newest_version Ignores Torn Snapshots ===")
+
+ col = setup_torn_snapshot_collection(DB_PATH_TORN)
+ try:
+ col.upsert_data([{"id": 1, "vector": [0.1] * 4, "data": "torn_1"}])
+ index = col.get_index("idx_torn")
+ version_dir = index.version_dir
+ index.persist()
+ valid_version = index.get_newest_version()
+ self.assertGreater(valid_version, 0)
+
+ # Torn variants: marker present, payload missing or 0 bytes.
+ make_torn_version(version_dir, valid_version + 1)
+ empty_meta_version = valid_version + 2
+ os.makedirs(str(Path(version_dir, str(empty_meta_version))))
+ Path(version_dir, str(empty_meta_version), "manager_meta.json").touch()
+ Path(version_dir, f"{empty_meta_version}.write_done").touch()
+
+ self.assertEqual(index.get_newest_version(), valid_version)
+ finally:
+ col.close()
+ print("[Main] get_newest_version rejected torn snapshots.")
+
+
if __name__ == "__main__":
unittest.main()