Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vectordb): validate snapshot payload in PersistentIndex newest-version selection (#5595) (#5784)
akushonkamen7928018bf9d9
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex b3104e338..85cb03088 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -22,7 +22,7 @@ from openviking.storage.vectordb.index.cuvs_index import ()from openviking.storage.vectordb.index.index import IIndexfrom openviking.storage.vectordb.store.data import CandidateData, DeltaRecord-from openviking.storage.vectordb.utils.constants import IndexFileMarkers+from openviking.storage.vectordb.utils.constants import MANAGER_META_FILE, IndexFileMarkersfrom openviking.storage.vectordb.utils.data_processor import DataProcessorfrom openviking.storage.vectordb.utils.json_safety import safe_json_dumpsfrom openviking.storage.vectordb.utils.path_safety import (@@ -1625,8 +1625,15 @@ class PersistentIndex(LocalIndex):- It has a corresponding .write_done marker file- The version directory exists- The version number is a valid integer timestamp-- Invalid or incomplete versions (without .write_done) are ignored.+ - It contains a non-empty manager_meta.json, the metadata file the+ native loader requires when opening a snapshot++ Invalid or incomplete versions are ignored. In particular, a torn+ snapshot that carries a .write_done marker but no payload (e.g. left+ behind by a crash after the marker was created but before the dumped+ blocks reached disk) is skipped, so selection falls back to the+ previous complete snapshot instead of returning a version the loader+ would reject."""if not os.path.exists(self.version_dir):return 0@@ -1647,6 +1654,13 @@ class PersistentIndex(LocalIndex):if not marker_path.exists():continue+ # Must contain a non-empty manager metadata file. This is the+ # file the native loader fail-fasts on, so a snapshot without it+ # can never be loaded even though its .write_done marker exists.+ manager_meta_path = version_path / MANAGER_META_FILE+ if not manager_meta_path.is_file() or manager_meta_path.stat().st_size == 0:+ continue+valid_versions.append(int(name))if not valid_versions:diff --git a/openviking/storage/vectordb/utils/constants.py b/openviking/storage/vectordb/utils/constants.pyindex 583df6132..8efb9fbad 100644--- a/openviking/storage/vectordb/utils/constants.py+++ b/openviking/storage/vectordb/utils/constants.py@@ -40,6 +40,13 @@ class IndexFileMarkers(str, Enum):WRITE_DONE = ".write_done" # Index write complete marker file suffix+# Metadata file the native engine requires inside every snapshot directory+# (see kMetaFile in src/index/detail/index_manager_impl.cpp); the loader+# fail-fasts when it is missing, so it is the minimum payload evidence of a+# completed snapshot.+MANAGER_META_FILE = "manager_meta.json"++# ==================== Scheduler related constants ====================DEFAULT_TTL_CLEANUP_SECONDS = 0 # TTL expired data cleanup interval (seconds)DEFAULT_INDEX_MAINTENANCE_SECONDS = 30 # Index maintenance task interval (seconds)diff --git a/tests/vectordb/test_crash_recovery.py b/tests/vectordb/test_crash_recovery.pyindex 463821e7c..80d040178 100644--- a/tests/vectordb/test_crash_recovery.py+++ b/tests/vectordb/test_crash_recovery.py@@ -7,6 +7,7 @@ import sysimport timeimport unittestfrom contextlib import ExitStack+from pathlib import Pathfrom unittest.mock import patchfrom openviking.storage.vectordb import engine@@ -15,6 +16,7 @@ from openviking.storage.vectordb.store.data import CandidateData, DeltaRecordDB_PATH_CRASH = "./test_data/test_db_crash_recovery"DB_PATH_ROBUST = "./test_data/test_db_robust_crash"+DB_PATH_TORN = "./test_data/test_db_torn_snapshot"LARGE_TEXT = "旧数据升级后的长文本" * 4000@@ -127,6 +129,40 @@ def setup_robust_collection(path):return col+def setup_torn_snapshot_collection(path):+ """Helper to setup collection config for torn-snapshot tests"""+ meta_data = {+ "CollectionName": "torn_snapshot_col",+ "Fields": [+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": 4},+ {"FieldName": "data", "FieldType": "string"},+ ],+ }+ col = get_or_create_local_collection(meta_data=meta_data, path=path)+ if not col.has_index("idx_torn"):+ col.create_index(+ "idx_torn",+ {"IndexName": "idx_torn", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},+ )+ return col+++def list_version_numbers(version_dir):+ """Return the snapshot version numbers currently on disk, sorted."""+ return sorted(+ int(name)+ for name in os.listdir(version_dir)+ if name.isdigit() and (Path(version_dir) / name).is_dir()+ )+++def make_torn_version(version_dir, version):+ """Create a marker-complete but payload-less (torn) snapshot version."""+ os.makedirs(str(Path(version_dir, str(version))))+ Path(version_dir, f"{version}.write_done").touch()++def worker_cycle_1_write(path, event_ready):"""Cycle 1: Write 500 items (ID 0-499).@@ -361,5 +397,85 @@ class TestCrashRecovery(unittest.TestCase):print("\n[Main] Robust recovery test passed.")+class TestTornSnapshotSelection(unittest.TestCase):+ """A snapshot marked .write_done but missing its payload must never be selected.++ A crash between the native dump and the payload blocks reaching disk can+ leave behind a version directory that carries a .write_done marker but no+ loadable payload. Selecting it as the newest version makes the next open+ hard-fail on the native loader before any recovery path runs, even though+ the previous complete snapshot is still on disk and loadable.+ """++ def setUp(self):+ if os.path.exists(DB_PATH_TORN):+ shutil.rmtree(DB_PATH_TORN)++ def tearDown(self):+ if os.path.exists(DB_PATH_TORN):+ shutil.rmtree(DB_PATH_TORN)++ def test_torn_newest_snapshot_is_skipped_on_reopen(self):+ print("\n=== Test Torn Newest Snapshot Is Skipped On Reopen ===")++ col = setup_torn_snapshot_collection(DB_PATH_TORN)+ data = [{"id": i, "vector": [0.1] * 4, "data": f"torn_{i}"} for i in range(10)]+ col.upsert_data(data)+ version_dir = col.get_index("idx_torn").version_dir+ col.close()++ # After close() only the newest complete version remains on disk.+ valid_versions = list_version_numbers(version_dir)+ self.assertEqual(len(valid_versions), 1)+ valid_version = valid_versions[0]+ self.assertTrue(Path(version_dir, str(valid_version), "manager_meta.json").is_file())++ # Crash artifact: a newer, marker-complete but payload-less version.+ torn_version = valid_version + 1+ make_torn_version(version_dir, torn_version)++ # Reopening must succeed and fall back to the previous complete+ # snapshot instead of hard-failing on the empty newest one.+ reopened = setup_torn_snapshot_collection(DB_PATH_TORN)+ reopened_index = reopened.get_index("idx_torn")+ self.assertEqual(int(reopened_index.now_version), valid_version)+ self.assertEqual(int(reopened_index.get_newest_version()), valid_version)++ res = reopened.fetch_data(list(range(10)))+ self.assertEqual(len(res.items), 10)+ self.assertEqual(+ {item.id for item in res.items}, set(range(10)), "All ids should be recovered"+ )+ search_res = reopened.search_by_vector("idx_torn", dense_vector=[0.1] * 4, limit=10)+ self.assertEqual({item.id for item in search_res.data}, set(range(10)))++ reopened.close()+ print("[Main] Torn snapshot skipped, previous complete snapshot reloaded.")++ def test_get_newest_version_ignores_missing_or_empty_meta(self):+ print("\n=== Test get_newest_version Ignores Torn Snapshots ===")++ col = setup_torn_snapshot_collection(DB_PATH_TORN)+ try:+ col.upsert_data([{"id": 1, "vector": [0.1] * 4, "data": "torn_1"}])+ index = col.get_index("idx_torn")+ version_dir = index.version_dir+ index.persist()+ valid_version = index.get_newest_version()+ self.assertGreater(valid_version, 0)++ # Torn variants: marker present, payload missing or 0 bytes.+ make_torn_version(version_dir, valid_version + 1)+ empty_meta_version = valid_version + 2+ os.makedirs(str(Path(version_dir, str(empty_meta_version))))+ Path(version_dir, str(empty_meta_version), "manager_meta.json").touch()+ Path(version_dir, f"{empty_meta_version}.write_done").touch()++ self.assertEqual(index.get_newest_version(), valid_version)+ finally:+ col.close()+ print("[Main] get_newest_version rejected torn snapshots.")++if __name__ == "__main__":unittest.main()