π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: treat a log shorter than its snapshot frontier as corruption
laurenspriem8e481298ad77
diff --git a/rust/crates/ml/src/vecdb/snapshot.rs b/rust/crates/ml/src/vecdb/snapshot.rsindex b5602c7f1a..9cfcde69c2 100644--- a/rust/crates/ml/src/vecdb/snapshot.rs+++ b/rust/crates/ml/src/vecdb/snapshot.rs@@ -42,7 +42,6 @@ pub(crate) fn write_snapshot(pub(crate) fn load_snapshot(log_path: &Path,expected_generation: [u8; 16],- log_end_offset: u64,) -> Option<LoadedSnapshot> {let path = snapshot_path(log_path);let bytes = match std::fs::read(&path) {@@ -53,7 +52,7 @@ pub(crate) fn load_snapshot(return None;}};- match decode_snapshot(&bytes, &expected_generation, log_end_offset) {+ match decode_snapshot(&bytes, &expected_generation) {Ok(loaded) => Some(loaded),Err(reason) => {log::debug!("discarding snapshot {}: {reason}", path.display());@@ -98,11 +97,7 @@ fn encode_snapshot(generation: &[u8; 16], covered_log_offset: u64, graph: &Graphbytes}-fn decode_snapshot(- bytes: &[u8],- expected_generation: &[u8; 16],- log_end_offset: u64,-) -> Result<LoadedSnapshot, String> {+fn decode_snapshot(bytes: &[u8], expected_generation: &[u8; 16]) -> Result<LoadedSnapshot, String> {if bytes.len() < MIN_SNAPSHOT_LEN {return Err(format!("{} bytes is shorter than the {MIN_SNAPSHOT_LEN}-byte minimum",@@ -141,11 +136,6 @@ fn decode_snapshot(return Err("generation does not match the log".to_string());}let covered_log_offset = reader.read_u64().ok_or("truncated header")?;- if covered_log_offset > log_end_offset {- return Err(format!(- "covered offset {covered_log_offset} is past the log end {log_end_offset}"- ));- }let entry_marker = reader.read_u32().ok_or("truncated header")?;let top_level = reader.read_u8().ok_or("truncated header")?;let node_count = reader.read_u32().ok_or("truncated header")? as usize;@@ -311,7 +301,7 @@ mod tests {}fn load_golden(log_path: &Path) -> Option<LoadedSnapshot> {- load_snapshot(log_path, golden_generation(), 300)+ load_snapshot(log_path, golden_generation())}fn refresh_crc(bytes: &mut [u8]) {@@ -450,7 +440,7 @@ mod tests {let generation = [7u8; 16];write_snapshot(&log_path, generation, 4096, &graph).unwrap();assert!(!temp_snapshot_path(&log_path).exists());- let loaded = load_snapshot(&log_path, generation, 4096).unwrap();+ let loaded = load_snapshot(&log_path, generation).unwrap();assert_eq!(loaded.covered_log_offset, 4096);assert_eq!(loaded.entry_point, graph.entry_point());let rebuilt =@@ -482,7 +472,7 @@ mod tests {std::fs::read(snapshot_path(&log_path)).unwrap().len(),MIN_SNAPSHOT_LEN);- let loaded = load_snapshot(&log_path, [3u8; 16], 32).unwrap();+ let loaded = load_snapshot(&log_path, [3u8; 16]).unwrap();assert_eq!(loaded.covered_log_offset, 32);assert_eq!(loaded.entry_point, None);assert!(loaded.parts.is_empty());@@ -538,7 +528,7 @@ mod tests {let mut decoded = 0usize;let mut searched = 0usize;let mut exercise = |bytes: &[u8]| {- let Ok(loaded) = decode_snapshot(bytes, &generation, 300) else {+ let Ok(loaded) = decode_snapshot(bytes, &generation) else {return;};decoded += 1;@@ -591,7 +581,7 @@ mod tests {bytes.extend_from_slice(&0u16.to_le_bytes());let crc = crc32(&bytes[MAGIC.len()..]);bytes.extend_from_slice(&crc.to_le_bytes());- let loaded = decode_snapshot(&bytes, &golden_generation(), 300).unwrap();+ let loaded = decode_snapshot(&bytes, &golden_generation()).unwrap();assert_eq!(loaded.entry_point, Some(slot));assert!(Graph::from_parts(loaded.entry_point, loaded.parts, 3).is_err());}@@ -614,18 +604,17 @@ mod tests {let dir = TempDir::new().unwrap();let log_path = dir.path().join("log");std::fs::write(snapshot_path(&log_path), GOLDEN).unwrap();- assert!(load_snapshot(&log_path, [9u8; 16], 300).is_none());+ assert!(load_snapshot(&log_path, [9u8; 16]).is_none());assert!(load_golden(&log_path).is_some());}#[test]- fn covered_offset_beyond_log_end_is_discarded() {+ fn loading_exposes_the_covered_offset_for_the_caller_to_judge() {let dir = TempDir::new().unwrap();let log_path = dir.path().join("log");std::fs::write(snapshot_path(&log_path), GOLDEN).unwrap();- assert!(load_snapshot(&log_path, golden_generation(), 299).is_none());- assert!(load_snapshot(&log_path, golden_generation(), 300).is_some());- assert!(load_snapshot(&log_path, golden_generation(), 301).is_some());+ let loaded = load_snapshot(&log_path, golden_generation()).unwrap();+ assert_eq!(loaded.covered_log_offset, 300);}#[test]@@ -715,7 +704,7 @@ mod tests {write_snapshot(&log_path, generation, 100, &golden_graph()).unwrap();write_snapshot(&log_path, generation, 999, &golden_graph()).unwrap();assert!(!temp_snapshot_path(&log_path).exists());- let loaded = load_snapshot(&log_path, generation, 1000).unwrap();+ let loaded = load_snapshot(&log_path, generation).unwrap();assert_eq!(loaded.covered_log_offset, 999);assert_eq!(loaded.parts.len(), 3);}diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rsindex 0028116fad..1f911204fb 100644--- a/rust/crates/ml/src/vecdb/store.rs+++ b/rust/crates/ml/src/vecdb/store.rs@@ -1069,7 +1069,17 @@ struct ReplayedState {}fn replay(log: &mut Log, dims: usize) -> Result<ReplayedState, VecDbError> {- let snapshot = load_snapshot(log.path(), log.generation(), log.current_end_offset());+ let snapshot = load_snapshot(log.path(), log.generation());+ if let Some(loaded) = &snapshot+ && loaded.covered_log_offset > log.current_end_offset()+ {+ return Err(VecDbError::Corrupt(format!(+ "{}: snapshot covers {} bytes but the log is only {} bytes; preserving the file",+ log.path().display(),+ loaded.covered_log_offset,+ log.current_end_offset()+ )));+ }let covered = snapshot.as_ref().map_or(u64::MAX, |loaded| loaded.covered_log_offset);@@ -1621,6 +1631,25 @@ mod tests {);}+ #[test]+ fn externally_shortened_log_below_the_snapshot_frontier_is_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ db.add("first", &seeded_unit_vector(1, DIMS)).unwrap();+ let boundary = db.stats().unwrap().log_bytes;+ db.add("second", &seeded_unit_vector(2, DIMS)).unwrap();+ db.flush().unwrap();+ drop(db);+ let clean = fs::read(&path).unwrap();+ fs::write(&path, &clean[..boundary as usize]).unwrap();+ assert_corrupt_open_preserves(&path);+ fs::write(&path, &clean).unwrap();+ let recovered = open_writer(&path);+ assert_eq!(recovered.len(), 2);+ assert!(recovered.contains("second"));+ }+#[test]fn mid_log_corruption_below_the_snapshot_frontier_is_preserved() {let dir = TempDir::new().unwrap();@@ -1795,23 +1824,6 @@ mod tests {assert_own_nearest(&reopened, "fresh", &seeded_unit_vector(1, DIMS));}- #[test]- fn snapshot_claiming_offset_beyond_log_end_is_discarded() {- let dir = TempDir::new().unwrap();- let path = dir.path().join("db");- let db = open_writer(&path);- bulk_add(&db, &bulk_entries(0, 5, 80)).unwrap();- db.flush().unwrap();- drop(db);- let bytes = fs::read(&path).unwrap();- fs::write(&path, &bytes[..bytes.len() - ADD_RECORD_LEN as usize]).unwrap();- let reopened = open_writer(&path);- assert_eq!(reopened.len(), 4);- assert!(reopened.contains("key-3"));- assert!(!reopened.contains("key-4"));- assert_own_nearest(&reopened, "key-0", &seeded_unit_vector(80, DIMS));- }-#[test]fn corrupted_snapshot_bytes_are_discarded() {let dir = TempDir::new().unwrap();@@ -2336,7 +2348,7 @@ mod tests {drop(open_writer(&path));assert!(snapshot_exists(&path));let log_len = fs::metadata(&path).unwrap().len();- let loaded = load_snapshot(&path, generation_of(&path), log_len).unwrap();+ let loaded = load_snapshot(&path, generation_of(&path)).unwrap();assert_eq!(loaded.covered_log_offset, log_len);let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();drop(open_writer(&path));@@ -2368,7 +2380,7 @@ mod tests {assert_eq!(stats.records_since_snapshot, 0);assert!(stats.log_bytes < log_bytes_before);assert_ne!(generation_of(&path), generation_before);- let loaded = load_snapshot(&path, generation_of(&path), stats.log_bytes).unwrap();+ let loaded = load_snapshot(&path, generation_of(&path)).unwrap();assert_eq!(loaded.covered_log_offset, stats.log_bytes);let live_vector = |index: u64| {if index < 40 {@@ -2453,7 +2465,7 @@ mod tests {assert_own_nearest(&reopened, "key-100", &seeded_unit_vector(310, DIMS));let stats = reopened.stats().unwrap();assert_eq!(stats.dead_count, 0);- let loaded = load_snapshot(&path, generation_of(&path), stats.log_bytes).unwrap();+ let loaded = load_snapshot(&path, generation_of(&path)).unwrap();assert_eq!(loaded.covered_log_offset, stats.log_bytes);assert_ne!(fs::read(snapshot_path(&path)).unwrap(), stale_snapshot);}