π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: preserve the log when corruption is detected mid-file
laurenspriem302b844d8be7
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rsindex e50d991519..4595b973e3 100644--- a/rust/crates/ml/src/vecdb/log.rs+++ b/rust/crates/ml/src/vecdb/log.rs@@ -228,6 +228,21 @@ impl Log {Ok(())}+ pub(crate) fn tail_contains_valid_record(&mut self, after: u64) -> Result<bool, VecDbError> {+ let start = after + 1;+ if start >= self.end_offset {+ return Ok(false);+ }+ self.file+ .seek(SeekFrom::Start(start))+ .map_err(|source| VecDbError::io(&self.path, source))?;+ let mut region = vec![0u8; (self.end_offset - start) as usize];+ self.file+ .read_exact(&mut region)+ .map_err(|source| VecDbError::io(&self.path, source))?;+ Ok((0..region.len()).any(|offset| valid_record_at(®ion[offset..], self.dims)))+ }+pub(crate) fn current_end_offset(&self) -> u64 {self.end_offset}@@ -459,6 +474,76 @@ fn decode_attrs(bytes: &[u8]) -> Option<Vec<Attribute>> {rest.is_empty().then_some(attrs)}+fn valid_record_at(bytes: &[u8], dims: usize) -> bool {+ let Some(record_len) = plausible_record_len(bytes, dims) else {+ return false;+ };+ let crc_start = record_len - RECORD_CRC_LEN;+ let stored = u32::from_le_bytes([+ bytes[crc_start],+ bytes[crc_start + 1],+ bytes[crc_start + 2],+ bytes[crc_start + 3],+ ]);+ crc32(&bytes[..crc_start]) == stored+}++fn plausible_record_len(bytes: &[u8], dims: usize) -> Option<usize> {+ if bytes.len() < RECORD_PREFIX_LEN + RECORD_CRC_LEN {+ return None;+ }+ let key_len = u16::from_le_bytes([bytes[1], bytes[2]]) as usize;+ if key_len == 0 || key_len > MAX_KEY_BYTES {+ return None;+ }+ let body_len = match bytes[0] {+ RECORD_TYPE_ADD => {+ let fixed = key_len + dims * size_of::<f32>();+ fixed + plausible_attrs_len(bytes.get(RECORD_PREFIX_LEN + fixed..)?)?+ }+ RECORD_TYPE_TOMBSTONE => key_len,+ _ => return None,+ };+ let total = RECORD_PREFIX_LEN + body_len + RECORD_CRC_LEN;+ (bytes.len() >= total).then_some(total)+}++fn plausible_attrs_len(bytes: &[u8]) -> Option<usize> {+ let (&count, mut rest) = bytes.split_first()?;+ if count as usize > MAX_ATTR_COUNT {+ return None;+ }+ let mut attr_bytes = 0usize;+ for _ in 0..count {+ let (&name_len, after) = rest.split_first()?;+ if name_len == 0 || name_len as usize > MAX_ATTR_NAME_BYTES {+ return None;+ }+ let (_, after) = after.split_at_checked(name_len as usize)?;+ let (&tag, after) = after.split_first()?;+ let value_len = match tag {+ ATTR_TAG_STR => {+ let (len_bytes, _) = after.split_at_checked(2)?;+ let str_len = u16::from_le_bytes([len_bytes[0], len_bytes[1]]) as usize;+ if str_len > MAX_ATTR_STR_BYTES {+ return None;+ }+ 2 + str_len+ }+ ATTR_TAG_BOOL => 1,+ ATTR_TAG_I64 | ATTR_TAG_F64 => 8,+ _ => return None,+ };+ let (_, after) = after.split_at_checked(value_len)?;+ attr_bytes += 2 + name_len as usize + value_len;+ if attr_bytes > MAX_ATTRS_TOTAL_BYTES {+ return None;+ }+ rest = after;+ }+ Some(1 + attr_bytes)+}+fn encode_header(dims: u32, generation: &[u8; 16]) -> [u8; HEADER_LEN] {let mut bytes = [0u8; HEADER_LEN];bytes[0..4].copy_from_slice(&MAGIC);@@ -1906,4 +1991,82 @@ mod tests {assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);}}++ #[test]+ fn tail_probe_finds_an_intact_record_beyond_mid_log_damage() {+ let dims = 8usize;+ let vector = seeded_vector(21, dims);+ let first = encoded_add("first", &vector);+ let mut tombstone = Vec::new();+ encode_record_into(&mut tombstone, &LogEntry::Tombstone { key: "gone" });+ let survivors = [encoded_add("second", &vector), tombstone];+ let dir = TempDir::new().unwrap();+ for (index, survivor) in survivors.iter().enumerate() {+ for garbage_len in [1usize, 7, 19] {+ let path = dir.path().join(format!("log-{index}-{garbage_len}"));+ let mut record_bytes = first.clone();+ record_bytes.extend_from_slice(&vec![0xAA; garbage_len]);+ record_bytes.extend_from_slice(survivor);+ write_log_file(&path, dims as u32, &record_bytes);+ let mut log = reopen(&path, dims);+ let (records, recoverable_end) = scan_all(&mut log);+ assert_eq!(records.len(), 1);+ assert_eq!(recoverable_end, HEADER_LEN as u64 + first.len() as u64);+ assert!(log.tail_contains_valid_record(recoverable_end).unwrap());+ }+ }+ }++ #[test]+ fn tail_probe_rejects_regions_without_a_crc_valid_record() {+ let dims = 8usize;+ let vector = seeded_vector(22, dims);+ let good = encoded_add("good", &vector);+ let payload: Vec<u8> = vector+ .iter()+ .flat_map(|value| value.to_le_bytes())+ .collect();+ let mut crc_flipped_add = encoded_add("evil", &vector);+ *crc_flipped_add.last_mut().unwrap() ^= 0x01;+ let mut crc_flipped_tombstone = Vec::new();+ encode_record_into(+ &mut crc_flipped_tombstone,+ &LogEntry::Tombstone { key: "gone" },+ );+ *crc_flipped_tombstone.last_mut().unwrap() ^= 0x01;+ let mut oversized_key_payload = payload.clone();+ oversized_key_payload.push(0);+ let mut over_counted_attrs = payload.clone();+ over_counted_attrs.push(17);+ let mut truncated = good.clone();+ truncated.pop();+ let tails = [+ vec![0xFF; 64],+ vec![0x00; 64],+ vec![0x01; 64],+ vec![0x02; 64],+ crc_flipped_add,+ crc_flipped_tombstone,+ raw_record(RECORD_TYPE_ADD, 257, &[b'k'; 257], &oversized_key_payload),+ raw_record(RECORD_TYPE_ADD, 2, b"k1", &over_counted_attrs),+ truncated,+ Vec::new(),+ ];+ let dir = TempDir::new().unwrap();+ for (index, tail) in tails.iter().enumerate() {+ let path = dir.path().join(format!("tail-{index}"));+ let mut record_bytes = good.clone();+ record_bytes.push(0xAA);+ record_bytes.extend_from_slice(tail);+ write_log_file(&path, dims as u32, &record_bytes);+ let mut log = reopen(&path, dims);+ let (records, recoverable_end) = scan_all(&mut log);+ assert_eq!(records.len(), 1, "case {index}");+ assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);+ assert!(+ !log.tail_contains_valid_record(recoverable_end).unwrap(),+ "case {index}"+ );+ }+ }}diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rsindex c25a1b30b1..0028116fad 100644--- a/rust/crates/ml/src/vecdb/store.rs+++ b/rust/crates/ml/src/vecdb/store.rs@@ -1110,6 +1110,24 @@ fn replay(log: &mut Log, dims: usize) -> Result<ReplayedState, VecDbError> {}recoverable_end = scanner.recoverable_end();}+ let physical_end = log.current_end_offset();+ if recoverable_end < physical_end {+ if let Some(loaded) = &snapshot+ && loaded.covered_log_offset > recoverable_end+ {+ return Err(VecDbError::Corrupt(format!(+ "{}: log unreadable past offset {recoverable_end}, but its snapshot covers {} of {physical_end} bytes; preserving the file",+ log.path().display(),+ loaded.covered_log_offset+ )));+ }+ if log.tail_contains_valid_record(recoverable_end)? {+ return Err(VecDbError::Corrupt(format!(+ "{}: log unreadable at offset {recoverable_end} despite intact records before the end at {physical_end}; preserving the file",+ log.path().display()+ )));+ }+ }let (graph, used_snapshot) = match snapshot {Some(loaded) if loaded.covered_log_offset <= recoverable_end => {let snapshot_slots = covered_slot_count.unwrap_or(arena.slot_count());@@ -1239,6 +1257,7 @@ mod tests {use tempfile::TempDir;+ use super::super::crc::crc32;use super::super::kernel::splitmix64;use super::*;@@ -1584,6 +1603,161 @@ mod tests {assert_eq!(fs::metadata(&path).unwrap().len(), clean_len);}+ fn assert_corrupt_open_preserves(path: &Path) {+ let log_bytes = fs::read(path).unwrap();+ let snapshot_bytes = snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap());+ assert!(matches!(+ VecDb::open(path, DIMS),+ Err(VecDbError::Corrupt(_))+ ));+ assert!(matches!(+ VecDb::open_read_only(path, DIMS),+ Err(VecDbError::Corrupt(_))+ ));+ assert_eq!(fs::read(path).unwrap(), log_bytes);+ assert_eq!(+ snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap()),+ snapshot_bytes+ );+ }++ #[test]+ fn mid_log_corruption_below_the_snapshot_frontier_is_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 5, 300)).unwrap();+ db.flush().unwrap();+ drop(db);+ let clean = fs::read(&path).unwrap();+ let mut bytes = clean.clone();+ bytes[HEADER_LEN + ADD_RECORD_LEN as usize + 10] ^= 0x01;+ fs::write(&path, &bytes).unwrap();+ assert_corrupt_open_preserves(&path);+ fs::write(&path, &clean).unwrap();+ let recovered = open_writer(&path);+ assert_eq!(recovered.len(), 5);+ assert_own_nearest(&recovered, "key-1", &seeded_unit_vector(301, DIMS));+ }++ #[test]+ fn corruption_at_the_first_record_with_a_snapshot_is_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 3, 420)).unwrap();+ db.flush().unwrap();+ drop(db);+ let mut bytes = fs::read(&path).unwrap();+ bytes[HEADER_LEN + 10] ^= 0x01;+ fs::write(&path, &bytes).unwrap();+ assert_corrupt_open_preserves(&path);+ }++ #[test]+ fn mid_log_corruption_in_the_unsnapshotted_tail_is_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 4, 320)).unwrap();+ drop(db);+ assert!(!snapshot_exists(&path));+ let clean = fs::read(&path).unwrap();+ let mut bytes = clean.clone();+ bytes[HEADER_LEN + ADD_RECORD_LEN as usize + 10] ^= 0x01;+ fs::write(&path, &bytes).unwrap();+ assert_corrupt_open_preserves(&path);+ assert!(!snapshot_exists(&path));+ fs::write(&path, &clean).unwrap();+ assert_eq!(open_writer(&path).len(), 4);+ }++ #[test]+ fn mid_log_corruption_above_the_snapshot_frontier_is_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 2, 340)).unwrap();+ db.flush().unwrap();+ bulk_add(&db, &bulk_entries(2, 2, 340)).unwrap();+ drop(db);+ let mut bytes = fs::read(&path).unwrap();+ bytes[HEADER_LEN + 2 * ADD_RECORD_LEN as usize + 10] ^= 0x01;+ fs::write(&path, &bytes).unwrap();+ assert_corrupt_open_preserves(&path);+ }++ #[test]+ fn unknown_record_type_mid_log_with_valid_records_after_is_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 3, 360)).unwrap();+ drop(db);+ let clean = fs::read(&path).unwrap();+ let first_end = HEADER_LEN + ADD_RECORD_LEN as usize;+ let mut bytes = clean[..first_end].to_vec();+ bytes.extend_from_slice(&[3, 5, 0, b'j', b'u', b'n', b'k', b'y', 9, 9, 9, 9]);+ bytes.extend_from_slice(&clean[first_end..]);+ fs::write(&path, &bytes).unwrap();+ assert_corrupt_open_preserves(&path);+ }++ #[test]+ fn damaged_final_record_truncates_even_with_a_snapshot_below_the_stop() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 3, 380)).unwrap();+ db.flush().unwrap();+ db.add("key-3", &seeded_unit_vector(383, DIMS)).unwrap();+ drop(db);+ let mut bytes = fs::read(&path).unwrap();+ let last_start = bytes.len() - ADD_RECORD_LEN as usize;+ bytes[last_start + 10] ^= 0x01;+ fs::write(&path, &bytes).unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ assert_eq!(read_only.len(), 3);+ assert!(!read_only.contains("key-3"));+ drop(read_only);+ assert_eq!(fs::read(&path).unwrap(), bytes);+ let reopened = open_writer(&path);+ assert_eq!(reopened.len(), 3);+ assert!(!reopened.contains("key-3"));+ assert_own_nearest(&reopened, "key-2", &seeded_unit_vector(382, DIMS));+ drop(reopened);+ assert_eq!(fs::metadata(&path).unwrap().len(), last_start as u64);+ }++ #[test]+ fn future_format_version_is_rejected_with_the_file_preserved() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 3, 400)).unwrap();+ db.flush().unwrap();+ drop(db);+ let clean = fs::read(&path).unwrap();+ let mut bytes = clean.clone();+ bytes[4..6].copy_from_slice(&2u16.to_le_bytes());+ let crc = crc32(&bytes[0..28]);+ bytes[28..32].copy_from_slice(&crc.to_le_bytes());+ fs::write(&path, &bytes).unwrap();+ let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();+ assert!(matches!(+ VecDb::open(&path, DIMS),+ Err(VecDbError::Corrupt(message)) if message.contains("version")+ ));+ assert!(matches!(+ VecDb::open_read_only(&path, DIMS),+ Err(VecDbError::Corrupt(message)) if message.contains("version")+ ));+ assert_eq!(fs::read(&path).unwrap(), bytes);+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_bytes);+ fs::write(&path, &clean).unwrap();+ assert_eq!(open_writer(&path).len(), 3);+ }+#[test]fn deleted_snapshot_recovers_by_rebuild() {let dir = TempDir::new().unwrap();