Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: preserve the log when corruption is detected mid-file

laurenspriem302b844d8be7
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rs
index e50d991519..4595b973e3 100644
--- a/rust/crates/ml/src/vecdb/log.rs
+++ b/rust/crates/ml/src/vecdb/log.rs
@@ -228,6 +228,21 @@ impl Log {
Ok(())
}
+ pub(crate) fn tail_contains_valid_record(&mut self, after: u64) -> Result<bool, VecDbError> {
+ let start = after + 1;
+ if start >= self.end_offset {
+ return Ok(false);
+ }
+ self.file
+ .seek(SeekFrom::Start(start))
+ .map_err(|source| VecDbError::io(&self.path, source))?;
+ let mut region = vec![0u8; (self.end_offset - start) as usize];
+ self.file
+ .read_exact(&mut region)
+ .map_err(|source| VecDbError::io(&self.path, source))?;
+ Ok((0..region.len()).any(|offset| valid_record_at(&region[offset..], self.dims)))
+ }
+
pub(crate) fn current_end_offset(&self) -> u64 {
self.end_offset
}
@@ -459,6 +474,76 @@ fn decode_attrs(bytes: &[u8]) -> Option<Vec<Attribute>> {
rest.is_empty().then_some(attrs)
}
+fn valid_record_at(bytes: &[u8], dims: usize) -> bool {
+ let Some(record_len) = plausible_record_len(bytes, dims) else {
+ return false;
+ };
+ let crc_start = record_len - RECORD_CRC_LEN;
+ let stored = u32::from_le_bytes([
+ bytes[crc_start],
+ bytes[crc_start + 1],
+ bytes[crc_start + 2],
+ bytes[crc_start + 3],
+ ]);
+ crc32(&bytes[..crc_start]) == stored
+}
+
+fn plausible_record_len(bytes: &[u8], dims: usize) -> Option<usize> {
+ if bytes.len() < RECORD_PREFIX_LEN + RECORD_CRC_LEN {
+ return None;
+ }
+ let key_len = u16::from_le_bytes([bytes[1], bytes[2]]) as usize;
+ if key_len == 0 || key_len > MAX_KEY_BYTES {
+ return None;
+ }
+ let body_len = match bytes[0] {
+ RECORD_TYPE_ADD => {
+ let fixed = key_len + dims * size_of::<f32>();
+ fixed + plausible_attrs_len(bytes.get(RECORD_PREFIX_LEN + fixed..)?)?
+ }
+ RECORD_TYPE_TOMBSTONE => key_len,
+ _ => return None,
+ };
+ let total = RECORD_PREFIX_LEN + body_len + RECORD_CRC_LEN;
+ (bytes.len() >= total).then_some(total)
+}
+
+fn plausible_attrs_len(bytes: &[u8]) -> Option<usize> {
+ let (&count, mut rest) = bytes.split_first()?;
+ if count as usize > MAX_ATTR_COUNT {
+ return None;
+ }
+ let mut attr_bytes = 0usize;
+ for _ in 0..count {
+ let (&name_len, after) = rest.split_first()?;
+ if name_len == 0 || name_len as usize > MAX_ATTR_NAME_BYTES {
+ return None;
+ }
+ let (_, after) = after.split_at_checked(name_len as usize)?;
+ let (&tag, after) = after.split_first()?;
+ let value_len = match tag {
+ ATTR_TAG_STR => {
+ let (len_bytes, _) = after.split_at_checked(2)?;
+ let str_len = u16::from_le_bytes([len_bytes[0], len_bytes[1]]) as usize;
+ if str_len > MAX_ATTR_STR_BYTES {
+ return None;
+ }
+ 2 + str_len
+ }
+ ATTR_TAG_BOOL => 1,
+ ATTR_TAG_I64 | ATTR_TAG_F64 => 8,
+ _ => return None,
+ };
+ let (_, after) = after.split_at_checked(value_len)?;
+ attr_bytes += 2 + name_len as usize + value_len;
+ if attr_bytes > MAX_ATTRS_TOTAL_BYTES {
+ return None;
+ }
+ rest = after;
+ }
+ Some(1 + attr_bytes)
+}
+
fn encode_header(dims: u32, generation: &[u8; 16]) -> [u8; HEADER_LEN] {
let mut bytes = [0u8; HEADER_LEN];
bytes[0..4].copy_from_slice(&MAGIC);
@@ -1906,4 +1991,82 @@ mod tests {
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
}
}
+
+ #[test]
+ fn tail_probe_finds_an_intact_record_beyond_mid_log_damage() {
+ let dims = 8usize;
+ let vector = seeded_vector(21, dims);
+ let first = encoded_add("first", &vector);
+ let mut tombstone = Vec::new();
+ encode_record_into(&mut tombstone, &LogEntry::Tombstone { key: "gone" });
+ let survivors = [encoded_add("second", &vector), tombstone];
+ let dir = TempDir::new().unwrap();
+ for (index, survivor) in survivors.iter().enumerate() {
+ for garbage_len in [1usize, 7, 19] {
+ let path = dir.path().join(format!("log-{index}-{garbage_len}"));
+ let mut record_bytes = first.clone();
+ record_bytes.extend_from_slice(&vec![0xAA; garbage_len]);
+ record_bytes.extend_from_slice(survivor);
+ write_log_file(&path, dims as u32, &record_bytes);
+ let mut log = reopen(&path, dims);
+ let (records, recoverable_end) = scan_all(&mut log);
+ assert_eq!(records.len(), 1);
+ assert_eq!(recoverable_end, HEADER_LEN as u64 + first.len() as u64);
+ assert!(log.tail_contains_valid_record(recoverable_end).unwrap());
+ }
+ }
+ }
+
+ #[test]
+ fn tail_probe_rejects_regions_without_a_crc_valid_record() {
+ let dims = 8usize;
+ let vector = seeded_vector(22, dims);
+ let good = encoded_add("good", &vector);
+ let payload: Vec<u8> = vector
+ .iter()
+ .flat_map(|value| value.to_le_bytes())
+ .collect();
+ let mut crc_flipped_add = encoded_add("evil", &vector);
+ *crc_flipped_add.last_mut().unwrap() ^= 0x01;
+ let mut crc_flipped_tombstone = Vec::new();
+ encode_record_into(
+ &mut crc_flipped_tombstone,
+ &LogEntry::Tombstone { key: "gone" },
+ );
+ *crc_flipped_tombstone.last_mut().unwrap() ^= 0x01;
+ let mut oversized_key_payload = payload.clone();
+ oversized_key_payload.push(0);
+ let mut over_counted_attrs = payload.clone();
+ over_counted_attrs.push(17);
+ let mut truncated = good.clone();
+ truncated.pop();
+ let tails = [
+ vec![0xFF; 64],
+ vec![0x00; 64],
+ vec![0x01; 64],
+ vec![0x02; 64],
+ crc_flipped_add,
+ crc_flipped_tombstone,
+ raw_record(RECORD_TYPE_ADD, 257, &[b'k'; 257], &oversized_key_payload),
+ raw_record(RECORD_TYPE_ADD, 2, b"k1", &over_counted_attrs),
+ truncated,
+ Vec::new(),
+ ];
+ let dir = TempDir::new().unwrap();
+ for (index, tail) in tails.iter().enumerate() {
+ let path = dir.path().join(format!("tail-{index}"));
+ let mut record_bytes = good.clone();
+ record_bytes.push(0xAA);
+ record_bytes.extend_from_slice(tail);
+ write_log_file(&path, dims as u32, &record_bytes);
+ let mut log = reopen(&path, dims);
+ let (records, recoverable_end) = scan_all(&mut log);
+ assert_eq!(records.len(), 1, "case {index}");
+ assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
+ assert!(
+ !log.tail_contains_valid_record(recoverable_end).unwrap(),
+ "case {index}"
+ );
+ }
+ }
}
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rs
index c25a1b30b1..0028116fad 100644
--- a/rust/crates/ml/src/vecdb/store.rs
+++ b/rust/crates/ml/src/vecdb/store.rs
@@ -1110,6 +1110,24 @@ fn replay(log: &mut Log, dims: usize) -> Result<ReplayedState, VecDbError> {
}
recoverable_end = scanner.recoverable_end();
}
+ let physical_end = log.current_end_offset();
+ if recoverable_end < physical_end {
+ if let Some(loaded) = &snapshot
+ && loaded.covered_log_offset > recoverable_end
+ {
+ return Err(VecDbError::Corrupt(format!(
+ "{}: log unreadable past offset {recoverable_end}, but its snapshot covers {} of {physical_end} bytes; preserving the file",
+ log.path().display(),
+ loaded.covered_log_offset
+ )));
+ }
+ if log.tail_contains_valid_record(recoverable_end)? {
+ return Err(VecDbError::Corrupt(format!(
+ "{}: log unreadable at offset {recoverable_end} despite intact records before the end at {physical_end}; preserving the file",
+ log.path().display()
+ )));
+ }
+ }
let (graph, used_snapshot) = match snapshot {
Some(loaded) if loaded.covered_log_offset <= recoverable_end => {
let snapshot_slots = covered_slot_count.unwrap_or(arena.slot_count());
@@ -1239,6 +1257,7 @@ mod tests {
use tempfile::TempDir;
+ use super::super::crc::crc32;
use super::super::kernel::splitmix64;
use super::*;
@@ -1584,6 +1603,161 @@ mod tests {
assert_eq!(fs::metadata(&path).unwrap().len(), clean_len);
}
+ fn assert_corrupt_open_preserves(path: &Path) {
+ let log_bytes = fs::read(path).unwrap();
+ let snapshot_bytes = snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap());
+ assert!(matches!(
+ VecDb::open(path, DIMS),
+ Err(VecDbError::Corrupt(_))
+ ));
+ assert!(matches!(
+ VecDb::open_read_only(path, DIMS),
+ Err(VecDbError::Corrupt(_))
+ ));
+ assert_eq!(fs::read(path).unwrap(), log_bytes);
+ assert_eq!(
+ snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap()),
+ snapshot_bytes
+ );
+ }
+
+ #[test]
+ fn mid_log_corruption_below_the_snapshot_frontier_is_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 5, 300)).unwrap();
+ db.flush().unwrap();
+ drop(db);
+ let clean = fs::read(&path).unwrap();
+ let mut bytes = clean.clone();
+ bytes[HEADER_LEN + ADD_RECORD_LEN as usize + 10] ^= 0x01;
+ fs::write(&path, &bytes).unwrap();
+ assert_corrupt_open_preserves(&path);
+ fs::write(&path, &clean).unwrap();
+ let recovered = open_writer(&path);
+ assert_eq!(recovered.len(), 5);
+ assert_own_nearest(&recovered, "key-1", &seeded_unit_vector(301, DIMS));
+ }
+
+ #[test]
+ fn corruption_at_the_first_record_with_a_snapshot_is_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 3, 420)).unwrap();
+ db.flush().unwrap();
+ drop(db);
+ let mut bytes = fs::read(&path).unwrap();
+ bytes[HEADER_LEN + 10] ^= 0x01;
+ fs::write(&path, &bytes).unwrap();
+ assert_corrupt_open_preserves(&path);
+ }
+
+ #[test]
+ fn mid_log_corruption_in_the_unsnapshotted_tail_is_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 4, 320)).unwrap();
+ drop(db);
+ assert!(!snapshot_exists(&path));
+ let clean = fs::read(&path).unwrap();
+ let mut bytes = clean.clone();
+ bytes[HEADER_LEN + ADD_RECORD_LEN as usize + 10] ^= 0x01;
+ fs::write(&path, &bytes).unwrap();
+ assert_corrupt_open_preserves(&path);
+ assert!(!snapshot_exists(&path));
+ fs::write(&path, &clean).unwrap();
+ assert_eq!(open_writer(&path).len(), 4);
+ }
+
+ #[test]
+ fn mid_log_corruption_above_the_snapshot_frontier_is_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 2, 340)).unwrap();
+ db.flush().unwrap();
+ bulk_add(&db, &bulk_entries(2, 2, 340)).unwrap();
+ drop(db);
+ let mut bytes = fs::read(&path).unwrap();
+ bytes[HEADER_LEN + 2 * ADD_RECORD_LEN as usize + 10] ^= 0x01;
+ fs::write(&path, &bytes).unwrap();
+ assert_corrupt_open_preserves(&path);
+ }
+
+ #[test]
+ fn unknown_record_type_mid_log_with_valid_records_after_is_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 3, 360)).unwrap();
+ drop(db);
+ let clean = fs::read(&path).unwrap();
+ let first_end = HEADER_LEN + ADD_RECORD_LEN as usize;
+ let mut bytes = clean[..first_end].to_vec();
+ bytes.extend_from_slice(&[3, 5, 0, b'j', b'u', b'n', b'k', b'y', 9, 9, 9, 9]);
+ bytes.extend_from_slice(&clean[first_end..]);
+ fs::write(&path, &bytes).unwrap();
+ assert_corrupt_open_preserves(&path);
+ }
+
+ #[test]
+ fn damaged_final_record_truncates_even_with_a_snapshot_below_the_stop() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 3, 380)).unwrap();
+ db.flush().unwrap();
+ db.add("key-3", &seeded_unit_vector(383, DIMS)).unwrap();
+ drop(db);
+ let mut bytes = fs::read(&path).unwrap();
+ let last_start = bytes.len() - ADD_RECORD_LEN as usize;
+ bytes[last_start + 10] ^= 0x01;
+ fs::write(&path, &bytes).unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ assert_eq!(read_only.len(), 3);
+ assert!(!read_only.contains("key-3"));
+ drop(read_only);
+ assert_eq!(fs::read(&path).unwrap(), bytes);
+ let reopened = open_writer(&path);
+ assert_eq!(reopened.len(), 3);
+ assert!(!reopened.contains("key-3"));
+ assert_own_nearest(&reopened, "key-2", &seeded_unit_vector(382, DIMS));
+ drop(reopened);
+ assert_eq!(fs::metadata(&path).unwrap().len(), last_start as u64);
+ }
+
+ #[test]
+ fn future_format_version_is_rejected_with_the_file_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 3, 400)).unwrap();
+ db.flush().unwrap();
+ drop(db);
+ let clean = fs::read(&path).unwrap();
+ let mut bytes = clean.clone();
+ bytes[4..6].copy_from_slice(&2u16.to_le_bytes());
+ let crc = crc32(&bytes[0..28]);
+ bytes[28..32].copy_from_slice(&crc.to_le_bytes());
+ fs::write(&path, &bytes).unwrap();
+ let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();
+ assert!(matches!(
+ VecDb::open(&path, DIMS),
+ Err(VecDbError::Corrupt(message)) if message.contains("version")
+ ));
+ assert!(matches!(
+ VecDb::open_read_only(&path, DIMS),
+ Err(VecDbError::Corrupt(message)) if message.contains("version")
+ ));
+ assert_eq!(fs::read(&path).unwrap(), bytes);
+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_bytes);
+ fs::write(&path, &clean).unwrap();
+ assert_eq!(open_writer(&path).len(), 3);
+ }
+
#[test]
fn deleted_snapshot_recovers_by_rebuild() {
let dir = TempDir::new().unwrap();