Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: treat a log shorter than its snapshot frontier as corruption

laurenspriem8e481298ad77
diff --git a/rust/crates/ml/src/vecdb/snapshot.rs b/rust/crates/ml/src/vecdb/snapshot.rs
index b5602c7f1a..9cfcde69c2 100644
--- a/rust/crates/ml/src/vecdb/snapshot.rs
+++ b/rust/crates/ml/src/vecdb/snapshot.rs
@@ -42,7 +42,6 @@ pub(crate) fn write_snapshot(
pub(crate) fn load_snapshot(
log_path: &Path,
expected_generation: [u8; 16],
- log_end_offset: u64,
) -> Option<LoadedSnapshot> {
let path = snapshot_path(log_path);
let bytes = match std::fs::read(&path) {
@@ -53,7 +52,7 @@ pub(crate) fn load_snapshot(
return None;
}
};
- match decode_snapshot(&bytes, &expected_generation, log_end_offset) {
+ match decode_snapshot(&bytes, &expected_generation) {
Ok(loaded) => Some(loaded),
Err(reason) => {
log::debug!("discarding snapshot {}: {reason}", path.display());
@@ -98,11 +97,7 @@ fn encode_snapshot(generation: &[u8; 16], covered_log_offset: u64, graph: &Graph
bytes
}
-fn decode_snapshot(
- bytes: &[u8],
- expected_generation: &[u8; 16],
- log_end_offset: u64,
-) -> Result<LoadedSnapshot, String> {
+fn decode_snapshot(bytes: &[u8], expected_generation: &[u8; 16]) -> Result<LoadedSnapshot, String> {
if bytes.len() < MIN_SNAPSHOT_LEN {
return Err(format!(
"{} bytes is shorter than the {MIN_SNAPSHOT_LEN}-byte minimum",
@@ -141,11 +136,6 @@ fn decode_snapshot(
return Err("generation does not match the log".to_string());
}
let covered_log_offset = reader.read_u64().ok_or("truncated header")?;
- if covered_log_offset > log_end_offset {
- return Err(format!(
- "covered offset {covered_log_offset} is past the log end {log_end_offset}"
- ));
- }
let entry_marker = reader.read_u32().ok_or("truncated header")?;
let top_level = reader.read_u8().ok_or("truncated header")?;
let node_count = reader.read_u32().ok_or("truncated header")? as usize;
@@ -311,7 +301,7 @@ mod tests {
}
fn load_golden(log_path: &Path) -> Option<LoadedSnapshot> {
- load_snapshot(log_path, golden_generation(), 300)
+ load_snapshot(log_path, golden_generation())
}
fn refresh_crc(bytes: &mut [u8]) {
@@ -450,7 +440,7 @@ mod tests {
let generation = [7u8; 16];
write_snapshot(&log_path, generation, 4096, &graph).unwrap();
assert!(!temp_snapshot_path(&log_path).exists());
- let loaded = load_snapshot(&log_path, generation, 4096).unwrap();
+ let loaded = load_snapshot(&log_path, generation).unwrap();
assert_eq!(loaded.covered_log_offset, 4096);
assert_eq!(loaded.entry_point, graph.entry_point());
let rebuilt =
@@ -482,7 +472,7 @@ mod tests {
std::fs::read(snapshot_path(&log_path)).unwrap().len(),
MIN_SNAPSHOT_LEN
);
- let loaded = load_snapshot(&log_path, [3u8; 16], 32).unwrap();
+ let loaded = load_snapshot(&log_path, [3u8; 16]).unwrap();
assert_eq!(loaded.covered_log_offset, 32);
assert_eq!(loaded.entry_point, None);
assert!(loaded.parts.is_empty());
@@ -538,7 +528,7 @@ mod tests {
let mut decoded = 0usize;
let mut searched = 0usize;
let mut exercise = |bytes: &[u8]| {
- let Ok(loaded) = decode_snapshot(bytes, &generation, 300) else {
+ let Ok(loaded) = decode_snapshot(bytes, &generation) else {
return;
};
decoded += 1;
@@ -591,7 +581,7 @@ mod tests {
bytes.extend_from_slice(&0u16.to_le_bytes());
let crc = crc32(&bytes[MAGIC.len()..]);
bytes.extend_from_slice(&crc.to_le_bytes());
- let loaded = decode_snapshot(&bytes, &golden_generation(), 300).unwrap();
+ let loaded = decode_snapshot(&bytes, &golden_generation()).unwrap();
assert_eq!(loaded.entry_point, Some(slot));
assert!(Graph::from_parts(loaded.entry_point, loaded.parts, 3).is_err());
}
@@ -614,18 +604,17 @@ mod tests {
let dir = TempDir::new().unwrap();
let log_path = dir.path().join("log");
std::fs::write(snapshot_path(&log_path), GOLDEN).unwrap();
- assert!(load_snapshot(&log_path, [9u8; 16], 300).is_none());
+ assert!(load_snapshot(&log_path, [9u8; 16]).is_none());
assert!(load_golden(&log_path).is_some());
}
#[test]
- fn covered_offset_beyond_log_end_is_discarded() {
+ fn loading_exposes_the_covered_offset_for_the_caller_to_judge() {
let dir = TempDir::new().unwrap();
let log_path = dir.path().join("log");
std::fs::write(snapshot_path(&log_path), GOLDEN).unwrap();
- assert!(load_snapshot(&log_path, golden_generation(), 299).is_none());
- assert!(load_snapshot(&log_path, golden_generation(), 300).is_some());
- assert!(load_snapshot(&log_path, golden_generation(), 301).is_some());
+ let loaded = load_snapshot(&log_path, golden_generation()).unwrap();
+ assert_eq!(loaded.covered_log_offset, 300);
}
#[test]
@@ -715,7 +704,7 @@ mod tests {
write_snapshot(&log_path, generation, 100, &golden_graph()).unwrap();
write_snapshot(&log_path, generation, 999, &golden_graph()).unwrap();
assert!(!temp_snapshot_path(&log_path).exists());
- let loaded = load_snapshot(&log_path, generation, 1000).unwrap();
+ let loaded = load_snapshot(&log_path, generation).unwrap();
assert_eq!(loaded.covered_log_offset, 999);
assert_eq!(loaded.parts.len(), 3);
}
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rs
index 0028116fad..1f911204fb 100644
--- a/rust/crates/ml/src/vecdb/store.rs
+++ b/rust/crates/ml/src/vecdb/store.rs
@@ -1069,7 +1069,17 @@ struct ReplayedState {
}
fn replay(log: &mut Log, dims: usize) -> Result<ReplayedState, VecDbError> {
- let snapshot = load_snapshot(log.path(), log.generation(), log.current_end_offset());
+ let snapshot = load_snapshot(log.path(), log.generation());
+ if let Some(loaded) = &snapshot
+ && loaded.covered_log_offset > log.current_end_offset()
+ {
+ return Err(VecDbError::Corrupt(format!(
+ "{}: snapshot covers {} bytes but the log is only {} bytes; preserving the file",
+ log.path().display(),
+ loaded.covered_log_offset,
+ log.current_end_offset()
+ )));
+ }
let covered = snapshot
.as_ref()
.map_or(u64::MAX, |loaded| loaded.covered_log_offset);
@@ -1621,6 +1631,25 @@ mod tests {
);
}
+ #[test]
+ fn externally_shortened_log_below_the_snapshot_frontier_is_preserved() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ db.add("first", &seeded_unit_vector(1, DIMS)).unwrap();
+ let boundary = db.stats().unwrap().log_bytes;
+ db.add("second", &seeded_unit_vector(2, DIMS)).unwrap();
+ db.flush().unwrap();
+ drop(db);
+ let clean = fs::read(&path).unwrap();
+ fs::write(&path, &clean[..boundary as usize]).unwrap();
+ assert_corrupt_open_preserves(&path);
+ fs::write(&path, &clean).unwrap();
+ let recovered = open_writer(&path);
+ assert_eq!(recovered.len(), 2);
+ assert!(recovered.contains("second"));
+ }
+
#[test]
fn mid_log_corruption_below_the_snapshot_frontier_is_preserved() {
let dir = TempDir::new().unwrap();
@@ -1795,23 +1824,6 @@ mod tests {
assert_own_nearest(&reopened, "fresh", &seeded_unit_vector(1, DIMS));
}
- #[test]
- fn snapshot_claiming_offset_beyond_log_end_is_discarded() {
- let dir = TempDir::new().unwrap();
- let path = dir.path().join("db");
- let db = open_writer(&path);
- bulk_add(&db, &bulk_entries(0, 5, 80)).unwrap();
- db.flush().unwrap();
- drop(db);
- let bytes = fs::read(&path).unwrap();
- fs::write(&path, &bytes[..bytes.len() - ADD_RECORD_LEN as usize]).unwrap();
- let reopened = open_writer(&path);
- assert_eq!(reopened.len(), 4);
- assert!(reopened.contains("key-3"));
- assert!(!reopened.contains("key-4"));
- assert_own_nearest(&reopened, "key-0", &seeded_unit_vector(80, DIMS));
- }
-
#[test]
fn corrupted_snapshot_bytes_are_discarded() {
let dir = TempDir::new().unwrap();
@@ -2336,7 +2348,7 @@ mod tests {
drop(open_writer(&path));
assert!(snapshot_exists(&path));
let log_len = fs::metadata(&path).unwrap().len();
- let loaded = load_snapshot(&path, generation_of(&path), log_len).unwrap();
+ let loaded = load_snapshot(&path, generation_of(&path)).unwrap();
assert_eq!(loaded.covered_log_offset, log_len);
let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();
drop(open_writer(&path));
@@ -2368,7 +2380,7 @@ mod tests {
assert_eq!(stats.records_since_snapshot, 0);
assert!(stats.log_bytes < log_bytes_before);
assert_ne!(generation_of(&path), generation_before);
- let loaded = load_snapshot(&path, generation_of(&path), stats.log_bytes).unwrap();
+ let loaded = load_snapshot(&path, generation_of(&path)).unwrap();
assert_eq!(loaded.covered_log_offset, stats.log_bytes);
let live_vector = |index: u64| {
if index < 40 {
@@ -2453,7 +2465,7 @@ mod tests {
assert_own_nearest(&reopened, "key-100", &seeded_unit_vector(310, DIMS));
let stats = reopened.stats().unwrap();
assert_eq!(stats.dead_count, 0);
- let loaded = load_snapshot(&path, generation_of(&path), stats.log_bytes).unwrap();
+ let loaded = load_snapshot(&path, generation_of(&path)).unwrap();
assert_eq!(loaded.covered_log_offset, stats.log_bytes);
assert_ne!(fs::read(snapshot_path(&path)).unwrap(), stale_snapshot);
}