Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: keep the lock sidecar as a persistent per-path singleton and scan the corruption tail probe through a bounded sliding window

laurenspriem2389b09ad8a5
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rs
index c6ee72a074..b22c93c23b 100644
--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs
+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs
@@ -774,7 +774,7 @@ mod tests {
delete_vec_db_files(dir.db_path()).unwrap();
assert!(!PathBuf::from(dir.db_path()).exists());
assert!(!PathBuf::from(format!("{}.graph", dir.db_path())).exists());
- assert!(!PathBuf::from(format!("{}.lock", dir.db_path())).exists());
+ assert!(PathBuf::from(format!("{}.lock", dir.db_path())).exists());
assert!(matches!(
db.get_index_stats(),
Err(RustVecDbError::Closed { .. })
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rs
index 41e5c97da0..abd5da0f4a 100644
--- a/rust/crates/ml/src/vecdb/log.rs
+++ b/rust/crates/ml/src/vecdb/log.rs
@@ -27,6 +27,7 @@ const ATTR_TAG_STR: u8 = 0;
const ATTR_TAG_BOOL: u8 = 1;
const ATTR_TAG_I64: u8 = 2;
const ATTR_TAG_F64: u8 = 3;
+const TAIL_PROBE_STEP_BYTES: usize = 4 * 1024 * 1024;
static GENERATION_COUNTER: AtomicU64 = AtomicU64::new(0);
@@ -242,18 +243,41 @@ impl Log {
}
pub(crate) fn tail_contains_valid_record(&mut self, after: u64) -> Result<bool, VecDbError> {
+ self.tail_contains_valid_record_windowed(after, TAIL_PROBE_STEP_BYTES)
+ }
+
+ fn tail_contains_valid_record_windowed(
+ &mut self,
+ after: u64,
+ step_bytes: usize,
+ ) -> Result<bool, VecDbError> {
let start = after + 1;
if start >= self.end_offset {
return Ok(false);
}
- self.file
- .seek(SeekFrom::Start(start))
- .map_err(|source| VecDbError::io(&self.path, source))?;
- let mut region = vec![0u8; (self.end_offset - start) as usize];
- self.file
- .read_exact(&mut region)
- .map_err(|source| VecDbError::io(&self.path, source))?;
- Ok((0..region.len()).any(|offset| valid_record_at(&region[offset..], self.dims)))
+ let tail_len = (self.end_offset - start) as usize;
+ let step = step_bytes.max(1);
+ let overlap = max_record_len(self.dims).saturating_sub(1);
+ let mut window = vec![0u8; step.saturating_add(overlap).min(tail_len)];
+ let mut base = 0usize;
+ loop {
+ let len = window.len().min(tail_len - base);
+ self.file
+ .seek(SeekFrom::Start(start + base as u64))
+ .map_err(|source| VecDbError::io(&self.path, source))?;
+ self.file
+ .read_exact(&mut window[..len])
+ .map_err(|source| VecDbError::io(&self.path, source))?;
+ let reaches_end = base + len == tail_len;
+ let probe_end = if reaches_end { len } else { step.min(len) };
+ if (0..probe_end).any(|offset| valid_record_at(&window[offset..len], self.dims)) {
+ return Ok(true);
+ }
+ if reaches_end {
+ return Ok(false);
+ }
+ base += step;
+ }
}
pub(crate) fn current_end_offset(&self) -> u64 {
@@ -487,6 +511,12 @@ fn decode_attrs(bytes: &[u8]) -> Option<Vec<Attribute>> {
rest.is_empty().then_some(attrs)
}
+fn max_record_len(dims: usize) -> usize {
+ dims.saturating_mul(size_of::<f32>()).saturating_add(
+ RECORD_PREFIX_LEN + MAX_KEY_BYTES + 1 + MAX_ATTRS_TOTAL_BYTES + RECORD_CRC_LEN,
+ )
+}
+
fn valid_record_at(bytes: &[u8], dims: usize) -> bool {
let Some(record_len) = plausible_record_len(bytes, dims) else {
return false;
@@ -2139,4 +2169,147 @@ mod tests {
);
}
}
+
+ #[test]
+ fn windowed_probe_finds_the_only_valid_record_straddling_a_window_boundary() {
+ let dims = 8usize;
+ let step = 4096usize;
+ let vector = seeded_vector(31, dims);
+ let good = encoded_add("good", &vector);
+ let straddler = encoded_add("straddler", &vector);
+ let mut record_bytes = good.clone();
+ record_bytes.extend_from_slice(&vec![0xAA; step - 19]);
+ record_bytes.extend_from_slice(&straddler);
+ record_bytes.extend_from_slice(&vec![0xAA; step + max_record_len(dims)]);
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("log");
+ write_log_file(&path, dims as u32, &record_bytes);
+ let mut log = reopen(&path, dims);
+ let (records, recoverable_end) = scan_all(&mut log);
+ assert_eq!(records.len(), 1);
+ assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
+ let straddler_start = (step - 20) as u64;
+ let straddler_end = straddler_start + straddler.len() as u64;
+ assert!(straddler_start < step as u64);
+ assert!(straddler_end > step as u64);
+ let tail_len = log.current_end_offset() - recoverable_end - 1;
+ assert!(tail_len as usize > step + max_record_len(dims) - 1);
+ assert!(
+ log.tail_contains_valid_record_windowed(recoverable_end, step)
+ .unwrap()
+ );
+ assert!(log.tail_contains_valid_record(recoverable_end).unwrap());
+ }
+
+ #[test]
+ fn windowed_probe_scans_an_all_garbage_multi_window_tail_without_a_false_positive() {
+ let dims = 8usize;
+ let step = 64usize;
+ let vector = seeded_vector(32, dims);
+ let good = encoded_add("good", &vector);
+ let mut record_bytes = good.clone();
+ record_bytes.extend_from_slice(&vec![0x01; 3 * (step + max_record_len(dims))]);
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("log");
+ write_log_file(&path, dims as u32, &record_bytes);
+ let mut log = reopen(&path, dims);
+ let (records, recoverable_end) = scan_all(&mut log);
+ assert_eq!(records.len(), 1);
+ assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
+ assert!(
+ !log.tail_contains_valid_record_windowed(recoverable_end, step)
+ .unwrap()
+ );
+ assert!(!log.tail_contains_valid_record(recoverable_end).unwrap());
+ }
+
+ #[test]
+ fn windowed_probe_with_a_step_smaller_than_a_record_terminates_and_finds_a_straddler() {
+ let dims = 8usize;
+ let step = 4usize;
+ let vector = seeded_vector(33, dims);
+ let good = encoded_add("good", &vector);
+ let straddler = encoded_add("straddler", &vector);
+ assert!(step < straddler.len());
+ let mut record_bytes = good.clone();
+ record_bytes.extend_from_slice(&vec![0x01; 5000]);
+ record_bytes.extend_from_slice(&straddler);
+ record_bytes.extend_from_slice(&[0x01; 20]);
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("log");
+ write_log_file(&path, dims as u32, &record_bytes);
+ let mut log = reopen(&path, dims);
+ let (records, recoverable_end) = scan_all(&mut log);
+ assert_eq!(records.len(), 1);
+ assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
+ let tail_len = log.current_end_offset() - recoverable_end - 1;
+ assert!(tail_len as usize > step + max_record_len(dims) - 1);
+ assert!(
+ log.tail_contains_valid_record_windowed(recoverable_end, step)
+ .unwrap()
+ );
+ assert!(
+ !log.tail_contains_valid_record_windowed(recoverable_end - 1 + tail_len, step)
+ .unwrap()
+ );
+ }
+
+ #[test]
+ fn windowed_probe_sees_a_max_length_record_ending_exactly_at_the_window_visible_end() {
+ let dims = 8usize;
+ let step = 4096usize;
+ let overlap = max_record_len(dims) - 1;
+ let vector = seeded_vector(34, dims);
+ let good = encoded_add("good", &vector);
+ let attrs: Vec<Attribute> = (b'a'..b'a' + 16)
+ .map(|name| Attribute {
+ name: (name as char).to_string(),
+ value: AttrValue::Str("v".repeat(251)),
+ })
+ .collect();
+ let key = "k".repeat(256);
+ let mut max_record = Vec::new();
+ encode_record_into(
+ &mut max_record,
+ &LogEntry::Add {
+ key: &key,
+ vector: &vector,
+ attrs: &attrs,
+ },
+ );
+ assert_eq!(max_record.len(), max_record_len(dims));
+ let mut record_bytes = good.clone();
+ record_bytes.extend_from_slice(&vec![0xAA; step]);
+ record_bytes.extend_from_slice(&max_record);
+ record_bytes.extend_from_slice(&vec![0xAA; 256]);
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("log");
+ write_log_file(&path, dims as u32, &record_bytes);
+ let mut log = reopen(&path, dims);
+ let (records, recoverable_end) = scan_all(&mut log);
+ assert_eq!(records.len(), 1);
+ assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
+ let record_tail_start = step - 1;
+ assert_eq!(record_tail_start + max_record.len(), step + overlap);
+ let tail_len = (log.current_end_offset() - recoverable_end - 1) as usize;
+ assert!(tail_len > step + overlap);
+ assert!(
+ log.tail_contains_valid_record_windowed(recoverable_end, step)
+ .unwrap()
+ );
+ assert!(log.tail_contains_valid_record(recoverable_end).unwrap());
+ let mut corrupted_bytes = record_bytes.clone();
+ let crc_last = good.len() + step + max_record.len() - 1;
+ corrupted_bytes[crc_last] ^= 0xFF;
+ let corrupted_path = dir.path().join("log-corrupted");
+ write_log_file(&corrupted_path, dims as u32, &corrupted_bytes);
+ let mut corrupted_log = reopen(&corrupted_path, dims);
+ let (corrupted_records, corrupted_end) = scan_all(&mut corrupted_log);
+ assert_eq!(corrupted_records.len(), 1);
+ assert!(
+ !corrupted_log
+ .tail_contains_valid_record_windowed(corrupted_end, step)
+ .unwrap()
+ );
+ }
}
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rs
index 5541a9c677..86e9723c14 100644
--- a/rust/crates/ml/src/vecdb/store.rs
+++ b/rust/crates/ml/src/vecdb/store.rs
@@ -11,7 +11,7 @@ use std::time::{Duration, Instant};
use super::arena::{UpsertOutcome, VECTORS_PER_CHUNK, VectorArena};
use super::graph::{Graph, search as graph_search, search_excluding};
-use super::lock::{WriterLock, lock_path};
+use super::lock::WriterLock;
use super::log::{
HEADER_LEN, Log, LogEntry, LogRecord, header_generation, remove_if_present,
remove_stale_temp_sibling, sync_parent_dir,
@@ -646,7 +646,7 @@ impl VecDb {
}
}
let lock = WriterLock::acquire(&key)?;
- remove_index_files(&key)?;
+ remove_data_files(&key)?;
drop(lock);
Ok(())
}
@@ -796,7 +796,7 @@ fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {
let mut st = shared.state_write();
shared.closed.store(true, Ordering::Release);
drop(log);
- let removed = remove_index_files(&shared.path);
+ let removed = remove_data_files(&shared.path);
if let Ok(empty) = VectorArena::new(shared.dims) {
st.arena = empty;
st.graph = Some(Graph::new());
@@ -1418,12 +1418,6 @@ fn promote_compacted_log(temp_path: &Path, path: &Path) -> Result<(), VecDbError
sync_parent_dir(path)
}
-fn remove_index_files(path: &Path) -> Result<(), VecDbError> {
- remove_data_files(path)?;
- remove_if_present(&lock_path(path))?;
- sync_parent_dir(path)
-}
-
fn remove_data_files(path: &Path) -> Result<(), VecDbError> {
remove_if_present(path)?;
remove_snapshot(path)?;
@@ -1490,6 +1484,7 @@ mod tests {
use super::super::crc::crc32;
use super::super::kernel::splitmix64;
+ use super::super::lock::lock_path;
use super::super::test_support;
use super::*;
@@ -3102,7 +3097,7 @@ mod tests {
}
#[test]
- fn delete_removes_all_files() {
+ fn delete_removes_data_files_and_keeps_the_lock_sidecar() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let db = open_writer(&path);
@@ -3116,7 +3111,39 @@ mod tests {
assert!(!path.exists());
assert!(!snapshot_exists(&path));
assert!(!temp_sibling(&path).exists());
- assert!(!lock_path(&path).exists());
+ assert!(lock_path(&path).exists());
+ }
+
+ #[test]
+ fn reopen_after_delete_reuses_the_persistent_lock_sidecar() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ db.add("doomed", &seeded_unit_vector(2, DIMS)).unwrap();
+ db.delete().unwrap();
+ assert!(lock_path(&path).exists());
+ #[cfg(unix)]
+ let sidecar_inode = {
+ use std::os::unix::fs::MetadataExt;
+ fs::metadata(lock_path(&path)).unwrap().ino()
+ };
+ let reopened = open_writer(&path);
+ assert!(reopened.is_empty());
+ assert!(matches!(
+ WriterLock::acquire(&path),
+ Err(VecDbError::Locked(_))
+ ));
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::MetadataExt;
+ assert_eq!(fs::metadata(lock_path(&path)).unwrap().ino(), sidecar_inode);
+ }
+ reopened.add("fresh", &seeded_unit_vector(3, DIMS)).unwrap();
+ drop(reopened);
+ let third = open_writer(&path);
+ assert_eq!(third.len(), 1);
+ assert!(third.contains("fresh"));
+ assert!(lock_path(&path).exists());
}
#[test]
@@ -3131,7 +3158,7 @@ mod tests {
first.delete().unwrap();
assert!(!path.exists());
assert!(!snapshot_exists(&path));
- assert!(!lock_path(&path).exists());
+ assert!(lock_path(&path).exists());
let query = seeded_unit_vector(1, DIMS);
assert!(matches!(
survivor.search(&query, &limit_params(3)),
@@ -3540,7 +3567,7 @@ mod tests {
assert!(!path.exists());
assert!(!snapshot_exists(&path));
assert!(!temp_sibling(&path).exists());
- assert!(!lock_path(&path).exists());
+ assert!(lock_path(&path).exists());
let query = seeded_unit_vector(1, DIMS);
assert!(matches!(
survivor.search(&query, &limit_params(3)),
@@ -3595,11 +3622,12 @@ mod tests {
assert!(!path.exists());
assert!(!snapshot_exists(&path));
assert!(!temp_sibling(&path).exists());
- assert!(!lock_path(&path).exists());
+ assert!(lock_path(&path).exists());
VecDb::purge(&path).unwrap();
+ assert!(lock_path(&path).exists());
let never_created = dir.path().join("never-created");
VecDb::purge(&never_created).unwrap();
- assert!(!lock_path(&never_created).exists());
+ assert!(lock_path(&never_created).exists());
let fresh = open_writer(&path);
assert!(fresh.is_empty());
fresh.add("fresh", &seeded_unit_vector(3, DIMS)).unwrap();
@@ -3621,7 +3649,7 @@ mod tests {
drop(held);
VecDb::purge(&path).unwrap();
assert!(!path.exists());
- assert!(!lock_path(&path).exists());
+ assert!(lock_path(&path).exists());
}
#[test]
@@ -3669,7 +3697,8 @@ mod tests {
VecDb::purge(&alias).unwrap();
assert!(!path.exists());
assert!(!snapshot_exists(&path));
- assert!(!lock_path(&path).exists());
+ assert!(lock_path(&path).exists());
+ assert!(!lock_path(&alias).exists());
assert!(fs::symlink_metadata(&alias).is_ok());
}