π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: treat an int8 log payload outside the quantized range as an undecodable record
laurenspriem262b6c94efe1
diff --git a/rust/crates/ml/src/vecdb/kernel.rs b/rust/crates/ml/src/vecdb/kernel.rsindex e090dd0675..b59b6b1994 100644--- a/rust/crates/ml/src/vecdb/kernel.rs+++ b/rust/crates/ml/src/vecdb/kernel.rs@@ -6,6 +6,10 @@ const I8_LIMIT: f32 = 127.0;pub(crate) const MAX_DIMS_I8: usize =(i32::MAX / ((I8_LIMIT as i32) * (I8_LIMIT as i32))) as usize / LANE_WIDTH_I8 * LANE_WIDTH_I8;+pub(crate) fn is_within_quantized_range(value: i8) -> bool {+ i32::from(value).abs() <= I8_LIMIT as i32+}+#[repr(C, align(32))]#[derive(Clone, Copy)]pub(crate) struct Lane([f32; LANE_WIDTH]);@@ -264,6 +268,14 @@ mod tests {values}+ #[test]+ fn the_quantized_range_excludes_only_the_int8_minimum() {+ assert!(is_within_quantized_range(I8_LIMIT as i8));+ assert!(is_within_quantized_range(-(I8_LIMIT as i8)));+ assert!(is_within_quantized_range(0));+ assert!(!is_within_quantized_range(i8::MIN));+ }+#[test]fn dot_matches_scalar_reference_bit_exactly() {for (dims, seed) in [(128, 11), (192, 22), (512, 33)] {diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rsindex 8a42ee38e8..46156ba2c9 100644--- a/rust/crates/ml/src/vecdb/log.rs+++ b/rust/crates/ml/src/vecdb/log.rs@@ -6,7 +6,9 @@ use std::time::{SystemTime, UNIX_EPOCH};use super::arena::{MAX_KEY_BYTES, validate_key};use super::crc::{Crc32, crc32};-use super::kernel::{StoredVector, VectorPayload, splitmix64, validate_dims};+use super::kernel::{+ StoredVector, VectorPayload, is_within_quantized_range, splitmix64, validate_dims,+};use super::{AttrValue, Attribute, StorageKind, VecDbError};pub(crate) const HEADER_LEN: usize = 32;@@ -527,35 +529,39 @@ fn decode_body(return Some(LogRecord::Tombstone { key });}let (payload, attr_bytes) = rest.split_at(vector_payload_len(storage, dims));- let vector = decode_vector(storage, payload);+ let vector = decode_vector(storage, payload)?;let attrs = decode_attrs(attr_bytes)?;Some(LogRecord::Add { key, vector, attrs })}-fn decode_vector(storage: StorageKind, payload: &[u8]) -> StoredVector {+fn decode_vector(storage: StorageKind, payload: &[u8]) -> Option<StoredVector> {match storage {- StorageKind::F32 => StoredVector::F32(+ StorageKind::F32 => Some(StoredVector::F32(payload.as_chunks::<4>().0.iter().map(|bytes| f32::from_le_bytes(*bytes)).collect(),- ),+ )),StorageKind::I8 => {let (scale_bytes, values) = payload.split_at(size_of::<f32>());- StoredVector::I8 {+ let values: Vec<i8> = values+ .iter()+ .map(|&byte| i8::from_le_bytes([byte]))+ .collect();+ if !values.iter().all(|&value| is_within_quantized_range(value)) {+ return None;+ }+ Some(StoredVector::I8 {scale: f32::from_le_bytes([scale_bytes[0],scale_bytes[1],scale_bytes[2],scale_bytes[3],]),- values: values- .iter()- .map(|&byte| i8::from_le_bytes([byte]))- .collect(),- }+ values,+ })}}}@@ -2544,6 +2550,39 @@ mod tests {bytes}+ #[test]+ fn scanner_stops_at_an_int8_payload_outside_the_quantized_range() {+ let dims = 32usize;+ let scale = 0.25f32;+ let within_range = vec![3i8; dims];+ let mut beyond_range = vec![5i8; dims];+ beyond_range[7] = i8::MIN;+ let first = encoded_i8_add("first", scale, &within_range);+ let rejected = encoded_i8_add("rejected", scale, &beyond_range);+ let survivor = encoded_i8_add("survivor", scale, &within_range);+ let dir = TempDir::new().unwrap();+ let cases = [+ ("mid-log", [rejected.clone(), survivor].concat(), true),+ ("tail", rejected, false),+ ];+ for (name, after_first, intact_record_beyond) in cases {+ let path = dir.path().join(name);+ let mut bytes = encode_header(dims as u32, &[9u8; 16], StorageKind::I8).to_vec();+ bytes.extend_from_slice(&first);+ bytes.extend_from_slice(&after_first);+ std::fs::write(&path, &bytes).unwrap();+ let mut log = reopen(&path, dims);+ let (records, recoverable_end) = scan_all(&mut log);+ assert_eq!(records.len(), 1);+ assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "first"));+ assert_eq!(recoverable_end, HEADER_LEN as u64 + first.len() as u64);+ assert_eq!(+ log.tail_contains_valid_record(recoverable_end).unwrap(),+ intact_record_beyond+ );+ }+ }+fn write_i8_log_file(path: &Path, dims: u32, record_bytes: &[u8]) {let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::I8).to_vec();bytes.extend_from_slice(record_bytes);