π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Purge corrupt indexes and skip keys vecdb rejects
laurensprieme41596ca2270
diff --git a/rust/crates/photos/src/ml_store/index.rs b/rust/crates/photos/src/ml_store/index.rsindex 8c3c86f76b..ee9eccf7f6 100644--- a/rust/crates/photos/src/ml_store/index.rs+++ b/rust/crates/photos/src/ml_store/index.rs@@ -3,7 +3,7 @@ use std::io::ErrorKind;use std::path::{Path, PathBuf};use std::sync::{PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};-use ente_vecdb::{VecDb, VecDbError};+use ente_vecdb::{VecDb, VecDbError, validate_key};use super::{Error, IndexResult, Result, state};use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, MlDb};@@ -78,6 +78,10 @@ impl Index {}pub(super) fn accepts(self, key: &str, vector: &[f32]) -> bool {+ if let Err(error) = validate_key(key) {+ log::warn!("skipping {} vector {key:?}: {error}", self.name());+ return false;+ }if vector.is_empty() {return false;}@@ -164,7 +168,7 @@ impl Slot {state::invalidate_lost_index(db, self.index, &self.path)?;}match VecDb::open(&self.path, self.index.dims(), None) {- Err(error) if is_header_mismatch(&error) => {+ Err(error) if is_unusable(&error) => {log::warn!("{}: {error}; purging the index", self.path.display());state::mark_stale(db, self.index)?;VecDb::purge(&self.path)?;@@ -187,10 +191,12 @@ impl Slot {}}-fn is_header_mismatch(error: &VecDbError) -> bool {+fn is_unusable(error: &VecDbError) -> bool {matches!(error,- VecDbError::DimensionMismatch { .. } | VecDbError::StorageMismatch { .. }+ VecDbError::Corrupt(_)+ | VecDbError::DimensionMismatch { .. }+ | VecDbError::StorageMismatch { .. })}@@ -203,15 +209,20 @@ fn unless_closed<T>(outcome: IndexResult<T>) -> Option<Result<T>> {#[cfg(test)]mod tests {+ use std::collections::HashMap;+ use std::fs;+ use std::path::Path;+use ente_vecdb::{OpenCost, StorageKind, VecDb};use super::Index;use crate::db::Connection;use crate::ml_db::CLIP_EMBEDDING_DIMENSIONS;use crate::ml_store::tests::{- DB_FILE, clips, index_path, live_count, lose_index_files, meta, one_hot, open,+ DB_FILE, centroid, clips, index_path, live_count, lose_index_files, meta, one_hot, open,+ pet,};- use crate::ml_store::{Error, FillOutcome, FillState, MlStore};+ use crate::ml_store::{Error, FillOutcome, FillReport, FillState, MlStore, Species};#[test]fn index_positions_follow_the_all_order() {@@ -260,6 +271,79 @@ mod tests {assert_eq!(live_count(&store, Index::Clip), 0);}+ fn corrupt_header(path: &Path) {+ let mut bytes = fs::read(path).unwrap();+ for byte in &mut bytes[..4] {+ *byte ^= 0xFF;+ }+ fs::write(path, bytes).unwrap();+ }++ fn corrupt_log_body(path: &Path) {+ let mut bytes = fs::read(path).unwrap();+ let middle = bytes.len() / 2;+ for byte in &mut bytes[middle..middle + 64] {+ *byte ^= 0x5A;+ }+ fs::write(path, bytes).unwrap();+ }++ #[test]+ fn corrupt_index_files_are_purged_and_marked_stale() {+ let corruptions: [fn(&Path); 2] = [corrupt_header, corrupt_log_body];+ for corrupt in corruptions {+ let (directory, store) = open();+ store.db().insert_clip_rows(&clips(1..=50)).unwrap();+ store.fill_clip_index(false).unwrap();+ store.release().unwrap();+ corrupt(&index_path(&directory, Index::Clip));++ assert!(!store.contains(Index::Clip, "1").unwrap());+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);+ assert_eq!(store.fill_clip_index(false).unwrap().indexed, 50);+ assert!(store.contains(Index::Clip, "1").unwrap());+ assert_eq!(live_count(&store, Index::Clip), 50);+ }+ }++ #[test]+ fn keys_vecdb_rejects_are_skipped_like_malformed_vectors() {+ let (_directory, store) = open();+ store.fill_cluster_centroid_index(false).unwrap();+ let long_id = "k".repeat(257);+ store+ .cluster_summary_update(&HashMap::from([+ centroid("", 1),+ centroid(&long_id, 2),+ centroid("c3", 3),+ ]))+ .unwrap();+ assert_eq!(store.db().count_cluster_summaries().unwrap(), 3);+ assert_eq!(live_count(&store, Index::ClusterCentroid), 1);+ assert!(store.contains(Index::ClusterCentroid, "c3").unwrap());+ assert_eq!(+ store.fill_cluster_centroid_index(true).unwrap(),+ FillReport {+ outcome: FillOutcome::Completed,+ rows: 3,+ indexed: 1,+ skipped: 2,+ resumed: false+ }+ );+ assert_eq!(live_count(&store, Index::ClusterCentroid), 1);++ let dims = Index::PetFace(Species::Dog).dims();+ store+ .store_pet_face_embeddings(&[+ pet("", Species::Dog, 1, dims),+ pet("p", Species::Dog, 2, dims),+ ])+ .unwrap();+ assert_eq!(live_count(&store, Index::PetFace(Species::Dog)), 1);+ assert!(store.contains(Index::PetFace(Species::Dog), "p").unwrap());+ }+#[test]fn failed_invalidation_preserves_the_mismatched_index() {let (directory, store) = open();diff --git a/rust/crates/vecdb/src/arena.rs b/rust/crates/vecdb/src/arena.rsindex 12451e0159..7a35cb21bc 100644--- a/rust/crates/vecdb/src/arena.rs+++ b/rust/crates/vecdb/src/arena.rs@@ -16,7 +16,7 @@ pub(crate) enum UpsertOutcome {Unchanged(u32),}-pub(crate) fn validate_key(key: &str) -> Result<(), VecDbError> {+pub fn validate_key(key: &str) -> Result<(), VecDbError> {if key.is_empty() {return Err(VecDbError::InvalidKey("key is empty".to_string()));}diff --git a/rust/crates/vecdb/src/lib.rs b/rust/crates/vecdb/src/lib.rsindex 1a573a908f..9c2fd95bbe 100644--- a/rust/crates/vecdb/src/lib.rs+++ b/rust/crates/vecdb/src/lib.rs@@ -11,6 +11,7 @@ mod log;mod snapshot;mod store;+pub use arena::validate_key;pub use store::{OpenCost, Stats, VecDb};#[derive(Debug, Clone, Copy, PartialEq, Eq)]