π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Store vecdb indexes next to the ML database
laurenspriemfdb1f6fa1b14
diff --git a/rust/crates/photos/src/ml_store/fill.rs b/rust/crates/photos/src/ml_store/fill.rsindex ca5e1c1aca..82adceff1f 100644--- a/rust/crates/photos/src/ml_store/fill.rs+++ b/rust/crates/photos/src/ml_store/fill.rs@@ -219,7 +219,7 @@ mod tests {use crate::ml_db::{ClipEmbedding, ClusterSummary, MlDb};use crate::ml_store::tests::{centroid, clips, empty, hot_position, index_path, live_count, lose_index_files, meta,- nearest, one_hot, open, open_in, open_with_unusable_index_dir,+ nearest, one_hot, open, open_in, open_with_unusable_clip_index,};use crate::ml_store::{Error, FillState, Index, MlStore, Result};@@ -418,7 +418,7 @@ mod tests {#[test]fn failed_fill_keeps_the_filling_state_and_cursor() {- let (_directory, store) = open_with_unusable_index_dir();+ let (_directory, store) = open_with_unusable_clip_index();store.db().insert_clip_rows(&clips(1..=5)).unwrap();store.db().set_meta("clip.fill", "filling").unwrap();store.db().set_meta("clip.cursor", "3").unwrap();diff --git a/rust/crates/photos/src/ml_store/index.rs b/rust/crates/photos/src/ml_store/index.rsindex ee9eccf7f6..8873ed36ac 100644--- a/rust/crates/photos/src/ml_store/index.rs+++ b/rust/crates/photos/src/ml_store/index.rs@@ -119,10 +119,10 @@ pub(super) struct Slot {}impl Slot {- pub(super) fn new(index: Index, index_dir: &Path, db_stem: &str) -> Self {+ pub(super) fn new(index: Index, db_path: &Path, db_stem: &str) -> Self {Self {index,- path: index_dir.join(format!("{db_stem}.vecdb.{}", index.name())),+ path: db_path.with_file_name(format!("{db_stem}.vecdb.{}", index.name())),handle: RwLock::new(None),}}@@ -234,18 +234,14 @@ mod tests {#[test]fn index_files_are_named_after_the_database_stem() {let directory = tempfile::tempdir().unwrap();- let store = MlStore::open(- directory.path().join("ente.ml.offline.db"),- directory.path(),- )- .unwrap();+ let store = MlStore::open(directory.path().join("ente.ml.offline.db")).unwrap();store.fill_clip_index(false).unwrap();assert_eq!(VecDb::open_cost(&directory.path().join("ente.ml.offline.vecdb.clip")),OpenCost::Ready);assert!(matches!(- MlStore::open(directory.path().join(".."), directory.path()),+ MlStore::open(directory.path().join("..")),Err(Error::InvalidArgument(_))));}diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rsindex f5894d180f..beb7f842ba 100644--- a/rust/crates/photos/src/ml_store/mod.rs+++ b/rust/crates/photos/src/ml_store/mod.rs@@ -40,14 +40,14 @@ pub struct MlStore {}impl MlStore {- pub fn open(db_path: impl AsRef<Path>, index_dir: impl AsRef<Path>) -> Result<Self> {+ pub fn open(db_path: impl AsRef<Path>) -> Result<Self> {let db_path = db_path.as_ref();let db_stem = index::db_stem(db_path)?;let db = MlDb::open(db_path)?;Ok(Self {db,locks: lock::for_database(db_path),- indexes: Index::ALL.map(|index| index::Slot::new(index, index_dir.as_ref(), &db_stem)),+ indexes: Index::ALL.map(|index| index::Slot::new(index, db_path, &db_stem)),})}@@ -162,14 +162,12 @@ mod tests {}pub(super) fn open_in(directory: &TempDir) -> MlStore {- MlStore::open(directory.path().join(DB_FILE), directory.path()).unwrap()+ MlStore::open(directory.path().join(DB_FILE)).unwrap()}- pub(super) fn open_with_unusable_index_dir() -> (TempDir, MlStore) {- let directory = tempfile::tempdir().unwrap();- let blocker = directory.path().join("indexes");- fs::write(&blocker, b"not a directory").unwrap();- let store = MlStore::open(directory.path().join(DB_FILE), &blocker).unwrap();+ pub(super) fn open_with_unusable_clip_index() -> (TempDir, MlStore) {+ let (directory, store) = open();+ fs::create_dir(index_path(&directory, Index::Clip)).unwrap();(directory, store)}diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rsindex 5720c93316..191ceace88 100644--- a/rust/crates/photos/src/ml_store/writes.rs+++ b/rust/crates/photos/src/ml_store/writes.rs@@ -222,7 +222,7 @@ mod tests {use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClusterSummary};use crate::ml_store::tests::{DB_FILE, centroid, clips, empty, hot_position, index_path, live_count, meta, nearest,- one_hot, open, open_with_unusable_index_dir, pet,+ one_hot, open, open_with_unusable_clip_index, pet,};use crate::ml_store::{Error, FillOutcome, FillState, Index, Species};@@ -531,7 +531,7 @@ mod tests {#[test]fn index_write_failure_marks_the_index_stale() {- let (_directory, store) = open_with_unusable_index_dir();+ let (_directory, store) = open_with_unusable_clip_index();store.db().set_meta("clip.fill", "filled").unwrap();assert!(matches!(store.put_clip(&clips([1])), Err(Error::Index(_))));assert_eq!(store.db().count_clip_rows().unwrap(), 1);