Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Store vecdb indexes next to the ML database

laurenspriemfdb1f6fa1b14
diff --git a/rust/crates/photos/src/ml_store/fill.rs b/rust/crates/photos/src/ml_store/fill.rs
index ca5e1c1aca..82adceff1f 100644
--- a/rust/crates/photos/src/ml_store/fill.rs
+++ b/rust/crates/photos/src/ml_store/fill.rs
@@ -219,7 +219,7 @@ mod tests {
use crate::ml_db::{ClipEmbedding, ClusterSummary, MlDb};
use crate::ml_store::tests::{
centroid, clips, empty, hot_position, index_path, live_count, lose_index_files, meta,
- nearest, one_hot, open, open_in, open_with_unusable_index_dir,
+ nearest, one_hot, open, open_in, open_with_unusable_clip_index,
};
use crate::ml_store::{Error, FillState, Index, MlStore, Result};
@@ -418,7 +418,7 @@ mod tests {
#[test]
fn failed_fill_keeps_the_filling_state_and_cursor() {
- let (_directory, store) = open_with_unusable_index_dir();
+ let (_directory, store) = open_with_unusable_clip_index();
store.db().insert_clip_rows(&clips(1..=5)).unwrap();
store.db().set_meta("clip.fill", "filling").unwrap();
store.db().set_meta("clip.cursor", "3").unwrap();
diff --git a/rust/crates/photos/src/ml_store/index.rs b/rust/crates/photos/src/ml_store/index.rs
index ee9eccf7f6..8873ed36ac 100644
--- a/rust/crates/photos/src/ml_store/index.rs
+++ b/rust/crates/photos/src/ml_store/index.rs
@@ -119,10 +119,10 @@ pub(super) struct Slot {
}
impl Slot {
- pub(super) fn new(index: Index, index_dir: &Path, db_stem: &str) -> Self {
+ pub(super) fn new(index: Index, db_path: &Path, db_stem: &str) -> Self {
Self {
index,
- path: index_dir.join(format!("{db_stem}.vecdb.{}", index.name())),
+ path: db_path.with_file_name(format!("{db_stem}.vecdb.{}", index.name())),
handle: RwLock::new(None),
}
}
@@ -234,18 +234,14 @@ mod tests {
#[test]
fn index_files_are_named_after_the_database_stem() {
let directory = tempfile::tempdir().unwrap();
- let store = MlStore::open(
- directory.path().join("ente.ml.offline.db"),
- directory.path(),
- )
- .unwrap();
+ let store = MlStore::open(directory.path().join("ente.ml.offline.db")).unwrap();
store.fill_clip_index(false).unwrap();
assert_eq!(
VecDb::open_cost(&directory.path().join("ente.ml.offline.vecdb.clip")),
OpenCost::Ready
);
assert!(matches!(
- MlStore::open(directory.path().join(".."), directory.path()),
+ MlStore::open(directory.path().join("..")),
Err(Error::InvalidArgument(_))
));
}
diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rs
index f5894d180f..beb7f842ba 100644
--- a/rust/crates/photos/src/ml_store/mod.rs
+++ b/rust/crates/photos/src/ml_store/mod.rs
@@ -40,14 +40,14 @@ pub struct MlStore {
}
impl MlStore {
- pub fn open(db_path: impl AsRef<Path>, index_dir: impl AsRef<Path>) -> Result<Self> {
+ pub fn open(db_path: impl AsRef<Path>) -> Result<Self> {
let db_path = db_path.as_ref();
let db_stem = index::db_stem(db_path)?;
let db = MlDb::open(db_path)?;
Ok(Self {
db,
locks: lock::for_database(db_path),
- indexes: Index::ALL.map(|index| index::Slot::new(index, index_dir.as_ref(), &db_stem)),
+ indexes: Index::ALL.map(|index| index::Slot::new(index, db_path, &db_stem)),
})
}
@@ -162,14 +162,12 @@ mod tests {
}
pub(super) fn open_in(directory: &TempDir) -> MlStore {
- MlStore::open(directory.path().join(DB_FILE), directory.path()).unwrap()
+ MlStore::open(directory.path().join(DB_FILE)).unwrap()
}
- pub(super) fn open_with_unusable_index_dir() -> (TempDir, MlStore) {
- let directory = tempfile::tempdir().unwrap();
- let blocker = directory.path().join("indexes");
- fs::write(&blocker, b"not a directory").unwrap();
- let store = MlStore::open(directory.path().join(DB_FILE), &blocker).unwrap();
+ pub(super) fn open_with_unusable_clip_index() -> (TempDir, MlStore) {
+ let (directory, store) = open();
+ fs::create_dir(index_path(&directory, Index::Clip)).unwrap();
(directory, store)
}
diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rs
index 5720c93316..191ceace88 100644
--- a/rust/crates/photos/src/ml_store/writes.rs
+++ b/rust/crates/photos/src/ml_store/writes.rs
@@ -222,7 +222,7 @@ mod tests {
use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClusterSummary};
use crate::ml_store::tests::{
DB_FILE, centroid, clips, empty, hot_position, index_path, live_count, meta, nearest,
- one_hot, open, open_with_unusable_index_dir, pet,
+ one_hot, open, open_with_unusable_clip_index, pet,
};
use crate::ml_store::{Error, FillOutcome, FillState, Index, Species};
@@ -531,7 +531,7 @@ mod tests {
#[test]
fn index_write_failure_marks_the_index_stale() {
- let (_directory, store) = open_with_unusable_index_dir();
+ let (_directory, store) = open_with_unusable_clip_index();
store.db().set_meta("clip.fill", "filled").unwrap();
assert!(matches!(store.put_clip(&clips([1])), Err(Error::Index(_))));
assert_eq!(store.db().count_clip_rows().unwrap(), 1);