Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Open the index before reporting its fill state

laurenspriem44bca540b902
diff --git a/mobile/apps/photos/lib/db/ml/rust_db.dart b/mobile/apps/photos/lib/db/ml/rust_db.dart
index 22ae2c7e48..af82f9f521 100644
--- a/mobile/apps/photos/lib/db/ml/rust_db.dart
+++ b/mobile/apps/photos/lib/db/ml/rust_db.dart
@@ -63,14 +63,8 @@ class RustMLDataDB implements IMLDataDB<int> {
}
}
- Future<rust.FillState> fillState(rust.Index index) async {
- final db = await store;
- await _openIndex(db, index);
- return db.fillState(index: index);
- }
-
- Future<void> _openIndex(rust.MlStore db, rust.Index index) =>
- db.stats(index: index);
+ Future<rust.FillState> fillState(rust.Index index) async =>
+ (await store).fillState(index: index);
Future<void> releaseVectorIndexes() async {
if (_dbFuture == null) return;
diff --git a/rust/crates/photos/src/ml_store/fill.rs b/rust/crates/photos/src/ml_store/fill.rs
index 5153380501..0d1ed37343 100644
--- a/rust/crates/photos/src/ml_store/fill.rs
+++ b/rust/crates/photos/src/ml_store/fill.rs
@@ -425,7 +425,11 @@ mod tests {
store.db().set_meta("clip.fill", "filling").unwrap();
store.db().set_meta("clip.cursor", "3").unwrap();
assert!(matches!(store.fill_clip_index(false), Err(Error::Index(_))));
- assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filling);
+ assert!(matches!(
+ store.fill_state(Index::Clip),
+ Err(Error::Index(_))
+ ));
+ assert_eq!(meta(&store, "clip.fill").as_deref(), Some("filling"));
assert_eq!(meta(&store, "clip.cursor").as_deref(), Some("3"));
}
diff --git a/rust/crates/photos/src/ml_store/index.rs b/rust/crates/photos/src/ml_store/index.rs
index efe4642130..949525eb5c 100644
--- a/rust/crates/photos/src/ml_store/index.rs
+++ b/rust/crates/photos/src/ml_store/index.rs
@@ -107,6 +107,20 @@ impl Slot {
operation(vecdb).map_err(Into::into)
}
+ pub(super) fn is_open(&self) -> bool {
+ self.read_handle().as_ref().is_some_and(is_live)
+ }
+
+ pub(super) fn reopen(&self, db: &MlDb) -> Result<()> {
+ let mut handle = self.write_handle();
+ if handle.as_ref().is_some_and(is_live) {
+ return Ok(());
+ }
+ *handle = None;
+ *handle = Some(self.open(db)?);
+ Ok(())
+ }
+
pub(super) fn release(&self) -> Result<()> {
match self.write_handle().take() {
Some(vecdb) => vecdb.flush().map_err(Into::into),
@@ -170,6 +184,10 @@ fn is_unusable(error: &VecDbError) -> bool {
)
}
+fn is_live(vecdb: &VecDb) -> bool {
+ !vecdb.is_closed()
+}
+
fn unless_closed<T>(outcome: IndexResult<T>) -> Option<Result<T>> {
match outcome {
Err(VecDbError::Closed) => None,
@@ -378,8 +396,12 @@ mod tests {
store.contains(Index::Clip, "7"),
Err(Error::Database(_))
));
+ assert!(matches!(
+ store.fill_state(Index::Clip),
+ Err(Error::Database(_))
+ ));
assert!(foreign.contains("7").unwrap());
- assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filled);
+ assert_eq!(meta(&store, "clip.fill").as_deref(), Some("filled"));
}
#[test]
@@ -482,4 +504,45 @@ mod tests {
assert_eq!(live_count(&store, Index::Clip), 3);
assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filled);
}
+
+ #[test]
+ fn fill_state_alone_detects_lost_index_files() {
+ let (directory, store) = open();
+ store.db().insert_clip_rows(&clips(1..=3)).unwrap();
+ store.fill_clip_index(false).unwrap();
+ lose_index_files(&store, &index_path(&directory, Index::Clip));
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);
+ assert_eq!(meta(&store, "clip.fill"), None);
+
+ store.fill_clip_index(false).unwrap();
+ VecDb::purge(&index_path(&directory, Index::Clip)).unwrap();
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);
+ assert_eq!(meta(&store, "clip.fill"), None);
+ assert_eq!(live_count(&store, Index::Clip), 0);
+ }
+
+ #[test]
+ fn fill_state_alone_purges_an_unusable_index() {
+ let (directory, store) = open();
+ store.db().insert_clip_rows(&clips(1..=3)).unwrap();
+ store.fill_clip_index(false).unwrap();
+ store.release().unwrap();
+ corrupt_header(&index_path(&directory, Index::Clip));
+
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);
+ assert_eq!(meta(&store, "clip.fill"), None);
+ assert_eq!(live_count(&store, Index::Clip), 0);
+ }
+
+ #[test]
+ fn fill_state_keeps_a_live_handle_open() {
+ let (_directory, store) = open();
+ store.fill_clip_index(false).unwrap();
+ store.put_clip(&clips([1, 2])).unwrap();
+ let unflushed = store.stats(Index::Clip).unwrap();
+ assert_ne!(unflushed.records_since_snapshot, 0);
+
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filled);
+ assert_eq!(store.stats(Index::Clip).unwrap(), unflushed);
+ }
}
diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rs
index 73243005f6..b63f879c50 100644
--- a/rust/crates/photos/src/ml_store/mod.rs
+++ b/rust/crates/photos/src/ml_store/mod.rs
@@ -61,6 +61,7 @@ impl MlStore {
}
pub fn fill_state(&self, index: Index) -> Result<FillState> {
+ self.ensure_index_open(index)?;
state::read(&self.db, index)
}
@@ -124,6 +125,15 @@ impl MlStore {
self.indexes[index.position()].with_open(&self.db, operation)
}
+ fn ensure_index_open(&self, index: Index) -> Result<()> {
+ let slot = &self.indexes[index.position()];
+ if slot.is_open() {
+ return Ok(());
+ }
+ let _mutations = self.lock_mutations();
+ slot.reopen(&self.db)
+ }
+
fn read_index<T>(
&self,
index: Index,
diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rs
index fab0d41247..f520d8fe97 100644
--- a/rust/crates/photos/src/ml_store/writes.rs
+++ b/rust/crates/photos/src/ml_store/writes.rs
@@ -546,7 +546,11 @@ mod tests {
store.db().set_meta("clip.fill", "filled").unwrap();
assert!(matches!(store.put_clip(&clips([1])), Err(Error::Index(_))));
assert_eq!(store.db().count_clip_rows().unwrap(), 1);
- assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);
+ assert!(matches!(
+ store.fill_state(Index::Clip),
+ Err(Error::Index(_))
+ ));
+ assert_eq!(meta(&store, "clip.fill"), None);
store.put_clip(&clips([2])).unwrap();
assert_eq!(store.db().count_clip_rows().unwrap(), 2);
}
diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rs
index 6aa3012352..d7b7442a78 100644
--- a/rust/crates/vecdb/src/store.rs
+++ b/rust/crates/vecdb/src/store.rs
@@ -954,6 +954,10 @@ impl VecDb {
Ok(self.open_state()?.arena.is_empty())
}
+ pub fn is_closed(&self) -> bool {
+ self.shared.is_closed()
+ }
+
fn open_state(&self) -> Result<RwLockReadGuard<'_, SearchState>, VecDbError> {
self.shared.ensure_open()?;
let st = self.shared.state_read();
@@ -2674,6 +2678,7 @@ mod tests {
}
fn assert_key_reads_open(db: &VecDb, key: &str, live: usize) {
+ assert!(!db.is_closed());
assert_eq!(db.len().unwrap(), live);
assert!(!db.is_empty().unwrap());
assert!(db.contains(key).unwrap());
@@ -2683,6 +2688,7 @@ mod tests {
}
fn assert_key_reads_closed(db: &VecDb) {
+ assert!(db.is_closed());
assert!(matches!(db.contains("key-0"), Err(VecDbError::Closed)));
assert!(matches!(db.get("key-0"), Err(VecDbError::Closed)));
assert!(matches!(db.get_attrs("key-0"), Err(VecDbError::Closed)));