π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Colocate Photos database tests with their modules
laurenspriem5d375ddf3853
diff --git a/rust/crates/photos/src/db/database.rs b/rust/crates/photos/src/db/database.rsindex 22289c0eea..f8d5d9d73c 100644--- a/rust/crates/photos/src/db/database.rs+++ b/rust/crates/photos/src/db/database.rs@@ -112,3 +112,74 @@ impl Database {})}}++#[cfg(test)]+mod tests {+ use super::Database;+ use crate::db::Error;++ const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";++ #[test]+ fn reader_queries_reject_writes() {+ let directory = tempfile::tempdir().unwrap();+ let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ db.execute("INSERT INTO items (id) VALUES (?)", [7])+ .unwrap();+ let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());+ assert!(matches!(deleted, Err(Error::Sqlite(_))));+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())+ .unwrap(),+ [7]+ );+ }++ #[test]+ fn failed_batch_rolls_back_and_writer_recovers() {+ let directory = tempfile::tempdir().unwrap();+ let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ assert!(+ db.write_batch("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]])+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ db.execute("INSERT INTO items (id) VALUES (?)", [3])+ .unwrap();+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())+ .unwrap(),+ [3]+ );+ }++ #[test]+ fn chunked_delete_handles_large_and_empty_id_lists() {+ let directory = tempfile::tempdir().unwrap();+ let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ let ids: Vec<i64> = (0..10_001).collect();+ db.write_batch(+ "INSERT INTO items (id) VALUES (?)",+ ids.iter().map(|id| [id]),+ )+ .unwrap();+ db.execute_chunked_in::<i64>("DELETE FROM items WHERE id IN ({})", &[])+ .unwrap();+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 10_001+ );+ db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)+ .unwrap();+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ }+}diff --git a/rust/crates/photos/src/db/migrate.rs b/rust/crates/photos/src/db/migrate.rsindex 1b656f6fb7..0f72dd5ccc 100644--- a/rust/crates/photos/src/db/migrate.rs+++ b/rust/crates/photos/src/db/migrate.rs@@ -28,3 +28,89 @@ fn check_not_downgrade(current: i64, target: i64) -> Result<()> {}Ok(())}++#[cfg(test)]+mod tests {+ use crate::db::{Connection, Database, Error};++ const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";+ const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";++ #[test]+ fn open_migrates_a_caller_schema_and_refuses_downgrade() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("items.db");+ {+ let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();+ db.execute("INSERT INTO items (id) VALUES (?)", [7])+ .unwrap();+ }+ {+ let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();+ assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);+ assert_eq!(+ db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [7])+ .unwrap(),+ Some("item".to_string())+ );+ assert_eq!(+ db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [8])+ .unwrap(),+ None+ );+ }+ let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 1+ );+ assert!(matches!(+ Database::open(&path, &[CREATE_ITEMS]),+ Err(Error::Downgrade {+ current: 2,+ target: 1+ })+ ));+ }++ #[test]+ fn failed_migration_keeps_previous_schema_and_version() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("items.db");+ {+ let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();+ db.execute("INSERT INTO items (id) VALUES (?)", [7])+ .unwrap();+ }+ assert!(+ Database::open(+ &path,+ &[+ CREATE_ITEMS,+ ADD_LABEL,+ "INSERT INTO missing_table VALUES (1)"+ ]+ )+ .is_err()+ );+ {+ let connection = Connection::open(&path).unwrap();+ let version: i64 = connection+ .pragma_query_value(None, "user_version", |row| row.get(0))+ .unwrap();+ assert_eq!(version, 1);+ let columns: Vec<String> = connection+ .prepare("SELECT name FROM pragma_table_info('items')")+ .unwrap()+ .query_map((), |row| row.get(0))+ .unwrap()+ .collect::<Result<_, _>>()+ .unwrap();+ assert_eq!(columns, ["id"]);+ }+ let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();+ assert_eq!(db.read_value::<i64>("SELECT id FROM items", ()).unwrap(), 7);+ assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);+ }+}diff --git a/rust/crates/photos/src/ml_db/caches.rs b/rust/crates/photos/src/ml_db/caches.rsindex 31462106f6..16876d4b5e 100644--- a/rust/crates/photos/src/ml_db/caches.rs+++ b/rust/crates/photos/src/ml_db/caches.rs@@ -82,3 +82,64 @@ fn now_millis() -> i64 {.map(|elapsed| elapsed.as_millis() as i64).unwrap_or(0)}++#[cfg(test)]+mod tests {+ use crate::db::Connection;+ use crate::ml_db::tests::{cases, check_seeded, seeded};++ #[test]+ fn seeded_optional_values() {+ check_seeded(&cases![+ "cached face p1": Some("1_0".to_string()) =>+ |db| db.get_face_id_used_for_person_or_cluster("p1"),+ "cached face p2": None => |db| db.get_face_id_used_for_person_or_cluster("p2"),+ ]);+ check_seeded(&cases![+ "cached dog": Some(vec![0.5, -1.0]) => |db| db.get_repeated_text_embedding_cache("dog"),+ "uncached cat": None => |db| db.get_repeated_text_embedding_cache("cat"),+ ]);+ }++ #[test]+ fn text_embedding_cache_replaces_and_expires() {+ let (directory, db) = seeded();+ db.put_repeated_text_embedding_cache("dog", &[2.0]).unwrap();+ assert_eq!(+ db.get_repeated_text_embedding_cache("dog").unwrap(),+ Some(vec![2.0f32])+ );++ let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();+ connection+ .execute(+ "UPDATE text_embeddings_cache SET created_at = 1 WHERE text_query = ?",+ ["dog"],+ )+ .unwrap();+ assert_eq!(db.get_repeated_text_embedding_cache("dog").unwrap(), None);+ let remaining: i64 = connection+ .query_row("SELECT COUNT(*) FROM text_embeddings_cache", (), |row| {+ row.get(0)+ })+ .unwrap();+ assert_eq!(remaining, 0);+ }++ #[test]+ fn face_cache_replaces_and_removes() {+ let (_directory, db) = seeded();+ db.put_face_id_cached_for_person_or_cluster("p1", "2_0")+ .unwrap();+ assert_eq!(+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),+ Some("2_0".to_string())+ );+ db.remove_face_id_cached_for_person_or_cluster("p1")+ .unwrap();+ assert_eq!(+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),+ None+ );+ }+}diff --git a/rust/crates/photos/src/ml_db/clip.rs b/rust/crates/photos/src/ml_db/clip.rsindex 2d0e2284da..fb96473ac3 100644--- a/rust/crates/photos/src/ml_db/clip.rs+++ b/rust/crates/photos/src/ml_db/clip.rs@@ -105,3 +105,88 @@ fn clip_row(embedding: &ClipEmbedding) -> (i64, Vec<u8>, i64) {embedding.version,)}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use super::ClipRow;+ use crate::ml_db::codec::encode_f32;+ use crate::ml_db::constants::CLIP_EMBEDDING_DIMENSIONS;+ use crate::ml_db::tests::{cases, check_seeded, clip, full_clip, seeded};++ #[test]+ fn seeded_counts() {+ check_seeded(&cases![+ "clip files": 4 => |db| db.get_clip_indexed_file_count(1),+ "clip files v2": 1 => |db| db.get_clip_indexed_file_count(2),+ "clip vectorizable": 2 => |db| db.get_clip_vectorizable_file_count(1),+ "clip vectorizable v2": 1 => |db| db.get_clip_vectorizable_file_count(2),+ "clip rows": 4 => |db| db.count_clip_rows(),+ ]);+ }++ #[test]+ fn seeded_versions() {+ check_seeded(&cases![+ "clip versions": HashMap::from([(1, 1), (2, 2), (3, 1), (4, 1)]) =>+ |db| db.clip_indexed_file_with_version(),+ ]);+ }++ #[test]+ fn seeded_rows() {+ let (_directory, db) = seeded();+ let mut vectors = db.get_all_clip_vectors().unwrap();+ vectors.sort_by_key(|vector| vector.file_id);+ let dimensions: Vec<(i64, usize)> = vectors+ .iter()+ .map(|v| (v.file_id, v.embedding.len()))+ .collect();+ assert_eq!(+ dimensions,+ vec![+ (1, CLIP_EMBEDDING_DIMENSIONS),+ (2, CLIP_EMBEDDING_DIMENSIONS),+ (3, 2)+ ]+ );+ assert_eq!(vectors[2].embedding, vec![1.0f32, 2.0]);+ let page = db.get_clip_rows_page(2, 1).unwrap();+ let clip_row = |file_id, embedding: Vec<f32>| ClipRow {+ file_id,+ embedding: encode_f32(embedding),+ };+ assert_eq!(+ page,+ vec![+ clip_row(3, vec![1.0, 2.0]),+ clip_row(2, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])+ ]+ );+ }++ #[test]+ fn clip_rows_replace_and_delete() {+ let (_directory, db) = seeded();+ db.insert_clip_rows(&[]).unwrap();+ db.insert_clip_rows(&[clip(1, vec![1.0])]).unwrap();+ db.insert_clip_rows(&[clip(3, vec![3.0]), full_clip(5)])+ .unwrap();+ assert_eq!(+ db.clip_indexed_file_with_version().unwrap(),+ HashMap::from([(1, 1), (2, 2), (3, 1), (4, 1), (5, 1)])+ );+ let mut vectors = db.get_all_clip_vectors().unwrap();+ vectors.sort_by_key(|vector| vector.file_id);+ assert_eq!(vectors[0].embedding, vec![1.0f32]);+ assert_eq!(vectors[2].embedding, vec![3.0f32]);+ assert_eq!(db.get_clip_vectorizable_file_count(1).unwrap(), 2);++ db.delete_clip_rows(&[]).unwrap();+ db.delete_clip_rows(&[1, 3]).unwrap();+ assert_eq!(db.count_clip_rows().unwrap(), 3);+ db.delete_all_clip_rows().unwrap();+ assert_eq!(db.count_clip_rows().unwrap(), 0);+ }+}diff --git a/rust/crates/photos/src/ml_db/filedata.rs b/rust/crates/photos/src/ml_db/filedata.rsindex ad1406755e..a6cc193e27 100644--- a/rust/crates/photos/src/ml_db/filedata.rs+++ b/rust/crates/photos/src/ml_db/filedata.rs@@ -55,3 +55,39 @@ impl MlDb {.map_err(Into::into)}}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use crate::ml_db::tests::{cases, check_seeded, ids, preview, seeded, status};++ #[test]+ fn seeded_file_id_sets() {+ check_seeded(&cases![+ "fd data": ids([1, 2, 3]) => |db| db.get_file_ids_with_fd_data(None),+ "fd mldata": ids([2]) => |db| db.get_file_ids_with_fd_data(Some("mldata")),+ ]);+ }++ #[test]+ fn seeded_video_previews() {+ check_seeded(&cases![+ "video previews": HashMap::from([(1, preview("obj1", 10)), (3, preview("obj3", 30))]) =>+ |db| db.get_file_ids_vid_preview(),+ ]);+ }++ #[test]+ fn filedata_upsert() {+ let (_directory, db) = seeded();+ db.put_fd_status(&[]).unwrap();+ db.put_fd_status(&[status(1, "vid_preview", Some("obj1b"))])+ .unwrap();+ assert_eq!(+ db.get_file_ids_vid_preview().unwrap(),+ HashMap::from([(1, preview("obj1b", 10)), (3, preview("obj3", 30))])+ );+ assert_eq!(db.get_file_ids_with_fd_data(None).unwrap(), ids([1, 2, 3]));+ }+}diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rsindex 4745f315b0..c7a044651f 100644--- a/rust/crates/photos/src/ml_db/mod.rs+++ b/rust/crates/photos/src/ml_db/mod.rs@@ -55,3 +55,235 @@ impl MlDb {.map_err(Into::into)}}++#[cfg(test)]+mod tests {+ use std::collections::HashSet;+ use std::fmt::Debug;+ use std::path::Path;++ use crate::db::Connection;+ use crate::ml_db::constants::CLIP_EMBEDDING_DIMENSIONS;+ use crate::ml_db::schema::ALL_TABLES;+ use crate::ml_db::{ClipEmbedding, Error, FdStatus, MlDb, PreviewInfo, TARGET_VERSION};+ use tempfile::TempDir;++ type Query<T> = fn(&MlDb) -> Result<T, Error>;++ macro_rules! cases {+ ($($name:literal: $expected:expr => |$db:ident| $query:expr),* $(,)?) => {+ [$(($name, $expected, |$db| $query)),*]+ };+ }++ pub(super) use cases;++ fn open() -> (TempDir, MlDb) {+ let directory = tempfile::tempdir().unwrap();+ let db = MlDb::open(directory.path().join("ente.ml.db")).unwrap();+ (directory, db)+ }++ fn user_version(path: &Path) -> i64 {+ Connection::open(path)+ .unwrap()+ .pragma_query_value(None, "user_version", |row| row.get(0))+ .unwrap()+ }++ fn index_count(path: &Path) -> i64 {+ Connection::open(path)+ .unwrap()+ .query_row(+ "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_fcClusterID'",+ (),+ |row| row.get(0),+ )+ .unwrap()+ }++ pub(super) fn clip(file_id: i64, embedding: Vec<f64>) -> ClipEmbedding {+ ClipEmbedding {+ file_id,+ embedding,+ version: 1,+ }+ }++ pub(super) fn full_clip(file_id: i64) -> ClipEmbedding {+ clip(file_id, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])+ }++ pub(super) fn status(file_id: i64, data_type: &str, object_id: Option<&str>) -> FdStatus {+ FdStatus {+ file_id,+ user_id: 42,+ data_type: data_type.to_string(),+ size: file_id * 10,+ object_id: object_id.map(str::to_string),+ object_nonce: None,+ updated_at: 1000,+ }+ }++ pub(super) fn preview(object_id: &str, object_size: i64) -> PreviewInfo {+ PreviewInfo {+ object_id: object_id.to_string(),+ object_size,+ }+ }++ pub(super) fn ids<const N: usize>(values: [i64; N]) -> HashSet<i64> {+ HashSet::from(values)+ }++ pub(super) fn check_seeded<T: PartialEq + Debug>(cases: &[(&str, T, Query<T>)]) {+ let (_directory, db) = seeded();+ for (name, expected, query) in cases {+ assert_eq!(&query(&db).unwrap(), expected, "{name}");+ }+ }++ pub(super) fn seeded() -> (TempDir, MlDb) {+ let (directory, db) = open();+ db.insert_clip_rows(&[+ full_clip(1),+ ClipEmbedding {+ version: 2,+ ..full_clip(2)+ },+ clip(3, vec![1.0, 2.0]),+ clip(4, vec![]),+ ])+ .unwrap();+ db.put_fd_status(&[+ status(1, "vid_preview", Some("obj1")),+ status(2, "mldata", None),+ status(3, "vid_preview", Some("obj3")),+ ])+ .unwrap();+ db.put_repeated_text_embedding_cache("dog", &[0.5, -1.0])+ .unwrap();+ db.put_face_id_cached_for_person_or_cluster("p1", "1_0")+ .unwrap();+ (directory, db)+ }++ #[test]+ fn ml_db_is_send_and_sync() {+ fn assert_send_sync<T: Send + Sync>() {}+ assert_send_sync::<MlDb>();+ }++ #[test]+ fn open_migrates_to_target_version_and_creates_all_tables() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("ente.ml.db");+ let _db = MlDb::open(&path).unwrap();+ assert_eq!(TARGET_VERSION, 15);+ assert_eq!(user_version(&path), 15);+ let connection = Connection::open(&path).unwrap();+ for table in ALL_TABLES {+ let count: i64 = connection+ .query_row(+ "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?",+ [table],+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(count, 1, "missing table {table}");+ }+ assert_eq!(index_count(&path), 1);+ }++ #[test]+ fn reopen_is_idempotent_and_keeps_data() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("ente.ml.db");+ {+ let db = MlDb::open(&path).unwrap();+ db.insert_clip_rows(&[full_clip(1)]).unwrap();+ }+ let db = MlDb::open(&path).unwrap();+ assert_eq!(user_version(&path), 15);+ assert_eq!(db.count_clip_rows().unwrap(), 1);+ }++ #[test]+ fn open_refuses_downgrade() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("ente.ml.db");+ drop(MlDb::open(&path).unwrap());+ Connection::open(&path)+ .unwrap()+ .pragma_update(None, "user_version", 16)+ .unwrap();+ match MlDb::open(&path) {+ Err(Error::Database(crate::db::Error::Downgrade { current, target })) => {+ assert_eq!(current, 16);+ assert_eq!(target, 15);+ }+ other => panic!("expected downgrade error, got {:?}", other.err()),+ }+ }++ #[test]+ fn partial_migration_runs_remaining_scripts() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("ente.ml.db");+ {+ let connection = Connection::open(&path).unwrap();+ connection+ .execute_batch(crate::ml_db::schema::CREATE_FACES_TABLE)+ .unwrap();+ connection.pragma_update(None, "user_version", 1).unwrap();+ }+ let db = MlDb::open(&path).unwrap();+ assert_eq!(user_version(&path), 15);+ assert_eq!(db.count_clip_rows().unwrap(), 0);+ }++ #[test]+ fn clear_non_pet_tables_leaves_pets_and_caches() {+ let (_directory, db) = seeded();+ db.clear_non_pet_tables().unwrap();+ assert_eq!(db.count_clip_rows().unwrap(), 0);+ assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());+ assert_eq!(+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),+ Some("1_0".to_string())+ );+ assert_eq!(+ db.get_repeated_text_embedding_cache("dog").unwrap(),+ Some(vec![0.5f32, -1.0])+ );+ }++ #[test]+ fn clear_pet_tables_leaves_non_pet_tables() {+ let (directory, db) = seeded();+ let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();+ connection+ .execute_batch(+ "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');+ INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",+ )+ .unwrap();+ db.clear_pet_tables().unwrap();+ let row_count = |table: &str| -> i64 {+ connection+ .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {+ row.get(0)+ })+ .unwrap()+ };+ assert_eq!(row_count("pet_face_vector_id_map"), 0);+ assert_eq!(row_count("pet_body_vector_id_map"), 0);+ assert_eq!(db.count_clip_rows().unwrap(), 4);+ assert_eq!(db.get_file_ids_with_fd_data(None).unwrap(), ids([1, 2, 3]));+ assert_eq!(+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),+ Some("1_0".to_string())+ );+ }+}diff --git a/rust/crates/photos/tests/db_test.rs b/rust/crates/photos/tests/db_test.rsdeleted file mode 100644index f15e71e53d..0000000000--- a/rust/crates/photos/tests/db_test.rs+++ /dev/null@@ -1,145 +0,0 @@-use ente_photos::db::{Connection, Database, Error};--const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";-const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";--#[test]-fn open_migrates_a_caller_schema_and_refuses_downgrade() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("items.db");- {- let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();- db.execute("INSERT INTO items (id) VALUES (?)", [7])- .unwrap();- }- {- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();- assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);- assert_eq!(- db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [7])- .unwrap(),- Some("item".to_string())- );- assert_eq!(- db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [8])- .unwrap(),- None- );- }- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 1- );- assert!(matches!(- Database::open(&path, &[CREATE_ITEMS]),- Err(Error::Downgrade {- current: 2,- target: 1- })- ));-}--#[test]-fn failed_migration_keeps_previous_schema_and_version() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("items.db");- {- let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();- db.execute("INSERT INTO items (id) VALUES (?)", [7])- .unwrap();- }- assert!(- Database::open(- &path,- &[- CREATE_ITEMS,- ADD_LABEL,- "INSERT INTO missing_table VALUES (1)"- ]- )- .is_err()- );- {- let connection = Connection::open(&path).unwrap();- let version: i64 = connection- .pragma_query_value(None, "user_version", |row| row.get(0))- .unwrap();- assert_eq!(version, 1);- let columns: Vec<String> = connection- .prepare("SELECT name FROM pragma_table_info('items')")- .unwrap()- .query_map((), |row| row.get(0))- .unwrap()- .collect::<Result<_, _>>()- .unwrap();- assert_eq!(columns, ["id"]);- }- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();- assert_eq!(db.read_value::<i64>("SELECT id FROM items", ()).unwrap(), 7);- assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);-}--#[test]-fn reader_queries_reject_writes() {- let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();- db.execute("INSERT INTO items (id) VALUES (?)", [7])- .unwrap();- let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());- assert!(matches!(deleted, Err(Error::Sqlite(_))));- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())- .unwrap(),- [7]- );-}--#[test]-fn failed_batch_rolls_back_and_writer_recovers() {- let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();- assert!(- db.write_batch("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]])- .is_err()- );- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );- db.execute("INSERT INTO items (id) VALUES (?)", [3])- .unwrap();- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())- .unwrap(),- [3]- );-}--#[test]-fn chunked_delete_handles_large_and_empty_id_lists() {- let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();- let ids: Vec<i64> = (0..10_001).collect();- db.write_batch(- "INSERT INTO items (id) VALUES (?)",- ids.iter().map(|id| [id]),- )- .unwrap();- db.execute_chunked_in::<i64>("DELETE FROM items WHERE id IN ({})", &[])- .unwrap();- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 10_001- );- db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)- .unwrap();- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );-}diff --git a/rust/crates/photos/tests/ml_db_test.rs b/rust/crates/photos/tests/ml_db_test.rsdeleted file mode 100644index be1e6c88af..0000000000--- a/rust/crates/photos/tests/ml_db_test.rs+++ /dev/null@@ -1,384 +0,0 @@-use std::collections::{HashMap, HashSet};-use std::fmt::Debug;-use std::path::Path;--use ente_photos::db::Connection;-use ente_photos::ml_db::codec::encode_f32;-use ente_photos::ml_db::constants::CLIP_EMBEDDING_DIMENSIONS;-use ente_photos::ml_db::schema::ALL_TABLES;-use ente_photos::ml_db::{- ClipEmbedding, ClipRow, Error, FdStatus, MlDb, PreviewInfo, TARGET_VERSION,-};-use tempfile::TempDir;--type Query<T> = fn(&MlDb) -> Result<T, Error>;--macro_rules! cases {- ($($name:literal: $expected:expr => |$db:ident| $query:expr),* $(,)?) => {- [$(($name, $expected, |$db| $query)),*]- };-}--fn open() -> (TempDir, MlDb) {- let directory = tempfile::tempdir().unwrap();- let db = MlDb::open(directory.path().join("ente.ml.db")).unwrap();- (directory, db)-}--fn user_version(path: &Path) -> i64 {- Connection::open(path)- .unwrap()- .pragma_query_value(None, "user_version", |row| row.get(0))- .unwrap()-}--fn index_count(path: &Path) -> i64 {- Connection::open(path)- .unwrap()- .query_row(- "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_fcClusterID'",- (),- |row| row.get(0),- )- .unwrap()-}--fn clip(file_id: i64, embedding: Vec<f64>) -> ClipEmbedding {- ClipEmbedding {- file_id,- embedding,- version: 1,- }-}--fn full_clip(file_id: i64) -> ClipEmbedding {- clip(file_id, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])-}--fn status(file_id: i64, data_type: &str, object_id: Option<&str>) -> FdStatus {- FdStatus {- file_id,- user_id: 42,- data_type: data_type.to_string(),- size: file_id * 10,- object_id: object_id.map(str::to_string),- object_nonce: None,- updated_at: 1000,- }-}--fn preview(object_id: &str, object_size: i64) -> PreviewInfo {- PreviewInfo {- object_id: object_id.to_string(),- object_size,- }-}--fn ids<const N: usize>(values: [i64; N]) -> HashSet<i64> {- HashSet::from(values)-}--fn check_seeded<T: PartialEq + Debug>(cases: &[(&str, T, Query<T>)]) {- let (_directory, db) = seeded();- for (name, expected, query) in cases {- assert_eq!(&query(&db).unwrap(), expected, "{name}");- }-}--fn seeded() -> (TempDir, MlDb) {- let (directory, db) = open();- db.insert_clip_rows(&[- full_clip(1),- ClipEmbedding {- version: 2,- ..full_clip(2)- },- clip(3, vec![1.0, 2.0]),- clip(4, vec![]),- ])- .unwrap();- db.put_fd_status(&[- status(1, "vid_preview", Some("obj1")),- status(2, "mldata", None),- status(3, "vid_preview", Some("obj3")),- ])- .unwrap();- db.put_repeated_text_embedding_cache("dog", &[0.5, -1.0])- .unwrap();- db.put_face_id_cached_for_person_or_cluster("p1", "1_0")- .unwrap();- (directory, db)-}--#[test]-fn ml_db_is_send_and_sync() {- fn assert_send_sync<T: Send + Sync>() {}- assert_send_sync::<MlDb>();-}--#[test]-fn open_migrates_to_target_version_and_creates_all_tables() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("ente.ml.db");- let _db = MlDb::open(&path).unwrap();- assert_eq!(TARGET_VERSION, 15);- assert_eq!(user_version(&path), 15);- let connection = Connection::open(&path).unwrap();- for table in ALL_TABLES {- let count: i64 = connection- .query_row(- "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?",- [table],- |row| row.get(0),- )- .unwrap();- assert_eq!(count, 1, "missing table {table}");- }- assert_eq!(index_count(&path), 1);-}--#[test]-fn reopen_is_idempotent_and_keeps_data() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("ente.ml.db");- {- let db = MlDb::open(&path).unwrap();- db.insert_clip_rows(&[full_clip(1)]).unwrap();- }- let db = MlDb::open(&path).unwrap();- assert_eq!(user_version(&path), 15);- assert_eq!(db.count_clip_rows().unwrap(), 1);-}--#[test]-fn open_refuses_downgrade() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("ente.ml.db");- drop(MlDb::open(&path).unwrap());- Connection::open(&path)- .unwrap()- .pragma_update(None, "user_version", 16)- .unwrap();- match MlDb::open(&path) {- Err(Error::Database(ente_photos::db::Error::Downgrade { current, target })) => {- assert_eq!(current, 16);- assert_eq!(target, 15);- }- other => panic!("expected downgrade error, got {:?}", other.err()),- }-}--#[test]-fn partial_migration_runs_remaining_scripts() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("ente.ml.db");- {- let connection = Connection::open(&path).unwrap();- connection- .execute_batch(ente_photos::ml_db::schema::CREATE_FACES_TABLE)- .unwrap();- connection.pragma_update(None, "user_version", 1).unwrap();- }- let db = MlDb::open(&path).unwrap();- assert_eq!(user_version(&path), 15);- assert_eq!(db.count_clip_rows().unwrap(), 0);-}--#[test]-fn seeded_counts() {- check_seeded(&cases![- "clip files": 4 => |db| db.get_clip_indexed_file_count(1),- "clip files v2": 1 => |db| db.get_clip_indexed_file_count(2),- "clip vectorizable": 2 => |db| db.get_clip_vectorizable_file_count(1),- "clip vectorizable v2": 1 => |db| db.get_clip_vectorizable_file_count(2),- "clip rows": 4 => |db| db.count_clip_rows(),- ]);-}--#[test]-fn seeded_file_id_sets() {- check_seeded(&cases![- "fd data": ids([1, 2, 3]) => |db| db.get_file_ids_with_fd_data(None),- "fd mldata": ids([2]) => |db| db.get_file_ids_with_fd_data(Some("mldata")),- ]);-}--#[test]-fn seeded_optional_values() {- check_seeded(&cases![- "cached face p1": Some("1_0".to_string()) =>- |db| db.get_face_id_used_for_person_or_cluster("p1"),- "cached face p2": None => |db| db.get_face_id_used_for_person_or_cluster("p2"),- ]);- check_seeded(&cases![- "cached dog": Some(vec![0.5, -1.0]) => |db| db.get_repeated_text_embedding_cache("dog"),- "uncached cat": None => |db| db.get_repeated_text_embedding_cache("cat"),- ]);-}--#[test]-fn seeded_maps() {- check_seeded(&cases![- "clip versions": HashMap::from([(1, 1), (2, 2), (3, 1), (4, 1)]) =>- |db| db.clip_indexed_file_with_version(),- ]);- check_seeded(&cases![- "video previews": HashMap::from([(1, preview("obj1", 10)), (3, preview("obj3", 30))]) =>- |db| db.get_file_ids_vid_preview(),- ]);-}--#[test]-fn seeded_rows() {- let (_directory, db) = seeded();- let mut vectors = db.get_all_clip_vectors().unwrap();- vectors.sort_by_key(|vector| vector.file_id);- let dimensions: Vec<(i64, usize)> = vectors- .iter()- .map(|v| (v.file_id, v.embedding.len()))- .collect();- assert_eq!(- dimensions,- vec![- (1, CLIP_EMBEDDING_DIMENSIONS),- (2, CLIP_EMBEDDING_DIMENSIONS),- (3, 2)- ]- );- assert_eq!(vectors[2].embedding, vec![1.0f32, 2.0]);- let page = db.get_clip_rows_page(2, 1).unwrap();- let clip_row = |file_id, embedding: Vec<f32>| ClipRow {- file_id,- embedding: encode_f32(embedding),- };- assert_eq!(- page,- vec![- clip_row(3, vec![1.0, 2.0]),- clip_row(2, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])- ]- );-}--#[test]-fn clip_rows_replace_and_delete() {- let (_directory, db) = seeded();- db.insert_clip_rows(&[]).unwrap();- db.insert_clip_rows(&[clip(1, vec![1.0])]).unwrap();- db.insert_clip_rows(&[clip(3, vec![3.0]), full_clip(5)])- .unwrap();- assert_eq!(- db.clip_indexed_file_with_version().unwrap(),- HashMap::from([(1, 1), (2, 2), (3, 1), (4, 1), (5, 1)])- );- let mut vectors = db.get_all_clip_vectors().unwrap();- vectors.sort_by_key(|vector| vector.file_id);- assert_eq!(vectors[0].embedding, vec![1.0f32]);- assert_eq!(vectors[2].embedding, vec![3.0f32]);- assert_eq!(db.get_clip_vectorizable_file_count(1).unwrap(), 2);-- db.delete_clip_rows(&[]).unwrap();- db.delete_clip_rows(&[1, 3]).unwrap();- assert_eq!(db.count_clip_rows().unwrap(), 3);- db.delete_all_clip_rows().unwrap();- assert_eq!(db.count_clip_rows().unwrap(), 0);-}--#[test]-fn filedata_upsert() {- let (_directory, db) = seeded();- db.put_fd_status(&[]).unwrap();- db.put_fd_status(&[status(1, "vid_preview", Some("obj1b"))])- .unwrap();- assert_eq!(- db.get_file_ids_vid_preview().unwrap(),- HashMap::from([(1, preview("obj1b", 10)), (3, preview("obj3", 30))])- );- assert_eq!(db.get_file_ids_with_fd_data(None).unwrap(), ids([1, 2, 3]));-}--#[test]-fn text_embedding_cache_replaces_and_expires() {- let (directory, db) = seeded();- db.put_repeated_text_embedding_cache("dog", &[2.0]).unwrap();- assert_eq!(- db.get_repeated_text_embedding_cache("dog").unwrap(),- Some(vec![2.0f32])- );-- let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();- connection- .execute(- "UPDATE text_embeddings_cache SET created_at = 1 WHERE text_query = ?",- ["dog"],- )- .unwrap();- assert_eq!(db.get_repeated_text_embedding_cache("dog").unwrap(), None);- let remaining: i64 = connection- .query_row("SELECT COUNT(*) FROM text_embeddings_cache", (), |row| {- row.get(0)- })- .unwrap();- assert_eq!(remaining, 0);-}--#[test]-fn face_cache_replaces_and_removes() {- let (_directory, db) = seeded();- db.put_face_id_cached_for_person_or_cluster("p1", "2_0")- .unwrap();- assert_eq!(- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),- Some("2_0".to_string())- );- db.remove_face_id_cached_for_person_or_cluster("p1")- .unwrap();- assert_eq!(- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),- None- );-}--#[test]-fn clear_non_pet_tables_leaves_pets_and_caches() {- let (_directory, db) = seeded();- db.clear_non_pet_tables().unwrap();- assert_eq!(db.count_clip_rows().unwrap(), 0);- assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());- assert_eq!(- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),- Some("1_0".to_string())- );- assert_eq!(- db.get_repeated_text_embedding_cache("dog").unwrap(),- Some(vec![0.5f32, -1.0])- );-}--#[test]-fn clear_pet_tables_leaves_non_pet_tables() {- let (directory, db) = seeded();- let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();- connection- .execute_batch(- "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');- INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",- )- .unwrap();- db.clear_pet_tables().unwrap();- let row_count = |table: &str| -> i64 {- connection- .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {- row.get(0)- })- .unwrap()- };- assert_eq!(row_count("pet_face_vector_id_map"), 0);- assert_eq!(row_count("pet_body_vector_id_map"), 0);- assert_eq!(db.count_clip_rows().unwrap(), 4);- assert_eq!(db.get_file_ids_with_fd_data(None).unwrap(), ids([1, 2, 3]));- assert_eq!(- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),- Some("1_0".to_string())- );-}