π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust][photos] Make bulk ML writes atomic
laurenspriem19da304b877b
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rsindex 64d63fcb10..27b76fce0d 100644--- a/rust/crates/photos/src/db.rs+++ b/rust/crates/photos/src/db.rs@@ -3,8 +3,8 @@ use std::path::Path;use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};use std::time::Duration;-use rusqlite::{OpenFlags, OptionalExtension};-pub use rusqlite::{Params, Row, ToSql, TransactionBehavior, params_from_iter, types::FromSql};+use rusqlite::{OpenFlags, OptionalExtension, TransactionBehavior};+pub use rusqlite::{Params, Row, ToSql, params_from_iter, types::FromSql};const BUSY_TIMEOUT: Duration = Duration::from_secs(30);@@ -54,11 +54,10 @@ impl Connection {Ok(self.0.pragma_update(None, name, value)?)}- pub fn transaction_with_behavior(- &mut self,- behavior: TransactionBehavior,- ) -> Result<Transaction<'_>> {- Ok(Transaction(self.0.transaction_with_behavior(behavior)?))+ pub fn immediate_transaction(&mut self) -> Result<Transaction<'_>> {+ Ok(Transaction(self.0.transaction_with_behavior(+ TransactionBehavior::Immediate,+ )?))}}@@ -187,8 +186,7 @@ impl Database {write: impl FnOnce(&Transaction<'_>) -> Result<T>,) -> Result<T> {self.write(|connection| {- let transaction =- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;+ let transaction = connection.immediate_transaction()?;let result = write(&transaction)?;transaction.commit()?;Ok(result)@@ -281,9 +279,7 @@ mod tests {use std::thread;use std::time::Duration;- use super::{- Connection, Database, Error, OpenOptions, Result, TransactionBehavior, lock, open_reader,- };+ use super::{Connection, Database, Error, OpenOptions, Result, lock, open_reader};fn open() -> (tempfile::TempDir, Database) {let directory = tempfile::tempdir().unwrap();@@ -475,21 +471,21 @@ mod tests {.unwrap();let counts = db.read(|connection| {- let transaction =- connection.transaction_with_behavior(TransactionBehavior::Deferred)?;- let before: i64 =- transaction- .query_row("SELECT COUNT(*) FROM items", (), |row| Ok(row.get(0)?))?;+ connection.execute_batch("BEGIN")?;+ let before: i64 = connection.query_row(+ "SELECT COUNT(*) FROM items",+ (),+ |row| Ok(row.get(0)?),+ )?;db.write(|writer| writer.execute("INSERT INTO items VALUES (?)", [2]))?;- let after: i64 =- transaction- .query_row("SELECT COUNT(*) FROM items", (), |row| Ok(row.get(0)?))?;+ let after: i64 = connection+ .query_row("SELECT COUNT(*) FROM items", (), |row| Ok(row.get(0)?))?;assert!(- transaction+ connection.execute("INSERT INTO items VALUES (?)", [3]).is_err());- transaction.commit()?;+ connection.execute_batch("COMMIT")?;Ok((before, after))}).unwrap();diff --git a/rust/crates/photos/src/ml_db/migrations.rs b/rust/crates/photos/src/ml_db/migrations.rsindex bb54ec9a80..5f4dd81fbb 100644--- a/rust/crates/photos/src/ml_db/migrations.rs+++ b/rust/crates/photos/src/ml_db/migrations.rs@@ -1,4 +1,4 @@-use crate::db::{Connection, TransactionBehavior};+use crate::db::Connection;use super::{Error, Result};@@ -10,7 +10,7 @@ pub(super) fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<(return Ok(());}- let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;+ let transaction = connection.immediate_transaction()?;let current: i64 = transaction.pragma_query_value("user_version", |row| Ok(row.get(0)?))?;check_not_downgrade(current, target)?;if current == target {diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rsindex d8c58f2392..b19efac9a7 100644--- a/rust/crates/photos/src/ml_db/queries/clusters.rs+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs@@ -48,11 +48,7 @@ impl MlDb {&self,face_id_to_cluster_id: &HashMap<String, String>,) -> Result<()> {- self.write_batches_committing_each(- UPSERT_FACE_CLUSTER,- const { NonZeroUsize::new(500).unwrap() },- face_id_to_cluster_id.iter(),- )+ self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())}pub fn cluster_id_to_face_count(&self) -> Result<HashMap<String, i64>> {@@ -184,7 +180,7 @@ impl MlDb {&self,face_id_to_cluster_id: &HashMap<String, String>,) -> Result<()> {- self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())+ self.update_face_id_to_cluster_id(face_id_to_cluster_id)}pub fn remove_face_id_to_cluster_id(@@ -265,9 +261,8 @@ impl MlDb {&self,summary: &HashMap<String, ClusterSummary>,) -> Result<()> {- self.write_batches_committing_each(+ self.write_batch_atomic(UPSERT_CLUSTER_SUMMARY,- const { NonZeroUsize::new(400).unwrap() },summary.iter().map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rsindex e7a09ac9f8..0e3897e1c4 100644--- a/rust/crates/photos/src/ml_db/queries/faces.rs+++ b/rust/crates/photos/src/ml_db/queries/faces.rs@@ -1,5 +1,4 @@use std::collections::{HashMap, HashSet};-use std::num::NonZeroUsize;use super::helpers::{MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,@@ -75,9 +74,8 @@ const UPSERT_FACE: &str = r#"impl MlDb {pub fn bulk_insert_faces(&self, faces: &[FaceRow]) -> Result<()> {- self.write_batches_committing_each(+ self.write_batch_atomic(UPSERT_FACE,- const { NonZeroUsize::new(500).unwrap() },faces.iter().map(|face| {(face.file_id,diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rsindex 3af6c6634a..b9bbf4a337 100644--- a/rust/crates/photos/src/ml_db/queries/helpers.rs+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs@@ -2,7 +2,7 @@ use std::collections::HashMap;use std::hash::Hash;use std::num::NonZeroUsize;-use crate::db::{self, FromSql, Params, Row, ToSql, TransactionBehavior, params_from_iter};+use crate::db::{self, FromSql, Params, Row, ToSql, params_from_iter};use crate::ml_db::{MlDb, Result};@@ -145,31 +145,6 @@ impl MlDb {}).map_err(Into::into)}-- pub(in crate::ml_db) fn write_batches_committing_each<P: Params>(- &self,- sql: &str,- batch_size: NonZeroUsize,- parameter_sets: impl IntoIterator<Item = P>,- ) -> Result<()> {- self.db- .write(|connection| {- let mut parameter_sets = parameter_sets.into_iter().peekable();- while parameter_sets.peek().is_some() {- let transaction =- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;- {- let mut statement = transaction.prepare_cached(sql)?;- for parameters in parameter_sets.by_ref().take(batch_size.get()) {- statement.execute(parameters)?;- }- }- transaction.commit()?;- }- Ok(())- })- .map_err(Into::into)- }}pub(super) fn bind_placeholders(count: usize) -> String {@@ -320,122 +295,6 @@ mod tests {);}- #[test]- fn batched_writes_commit_each_completed_batch() {- let (_directory, db) = open();- db.write_batches_committing_each(- "INSERT INTO missing_table VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- Vec::<[i64; 1]>::new(),- )- .unwrap();- assert!(- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- [[1], [2], [3], [4], [1]]- )- .is_err()- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- [1, 2, 3, 4]- );- assert!(- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- [[5], [6], [7], [5], [8]]- )- .is_err()- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- [1, 2, 3, 4, 5, 6]- );- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- [[7], [8], [9]],- )- .unwrap();- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- [1, 2, 3, 4, 5, 6, 7, 8, 9]- );- assert!(- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- NonZeroUsize::MIN,- [[10], [11], [10]],- )- .is_err()- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- (1..=11).collect::<Vec<_>>()- );- }-- #[test]- fn batched_writes_hold_the_writer_between_commits() {- use std::sync::mpsc::{self, RecvTimeoutError};- use std::thread;- use std::time::Duration;-- let (_directory, db) = open();- thread::scope(|scope| {- let (between_tx, between_rx) = mpsc::channel();- let (release_tx, release_rx) = mpsc::channel();- let db = &db;- scope.spawn(move || {- db.write_batches_committing_each(- "INSERT INTO items VALUES (?)",- NonZeroUsize::MIN,- (1..=2).map(|id| {- if id == 2 {- between_tx.send(()).unwrap();- release_rx.recv().unwrap();- }- [id]- }),- )- .unwrap();- });- between_rx.recv_timeout(Duration::from_secs(5)).unwrap();- let committed_count = db- .read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap();- let (started_tx, started_rx) = mpsc::channel();- let (finished_tx, finished_rx) = mpsc::channel();- scope.spawn(move || {- started_tx.send(()).unwrap();- finished_tx- .send(db.execute("INSERT INTO items VALUES (3)", ()))- .unwrap();- });- started_rx.recv_timeout(Duration::from_secs(5)).unwrap();- let was_blocked = matches!(- finished_rx.recv_timeout(Duration::from_millis(100)),- Err(RecvTimeoutError::Timeout)- );- release_tx.send(()).unwrap();- let result = finished_rx.recv_timeout(Duration::from_secs(5)).unwrap();- assert_eq!(committed_count, 1);- assert!(was_blocked);- assert_eq!(result.unwrap(), 1);- });- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 3- );- }-#[test]fn chunked_reads_query_each_chunk_in_order() {let (_directory, db) = open();diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rsindex 63309f1407..3510a6282f 100644--- a/rust/crates/photos/src/ml_db/queries/pets.rs+++ b/rust/crates/photos/src/ml_db/queries/pets.rs@@ -88,9 +88,8 @@ const UPSERT_PET_BODY: &str = r#"impl MlDb {pub fn bulk_insert_pet_faces(&self, pet_faces: &[PetFaceRow]) -> Result<()> {- self.write_batches_committing_each(+ self.write_batch_atomic(UPSERT_PET_FACE,- const { NonZeroUsize::new(500).unwrap() },pet_faces.iter().map(|pet_face| {(pet_face.file_id,@@ -108,9 +107,8 @@ impl MlDb {}pub fn bulk_insert_pet_bodies(&self, pet_bodies: &[PetBodyRow]) -> Result<()> {- self.write_batches_committing_each(+ self.write_batch_atomic(UPSERT_PET_BODY,- const { NonZeroUsize::new(500).unwrap() },pet_bodies.iter().map(|pet_body| {(pet_body.file_id,@@ -134,9 +132,8 @@ impl MlDb {if pet_face_id_to_vector_id.is_empty() {return Ok(());}- self.write_batches_committing_each(+ self.write_batch_atomic("UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",- const { NonZeroUsize::new(500).unwrap() },pet_face_id_to_vector_id.iter().map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),@@ -150,9 +147,8 @@ impl MlDb {if pet_body_id_to_vector_id.is_empty() {return Ok(());}- self.write_batches_committing_each(+ self.write_batch_atomic("UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",- const { NonZeroUsize::new(500).unwrap() },pet_body_id_to_vector_id.iter().map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),