π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Require nonzero database batch and chunk sizes
laurenspriem9eb38f0692ca
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rsindex 29e642bd7e..82c1537847 100644--- a/rust/crates/photos/src/db.rs+++ b/rust/crates/photos/src/db.rs@@ -1,5 +1,6 @@use std::collections::HashMap;use std::hash::Hash;+use std::num::NonZeroUsize;use std::ops::{Deref, DerefMut};use std::path::Path;use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};@@ -163,11 +164,11 @@ impl Database {&self,sql: &str,ids: &[I],- chunk_size: usize,+ chunk_size: NonZeroUsize,mut map: impl FnMut(&Row<'_>) -> SqliteResult<T>,) -> Result<C> {let mut rows = Vec::new();- for chunk in ids.chunks(chunk_size) {+ for chunk in ids.chunks(chunk_size.get()) {let sql = expand_in_clause(sql, chunk.len());let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;rows.extend(chunk_rows);@@ -248,7 +249,7 @@ impl Database {pub fn write_batches_committing_each<P: Params>(&self,sql: &str,- batch_size: usize,+ batch_size: NonZeroUsize,parameter_sets: impl IntoIterator<Item = P>,) -> Result<()> {self.pool.write(|connection| {@@ -258,7 +259,7 @@ impl Database {connection.transaction_with_behavior(TransactionBehavior::Immediate)?;{let mut statement = transaction.prepare_cached(sql)?;- for parameters in parameter_sets.by_ref().take(batch_size) {+ for parameters in parameter_sets.by_ref().take(batch_size.get()) {statement.execute(parameters)?;}}@@ -389,6 +390,7 @@ pub(crate) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(#[cfg(test)]mod tests {use std::collections::{HashMap, HashSet};+ use std::num::NonZeroUsize;use std::panic::{AssertUnwindSafe, catch_unwind};use std::sync::mpsc::{self, RecvTimeoutError};use std::thread;@@ -719,14 +721,14 @@ mod tests {let (_directory, db) = open();db.write_batches_committing_each("INSERT INTO missing_table VALUES (?)",- 2,+ const { NonZeroUsize::new(2).unwrap() },Vec::<[i64; 1]>::new(),).unwrap();assert!(db.write_batches_committing_each("INSERT INTO items (id) VALUES (?)",- 2,+ const { NonZeroUsize::new(2).unwrap() },[[1], [2], [3], [4], [1]]).is_err()@@ -739,7 +741,7 @@ mod tests {assert!(db.write_batches_committing_each("INSERT INTO items (id) VALUES (?)",- 2,+ const { NonZeroUsize::new(2).unwrap() },[[5], [6], [7], [5], [8]]).is_err()@@ -749,13 +751,30 @@ mod tests {.unwrap(),[1, 2, 3, 4, 5, 6]);- db.write_batches_committing_each("INSERT INTO items (id) VALUES (?)", 2, [[7], [8], [9]])- .unwrap();+ db.write_batches_committing_each(+ "INSERT INTO items (id) VALUES (?)",+ const { NonZeroUsize::new(2).unwrap() },+ [[7], [8], [9]],+ )+ .unwrap();assert_eq!(db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ()).unwrap(),[1, 2, 3, 4, 5, 6, 7, 8, 9]);+ assert!(+ db.write_batches_committing_each(+ "INSERT INTO items (id) VALUES (?)",+ NonZeroUsize::MIN,+ [[10], [11], [10]],+ )+ .is_err()+ );+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())+ .unwrap(),+ (1..=11).collect::<Vec<_>>()+ );}#[test]@@ -764,11 +783,20 @@ mod tests {db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id])).unwrap();let ids: Vec<i64> = (1..=7).collect();+ let one_at_a_time: Vec<i64> = db+ .read_chunked_in(+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",+ &ids,+ NonZeroUsize::MIN,+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(one_at_a_time, ids);let by_chunk: Vec<i64> = db.read_chunked_in("SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",&ids,- 3,+ const { NonZeroUsize::new(3).unwrap() },|row| row.get(0),).unwrap();@@ -777,7 +805,7 @@ mod tests {.read_chunked_in("SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",&ids,- 10,+ const { NonZeroUsize::new(10).unwrap() },|row| row.get(0),).unwrap();@@ -786,7 +814,7 @@ mod tests {.read_chunked_in("SELECT id FROM missing_table WHERE id IN ({})",&Vec::<i64>::new(),- 3,+ const { NonZeroUsize::new(3).unwrap() },|row| row.get(0),).unwrap();@@ -796,7 +824,7 @@ mod tests {.read_chunked_in("SELECT id FROM items WHERE id IN ({})",&many,- MAX_SQL_BIND_PARAMS_PER_QUERY,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },|row| row.get(0),).unwrap();diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rsindex 28cc4a6b3b..976a0fdebf 100644--- a/rust/crates/photos/src/ml_db/queries/clusters.rs+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs@@ -1,4 +1,5 @@use std::collections::{HashMap, HashSet};+use std::num::NonZeroUsize;use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, bind_placeholders, group_into,@@ -48,7 +49,11 @@ impl MlDb {face_id_to_cluster_id: &HashMap<String, String>,) -> Result<()> {self.db- .write_batches_committing_each(UPSERT_FACE_CLUSTER, 500, face_id_to_cluster_id.iter())+ .write_batches_committing_each(+ UPSERT_FACE_CLUSTER,+ const { NonZeroUsize::new(500).unwrap() },+ face_id_to_cluster_id.iter(),+ ).map_err(Into::into)}@@ -100,7 +105,7 @@ impl MlDb {let rows: Vec<(String, String)> = self.db.read_chunked_in("SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",&cluster_id_list,- MAX_SQL_BIND_PARAMS_PER_QUERY,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },pair,)?;Ok(group_into(rows))@@ -177,7 +182,7 @@ impl MlDb {.read_chunked_in("SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",face_ids,- MAX_SQL_BIND_PARAMS_PER_QUERY,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },pair,).map_err(Into::into)@@ -263,7 +268,7 @@ impl MlDb {WHERE cluster_id IN ({})"#,&unique_cluster_ids,- 800,+ const { NonZeroUsize::new(800).unwrap() },pair,).map_err(Into::into)@@ -290,7 +295,7 @@ impl MlDb {self.db.write_batches_committing_each(UPSERT_CLUSTER_SUMMARY,- 400,+ const { NonZeroUsize::new(400).unwrap() },summary.iter().map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rsindex f34140482a..aeea11fc45 100644--- a/rust/crates/photos/src/ml_db/queries/faces.rs+++ b/rust/crates/photos/src/ml_db/queries/faces.rs@@ -1,4 +1,5 @@use std::collections::{HashMap, HashSet};+use std::num::NonZeroUsize;use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, ToSql, bind_placeholders, group_into,@@ -77,7 +78,7 @@ impl MlDb {self.db.write_batches_committing_each(UPSERT_FACE,- 500,+ const { NonZeroUsize::new(500).unwrap() },faces.iter().map(|face| {(face.file_id,diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rsindex d1b900f0dc..6c43340720 100644--- a/rust/crates/photos/src/ml_db/queries/persons.rs+++ b/rust/crates/photos/src/ml_db/queries/persons.rs@@ -1,4 +1,5 @@use std::collections::{HashMap, HashSet};+use std::num::NonZeroUsize;use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};@@ -75,7 +76,7 @@ impl MlDb {.read_chunked_in("SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",person_ids,- MAX_SQL_BIND_PARAMS_PER_QUERY,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },|row| row.get(0),).map_err(Into::into)@@ -124,7 +125,7 @@ impl MlDb {WHERE face_id IN ({})"#,face_ids,- MAX_SQL_BIND_PARAMS_PER_QUERY,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },pair,).map_err(Into::into)diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rsindex 3f2e0b2150..54ed4eb595 100644--- a/rust/crates/photos/src/ml_db/queries/pets.rs+++ b/rust/crates/photos/src/ml_db/queries/pets.rs@@ -1,4 +1,5 @@use std::collections::HashMap;+use std::num::NonZeroUsize;use crate::db::{Row, SqliteResult, bind_placeholders, pair, params_from_iter};@@ -89,7 +90,7 @@ impl MlDb {self.db.write_batches_committing_each(UPSERT_PET_FACE,- 500,+ const { NonZeroUsize::new(500).unwrap() },pet_faces.iter().map(|pet_face| {(pet_face.file_id,@@ -111,7 +112,7 @@ impl MlDb {self.db.write_batches_committing_each(UPSERT_PET_BODY,- 500,+ const { NonZeroUsize::new(500).unwrap() },pet_bodies.iter().map(|pet_body| {(pet_body.file_id,@@ -139,7 +140,7 @@ impl MlDb {self.db.write_batches_committing_each("UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",- 500,+ const { NonZeroUsize::new(500).unwrap() },pet_face_id_to_vector_id.iter().map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),@@ -157,7 +158,7 @@ impl MlDb {self.db.write_batches_committing_each("UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",- 500,+ const { NonZeroUsize::new(500).unwrap() },pet_body_id_to_vector_id.iter().map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),@@ -332,7 +333,12 @@ impl MlDb {}let select_sql = select_prefix.to_owned() + " ({})";self.db- .read_chunked_in(&select_sql, &unique_ids, 800, pair)+ .read_chunked_in(+ &select_sql,+ &unique_ids,+ const { NonZeroUsize::new(800).unwrap() },+ pair,+ ).map_err(Into::into)}}