π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust][photos] Use one chunk size for IN-list reads
laurenspriem79b284477d89
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rsindex b19efac9a7..7fa6374868 100644--- a/rust/crates/photos/src/ml_db/queries/clusters.rs+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs@@ -1,9 +1,6 @@use std::collections::{HashMap, HashSet};-use std::num::NonZeroUsize;-use super::helpers::{- MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,-};+use super::helpers::{bind_placeholders, group_into, optional_parameter, pair};use crate::db::{Result as SqliteResult, Row, params_from_iter};use super::faces::{file_id_from_face_id, is_bad_face_for_clustering};@@ -97,7 +94,6 @@ impl MlDb {let rows: Vec<(String, String)> = self.read_chunked_in("SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",&cluster_id_list,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },pair,)?;Ok(group_into(rows))@@ -166,7 +162,6 @@ impl MlDb {self.read_chunked_in("SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",face_ids,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },pair,)}@@ -240,7 +235,6 @@ impl MlDb {WHERE cluster_id IN ({})"#,&unique_cluster_ids,- const { NonZeroUsize::new(800).unwrap() },pair,)}diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rsindex b9bbf4a337..38d712409a 100644--- a/rust/crates/photos/src/ml_db/queries/helpers.rs+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs@@ -1,6 +1,5 @@use std::collections::HashMap;use std::hash::Hash;-use std::num::NonZeroUsize;use crate::db::{self, FromSql, Params, Row, ToSql, params_from_iter};@@ -79,11 +78,10 @@ impl MlDb {&self,sql: &str,ids: &[I],- chunk_size: NonZeroUsize,mut map: impl FnMut(&Row<'_>) -> db::Result<T>,) -> Result<C> {let mut rows = Vec::new();- for chunk in ids.chunks(chunk_size.get()) {+ for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {let sql = expand_in_clause(sql, chunk.len());let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;rows.extend(chunk_rows);@@ -179,7 +177,6 @@ pub(super) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(#[cfg(test)]mod tests {use std::collections::{HashMap, HashSet};- use std::num::NonZeroUsize;use crate::db::{Database, OpenOptions};@@ -298,55 +295,29 @@ mod tests {#[test]fn chunked_reads_query_each_chunk_in_order() {let (_directory, db) = open();- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))- .unwrap();- let ids: Vec<i64> = (1..=7).collect();- let one_at_a_time: Vec<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",- &ids,- NonZeroUsize::MIN,- |row| Ok(row.get(0)?),- )- .unwrap();- assert_eq!(one_at_a_time, ids);+ let boundary = MAX_SQL_BIND_PARAMS_PER_QUERY as i64;+ db.write_batch_atomic(+ "INSERT INTO items (id) VALUES (?)",+ [1, 2, boundary - 1, boundary, boundary + 1].map(|id| [id]),+ )+ .unwrap();+ let ids: Vec<i64> = (0..=boundary + 1).collect();let by_chunk: Vec<i64> = db.read_chunked_in("SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",&ids,- const { NonZeroUsize::new(3).unwrap() },|row| Ok(row.get(0)?),).unwrap();- assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);- let single_chunk: Vec<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",- &ids,- const { NonZeroUsize::new(10).unwrap() },- |row| Ok(row.get(0)?),- )- .unwrap();- assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);+ assert_eq!(by_chunk, [boundary - 1, 2, 1, boundary + 1, boundary]);let none: Vec<i64> = db.read_chunked_in("SELECT id FROM missing_table WHERE id IN ({})",&Vec::<i64>::new(),- const { NonZeroUsize::new(3).unwrap() },|row| Ok(row.get(0)?),).unwrap();assert!(none.is_empty());- let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();- let found: HashSet<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({})",- &many,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },- |row| Ok(row.get(0)?),- )- .unwrap();- assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));}#[test]diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rsindex b3f0b485ab..284fed900a 100644--- a/rust/crates/photos/src/ml_db/queries/persons.rs+++ b/rust/crates/photos/src/ml_db/queries/persons.rs@@ -1,7 +1,6 @@use std::collections::{HashMap, HashSet};-use std::num::NonZeroUsize;-use super::helpers::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};+use super::helpers::pair;use super::clusters::file_id_to_cluster_ids;use crate::ml_db::schema::{@@ -69,7 +68,6 @@ impl MlDb {self.read_chunked_in("SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",person_ids,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },|row| Ok(row.get(0)?),)}@@ -114,7 +112,6 @@ impl MlDb {WHERE face_id IN ({})"#,face_ids,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },pair,)}diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rsindex 3510a6282f..05be99f746 100644--- a/rust/crates/photos/src/ml_db/queries/pets.rs+++ b/rust/crates/photos/src/ml_db/queries/pets.rs@@ -1,5 +1,4 @@use std::collections::HashMap;-use std::num::NonZeroUsize;use super::helpers::{bind_placeholders, pair};use crate::db::{Result as SqliteResult, Row, params_from_iter};@@ -308,12 +307,7 @@ impl MlDb {self.write_batch_atomic(insert_sql, unique_ids.iter().map(|id| [id]))?;}let select_sql = select_prefix.to_owned() + " ({})";- self.read_chunked_in(- &select_sql,- &unique_ids,- const { NonZeroUsize::new(800).unwrap() },- pair,- )+ self.read_chunked_in(&select_sql, &unique_ids, pair)}}