Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][photos] Use one chunk size for IN-list reads

laurenspriem79b284477d89
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index b19efac9a7..7fa6374868 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -1,9 +1,6 @@
use std::collections::{HashMap, HashSet};
-use std::num::NonZeroUsize;
-use super::helpers::{
- MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,
-};
+use super::helpers::{bind_placeholders, group_into, optional_parameter, pair};
use crate::db::{Result as SqliteResult, Row, params_from_iter};
use super::faces::{file_id_from_face_id, is_bad_face_for_clustering};
@@ -97,7 +94,6 @@ impl MlDb {
let rows: Vec<(String, String)> = self.read_chunked_in(
"SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",
&cluster_id_list,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)?;
Ok(group_into(rows))
@@ -166,7 +162,6 @@ impl MlDb {
self.read_chunked_in(
"SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",
face_ids,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)
}
@@ -240,7 +235,6 @@ impl MlDb {
WHERE cluster_id IN ({})
"#,
&unique_cluster_ids,
- const { NonZeroUsize::new(800).unwrap() },
pair,
)
}
diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rs
index b9bbf4a337..38d712409a 100644
--- a/rust/crates/photos/src/ml_db/queries/helpers.rs
+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs
@@ -1,6 +1,5 @@
use std::collections::HashMap;
use std::hash::Hash;
-use std::num::NonZeroUsize;
use crate::db::{self, FromSql, Params, Row, ToSql, params_from_iter};
@@ -79,11 +78,10 @@ impl MlDb {
&self,
sql: &str,
ids: &[I],
- chunk_size: NonZeroUsize,
mut map: impl FnMut(&Row<'_>) -> db::Result<T>,
) -> Result<C> {
let mut rows = Vec::new();
- for chunk in ids.chunks(chunk_size.get()) {
+ for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
let sql = expand_in_clause(sql, chunk.len());
let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;
rows.extend(chunk_rows);
@@ -179,7 +177,6 @@ pub(super) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(
#[cfg(test)]
mod tests {
use std::collections::{HashMap, HashSet};
- use std::num::NonZeroUsize;
use crate::db::{Database, OpenOptions};
@@ -298,55 +295,29 @@ mod tests {
#[test]
fn chunked_reads_query_each_chunk_in_order() {
let (_directory, db) = open();
- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))
- .unwrap();
- let ids: Vec<i64> = (1..=7).collect();
- let one_at_a_time: Vec<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
- &ids,
- NonZeroUsize::MIN,
- |row| Ok(row.get(0)?),
- )
- .unwrap();
- assert_eq!(one_at_a_time, ids);
+ let boundary = MAX_SQL_BIND_PARAMS_PER_QUERY as i64;
+ db.write_batch_atomic(
+ "INSERT INTO items (id) VALUES (?)",
+ [1, 2, boundary - 1, boundary, boundary + 1].map(|id| [id]),
+ )
+ .unwrap();
+ let ids: Vec<i64> = (0..=boundary + 1).collect();
let by_chunk: Vec<i64> = db
.read_chunked_in(
"SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
&ids,
- const { NonZeroUsize::new(3).unwrap() },
|row| Ok(row.get(0)?),
)
.unwrap();
- assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);
- let single_chunk: Vec<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
- &ids,
- const { NonZeroUsize::new(10).unwrap() },
- |row| Ok(row.get(0)?),
- )
- .unwrap();
- assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);
+ assert_eq!(by_chunk, [boundary - 1, 2, 1, boundary + 1, boundary]);
let none: Vec<i64> = db
.read_chunked_in(
"SELECT id FROM missing_table WHERE id IN ({})",
&Vec::<i64>::new(),
- const { NonZeroUsize::new(3).unwrap() },
|row| Ok(row.get(0)?),
)
.unwrap();
assert!(none.is_empty());
- let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();
- let found: HashSet<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({})",
- &many,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
- |row| Ok(row.get(0)?),
- )
- .unwrap();
- assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));
}
#[test]
diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rs
index b3f0b485ab..284fed900a 100644
--- a/rust/crates/photos/src/ml_db/queries/persons.rs
+++ b/rust/crates/photos/src/ml_db/queries/persons.rs
@@ -1,7 +1,6 @@
use std::collections::{HashMap, HashSet};
-use std::num::NonZeroUsize;
-use super::helpers::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};
+use super::helpers::pair;
use super::clusters::file_id_to_cluster_ids;
use crate::ml_db::schema::{
@@ -69,7 +68,6 @@ impl MlDb {
self.read_chunked_in(
"SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",
person_ids,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
|row| Ok(row.get(0)?),
)
}
@@ -114,7 +112,6 @@ impl MlDb {
WHERE face_id IN ({})
"#,
face_ids,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)
}
diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rs
index 3510a6282f..05be99f746 100644
--- a/rust/crates/photos/src/ml_db/queries/pets.rs
+++ b/rust/crates/photos/src/ml_db/queries/pets.rs
@@ -1,5 +1,4 @@
use std::collections::HashMap;
-use std::num::NonZeroUsize;
use super::helpers::{bind_placeholders, pair};
use crate::db::{Result as SqliteResult, Row, params_from_iter};
@@ -308,12 +307,7 @@ impl MlDb {
self.write_batch_atomic(insert_sql, unique_ids.iter().map(|id| [id]))?;
}
let select_sql = select_prefix.to_owned() + " ({})";
- self.read_chunked_in(
- &select_sql,
- &unique_ids,
- const { NonZeroUsize::new(800).unwrap() },
- pair,
- )
+ self.read_chunked_in(&select_sql, &unique_ids, pair)
}
}