Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Require nonzero database batch and chunk sizes

laurenspriem9eb38f0692ca
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rs
index 29e642bd7e..82c1537847 100644
--- a/rust/crates/photos/src/db.rs
+++ b/rust/crates/photos/src/db.rs
@@ -1,5 +1,6 @@
use std::collections::HashMap;
use std::hash::Hash;
+use std::num::NonZeroUsize;
use std::ops::{Deref, DerefMut};
use std::path::Path;
use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};
@@ -163,11 +164,11 @@ impl Database {
&self,
sql: &str,
ids: &[I],
- chunk_size: usize,
+ chunk_size: NonZeroUsize,
mut map: impl FnMut(&Row<'_>) -> SqliteResult<T>,
) -> Result<C> {
let mut rows = Vec::new();
- for chunk in ids.chunks(chunk_size) {
+ for chunk in ids.chunks(chunk_size.get()) {
let sql = expand_in_clause(sql, chunk.len());
let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;
rows.extend(chunk_rows);
@@ -248,7 +249,7 @@ impl Database {
pub fn write_batches_committing_each<P: Params>(
&self,
sql: &str,
- batch_size: usize,
+ batch_size: NonZeroUsize,
parameter_sets: impl IntoIterator<Item = P>,
) -> Result<()> {
self.pool.write(|connection| {
@@ -258,7 +259,7 @@ impl Database {
connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
{
let mut statement = transaction.prepare_cached(sql)?;
- for parameters in parameter_sets.by_ref().take(batch_size) {
+ for parameters in parameter_sets.by_ref().take(batch_size.get()) {
statement.execute(parameters)?;
}
}
@@ -389,6 +390,7 @@ pub(crate) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(
#[cfg(test)]
mod tests {
use std::collections::{HashMap, HashSet};
+ use std::num::NonZeroUsize;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::mpsc::{self, RecvTimeoutError};
use std::thread;
@@ -719,14 +721,14 @@ mod tests {
let (_directory, db) = open();
db.write_batches_committing_each(
"INSERT INTO missing_table VALUES (?)",
- 2,
+ const { NonZeroUsize::new(2).unwrap() },
Vec::<[i64; 1]>::new(),
)
.unwrap();
assert!(
db.write_batches_committing_each(
"INSERT INTO items (id) VALUES (?)",
- 2,
+ const { NonZeroUsize::new(2).unwrap() },
[[1], [2], [3], [4], [1]]
)
.is_err()
@@ -739,7 +741,7 @@ mod tests {
assert!(
db.write_batches_committing_each(
"INSERT INTO items (id) VALUES (?)",
- 2,
+ const { NonZeroUsize::new(2).unwrap() },
[[5], [6], [7], [5], [8]]
)
.is_err()
@@ -749,13 +751,30 @@ mod tests {
.unwrap(),
[1, 2, 3, 4, 5, 6]
);
- db.write_batches_committing_each("INSERT INTO items (id) VALUES (?)", 2, [[7], [8], [9]])
- .unwrap();
+ db.write_batches_committing_each(
+ "INSERT INTO items (id) VALUES (?)",
+ const { NonZeroUsize::new(2).unwrap() },
+ [[7], [8], [9]],
+ )
+ .unwrap();
assert_eq!(
db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
.unwrap(),
[1, 2, 3, 4, 5, 6, 7, 8, 9]
);
+ assert!(
+ db.write_batches_committing_each(
+ "INSERT INTO items (id) VALUES (?)",
+ NonZeroUsize::MIN,
+ [[10], [11], [10]],
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ (1..=11).collect::<Vec<_>>()
+ );
}
#[test]
@@ -764,11 +783,20 @@ mod tests {
db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))
.unwrap();
let ids: Vec<i64> = (1..=7).collect();
+ let one_at_a_time: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
+ &ids,
+ NonZeroUsize::MIN,
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(one_at_a_time, ids);
let by_chunk: Vec<i64> = db
.read_chunked_in(
"SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
&ids,
- 3,
+ const { NonZeroUsize::new(3).unwrap() },
|row| row.get(0),
)
.unwrap();
@@ -777,7 +805,7 @@ mod tests {
.read_chunked_in(
"SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
&ids,
- 10,
+ const { NonZeroUsize::new(10).unwrap() },
|row| row.get(0),
)
.unwrap();
@@ -786,7 +814,7 @@ mod tests {
.read_chunked_in(
"SELECT id FROM missing_table WHERE id IN ({})",
&Vec::<i64>::new(),
- 3,
+ const { NonZeroUsize::new(3).unwrap() },
|row| row.get(0),
)
.unwrap();
@@ -796,7 +824,7 @@ mod tests {
.read_chunked_in(
"SELECT id FROM items WHERE id IN ({})",
&many,
- MAX_SQL_BIND_PARAMS_PER_QUERY,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
|row| row.get(0),
)
.unwrap();
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index 28cc4a6b3b..976a0fdebf 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -1,4 +1,5 @@
use std::collections::{HashMap, HashSet};
+use std::num::NonZeroUsize;
use crate::db::{
MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, bind_placeholders, group_into,
@@ -48,7 +49,11 @@ impl MlDb {
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
self.db
- .write_batches_committing_each(UPSERT_FACE_CLUSTER, 500, face_id_to_cluster_id.iter())
+ .write_batches_committing_each(
+ UPSERT_FACE_CLUSTER,
+ const { NonZeroUsize::new(500).unwrap() },
+ face_id_to_cluster_id.iter(),
+ )
.map_err(Into::into)
}
@@ -100,7 +105,7 @@ impl MlDb {
let rows: Vec<(String, String)> = self.db.read_chunked_in(
"SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",
&cluster_id_list,
- MAX_SQL_BIND_PARAMS_PER_QUERY,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)?;
Ok(group_into(rows))
@@ -177,7 +182,7 @@ impl MlDb {
.read_chunked_in(
"SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",
face_ids,
- MAX_SQL_BIND_PARAMS_PER_QUERY,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)
.map_err(Into::into)
@@ -263,7 +268,7 @@ impl MlDb {
WHERE cluster_id IN ({})
"#,
&unique_cluster_ids,
- 800,
+ const { NonZeroUsize::new(800).unwrap() },
pair,
)
.map_err(Into::into)
@@ -290,7 +295,7 @@ impl MlDb {
self.db
.write_batches_committing_each(
UPSERT_CLUSTER_SUMMARY,
- 400,
+ const { NonZeroUsize::new(400).unwrap() },
summary
.iter()
.map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index f34140482a..aeea11fc45 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -1,4 +1,5 @@
use std::collections::{HashMap, HashSet};
+use std::num::NonZeroUsize;
use crate::db::{
MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, ToSql, bind_placeholders, group_into,
@@ -77,7 +78,7 @@ impl MlDb {
self.db
.write_batches_committing_each(
UPSERT_FACE,
- 500,
+ const { NonZeroUsize::new(500).unwrap() },
faces.iter().map(|face| {
(
face.file_id,
diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rs
index d1b900f0dc..6c43340720 100644
--- a/rust/crates/photos/src/ml_db/queries/persons.rs
+++ b/rust/crates/photos/src/ml_db/queries/persons.rs
@@ -1,4 +1,5 @@
use std::collections::{HashMap, HashSet};
+use std::num::NonZeroUsize;
use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};
@@ -75,7 +76,7 @@ impl MlDb {
.read_chunked_in(
"SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",
person_ids,
- MAX_SQL_BIND_PARAMS_PER_QUERY,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
|row| row.get(0),
)
.map_err(Into::into)
@@ -124,7 +125,7 @@ impl MlDb {
WHERE face_id IN ({})
"#,
face_ids,
- MAX_SQL_BIND_PARAMS_PER_QUERY,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)
.map_err(Into::into)
diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rs
index 3f2e0b2150..54ed4eb595 100644
--- a/rust/crates/photos/src/ml_db/queries/pets.rs
+++ b/rust/crates/photos/src/ml_db/queries/pets.rs
@@ -1,4 +1,5 @@
use std::collections::HashMap;
+use std::num::NonZeroUsize;
use crate::db::{Row, SqliteResult, bind_placeholders, pair, params_from_iter};
@@ -89,7 +90,7 @@ impl MlDb {
self.db
.write_batches_committing_each(
UPSERT_PET_FACE,
- 500,
+ const { NonZeroUsize::new(500).unwrap() },
pet_faces.iter().map(|pet_face| {
(
pet_face.file_id,
@@ -111,7 +112,7 @@ impl MlDb {
self.db
.write_batches_committing_each(
UPSERT_PET_BODY,
- 500,
+ const { NonZeroUsize::new(500).unwrap() },
pet_bodies.iter().map(|pet_body| {
(
pet_body.file_id,
@@ -139,7 +140,7 @@ impl MlDb {
self.db
.write_batches_committing_each(
"UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",
- 500,
+ const { NonZeroUsize::new(500).unwrap() },
pet_face_id_to_vector_id
.iter()
.map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),
@@ -157,7 +158,7 @@ impl MlDb {
self.db
.write_batches_committing_each(
"UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",
- 500,
+ const { NonZeroUsize::new(500).unwrap() },
pet_body_id_to_vector_id
.iter()
.map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),
@@ -332,7 +333,12 @@ impl MlDb {
}
let select_sql = select_prefix.to_owned() + " ({})";
self.db
- .read_chunked_in(&select_sql, &unique_ids, 800, pair)
+ .read_chunked_in(
+ &select_sql,
+ &unique_ids,
+ const { NonZeroUsize::new(800).unwrap() },
+ pair,
+ )
.map_err(Into::into)
}
}