Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][photos] Make bulk ML writes atomic

laurenspriem19da304b877b
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rs
index 64d63fcb10..27b76fce0d 100644
--- a/rust/crates/photos/src/db.rs
+++ b/rust/crates/photos/src/db.rs
@@ -3,8 +3,8 @@ use std::path::Path;
use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
-use rusqlite::{OpenFlags, OptionalExtension};
-pub use rusqlite::{Params, Row, ToSql, TransactionBehavior, params_from_iter, types::FromSql};
+use rusqlite::{OpenFlags, OptionalExtension, TransactionBehavior};
+pub use rusqlite::{Params, Row, ToSql, params_from_iter, types::FromSql};
const BUSY_TIMEOUT: Duration = Duration::from_secs(30);
@@ -54,11 +54,10 @@ impl Connection {
Ok(self.0.pragma_update(None, name, value)?)
}
- pub fn transaction_with_behavior(
- &mut self,
- behavior: TransactionBehavior,
- ) -> Result<Transaction<'_>> {
- Ok(Transaction(self.0.transaction_with_behavior(behavior)?))
+ pub fn immediate_transaction(&mut self) -> Result<Transaction<'_>> {
+ Ok(Transaction(self.0.transaction_with_behavior(
+ TransactionBehavior::Immediate,
+ )?))
}
}
@@ -187,8 +186,7 @@ impl Database {
write: impl FnOnce(&Transaction<'_>) -> Result<T>,
) -> Result<T> {
self.write(|connection| {
- let transaction =
- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let transaction = connection.immediate_transaction()?;
let result = write(&transaction)?;
transaction.commit()?;
Ok(result)
@@ -281,9 +279,7 @@ mod tests {
use std::thread;
use std::time::Duration;
- use super::{
- Connection, Database, Error, OpenOptions, Result, TransactionBehavior, lock, open_reader,
- };
+ use super::{Connection, Database, Error, OpenOptions, Result, lock, open_reader};
fn open() -> (tempfile::TempDir, Database) {
let directory = tempfile::tempdir().unwrap();
@@ -475,21 +471,21 @@ mod tests {
.unwrap();
let counts = db
.read(|connection| {
- let transaction =
- connection.transaction_with_behavior(TransactionBehavior::Deferred)?;
- let before: i64 =
- transaction
- .query_row("SELECT COUNT(*) FROM items", (), |row| Ok(row.get(0)?))?;
+ connection.execute_batch("BEGIN")?;
+ let before: i64 = connection.query_row(
+ "SELECT COUNT(*) FROM items",
+ (),
+ |row| Ok(row.get(0)?),
+ )?;
db.write(|writer| writer.execute("INSERT INTO items VALUES (?)", [2]))?;
- let after: i64 =
- transaction
- .query_row("SELECT COUNT(*) FROM items", (), |row| Ok(row.get(0)?))?;
+ let after: i64 = connection
+ .query_row("SELECT COUNT(*) FROM items", (), |row| Ok(row.get(0)?))?;
assert!(
- transaction
+ connection
.execute("INSERT INTO items VALUES (?)", [3])
.is_err()
);
- transaction.commit()?;
+ connection.execute_batch("COMMIT")?;
Ok((before, after))
})
.unwrap();
diff --git a/rust/crates/photos/src/ml_db/migrations.rs b/rust/crates/photos/src/ml_db/migrations.rs
index bb54ec9a80..5f4dd81fbb 100644
--- a/rust/crates/photos/src/ml_db/migrations.rs
+++ b/rust/crates/photos/src/ml_db/migrations.rs
@@ -1,4 +1,4 @@
-use crate::db::{Connection, TransactionBehavior};
+use crate::db::Connection;
use super::{Error, Result};
@@ -10,7 +10,7 @@ pub(super) fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<(
return Ok(());
}
- let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let transaction = connection.immediate_transaction()?;
let current: i64 = transaction.pragma_query_value("user_version", |row| Ok(row.get(0)?))?;
check_not_downgrade(current, target)?;
if current == target {
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index d8c58f2392..b19efac9a7 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -48,11 +48,7 @@ impl MlDb {
&self,
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
- self.write_batches_committing_each(
- UPSERT_FACE_CLUSTER,
- const { NonZeroUsize::new(500).unwrap() },
- face_id_to_cluster_id.iter(),
- )
+ self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())
}
pub fn cluster_id_to_face_count(&self) -> Result<HashMap<String, i64>> {
@@ -184,7 +180,7 @@ impl MlDb {
&self,
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
- self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())
+ self.update_face_id_to_cluster_id(face_id_to_cluster_id)
}
pub fn remove_face_id_to_cluster_id(
@@ -265,9 +261,8 @@ impl MlDb {
&self,
summary: &HashMap<String, ClusterSummary>,
) -> Result<()> {
- self.write_batches_committing_each(
+ self.write_batch_atomic(
UPSERT_CLUSTER_SUMMARY,
- const { NonZeroUsize::new(400).unwrap() },
summary
.iter()
.map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index e7a09ac9f8..0e3897e1c4 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -1,5 +1,4 @@
use std::collections::{HashMap, HashSet};
-use std::num::NonZeroUsize;
use super::helpers::{
MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,
@@ -75,9 +74,8 @@ const UPSERT_FACE: &str = r#"
impl MlDb {
pub fn bulk_insert_faces(&self, faces: &[FaceRow]) -> Result<()> {
- self.write_batches_committing_each(
+ self.write_batch_atomic(
UPSERT_FACE,
- const { NonZeroUsize::new(500).unwrap() },
faces.iter().map(|face| {
(
face.file_id,
diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rs
index 3af6c6634a..b9bbf4a337 100644
--- a/rust/crates/photos/src/ml_db/queries/helpers.rs
+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
use std::hash::Hash;
use std::num::NonZeroUsize;
-use crate::db::{self, FromSql, Params, Row, ToSql, TransactionBehavior, params_from_iter};
+use crate::db::{self, FromSql, Params, Row, ToSql, params_from_iter};
use crate::ml_db::{MlDb, Result};
@@ -145,31 +145,6 @@ impl MlDb {
})
.map_err(Into::into)
}
-
- pub(in crate::ml_db) fn write_batches_committing_each<P: Params>(
- &self,
- sql: &str,
- batch_size: NonZeroUsize,
- parameter_sets: impl IntoIterator<Item = P>,
- ) -> Result<()> {
- self.db
- .write(|connection| {
- let mut parameter_sets = parameter_sets.into_iter().peekable();
- while parameter_sets.peek().is_some() {
- let transaction =
- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
- {
- let mut statement = transaction.prepare_cached(sql)?;
- for parameters in parameter_sets.by_ref().take(batch_size.get()) {
- statement.execute(parameters)?;
- }
- }
- transaction.commit()?;
- }
- Ok(())
- })
- .map_err(Into::into)
- }
}
pub(super) fn bind_placeholders(count: usize) -> String {
@@ -320,122 +295,6 @@ mod tests {
);
}
- #[test]
- fn batched_writes_commit_each_completed_batch() {
- let (_directory, db) = open();
- db.write_batches_committing_each(
- "INSERT INTO missing_table VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- Vec::<[i64; 1]>::new(),
- )
- .unwrap();
- assert!(
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- [[1], [2], [3], [4], [1]]
- )
- .is_err()
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- [1, 2, 3, 4]
- );
- assert!(
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- [[5], [6], [7], [5], [8]]
- )
- .is_err()
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- [1, 2, 3, 4, 5, 6]
- );
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- [[7], [8], [9]],
- )
- .unwrap();
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- [1, 2, 3, 4, 5, 6, 7, 8, 9]
- );
- assert!(
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- NonZeroUsize::MIN,
- [[10], [11], [10]],
- )
- .is_err()
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- (1..=11).collect::<Vec<_>>()
- );
- }
-
- #[test]
- fn batched_writes_hold_the_writer_between_commits() {
- use std::sync::mpsc::{self, RecvTimeoutError};
- use std::thread;
- use std::time::Duration;
-
- let (_directory, db) = open();
- thread::scope(|scope| {
- let (between_tx, between_rx) = mpsc::channel();
- let (release_tx, release_rx) = mpsc::channel();
- let db = &db;
- scope.spawn(move || {
- db.write_batches_committing_each(
- "INSERT INTO items VALUES (?)",
- NonZeroUsize::MIN,
- (1..=2).map(|id| {
- if id == 2 {
- between_tx.send(()).unwrap();
- release_rx.recv().unwrap();
- }
- [id]
- }),
- )
- .unwrap();
- });
- between_rx.recv_timeout(Duration::from_secs(5)).unwrap();
- let committed_count = db
- .read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap();
- let (started_tx, started_rx) = mpsc::channel();
- let (finished_tx, finished_rx) = mpsc::channel();
- scope.spawn(move || {
- started_tx.send(()).unwrap();
- finished_tx
- .send(db.execute("INSERT INTO items VALUES (3)", ()))
- .unwrap();
- });
- started_rx.recv_timeout(Duration::from_secs(5)).unwrap();
- let was_blocked = matches!(
- finished_rx.recv_timeout(Duration::from_millis(100)),
- Err(RecvTimeoutError::Timeout)
- );
- release_tx.send(()).unwrap();
- let result = finished_rx.recv_timeout(Duration::from_secs(5)).unwrap();
- assert_eq!(committed_count, 1);
- assert!(was_blocked);
- assert_eq!(result.unwrap(), 1);
- });
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 3
- );
- }
-
#[test]
fn chunked_reads_query_each_chunk_in_order() {
let (_directory, db) = open();
diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rs
index 63309f1407..3510a6282f 100644
--- a/rust/crates/photos/src/ml_db/queries/pets.rs
+++ b/rust/crates/photos/src/ml_db/queries/pets.rs
@@ -88,9 +88,8 @@ const UPSERT_PET_BODY: &str = r#"
impl MlDb {
pub fn bulk_insert_pet_faces(&self, pet_faces: &[PetFaceRow]) -> Result<()> {
- self.write_batches_committing_each(
+ self.write_batch_atomic(
UPSERT_PET_FACE,
- const { NonZeroUsize::new(500).unwrap() },
pet_faces.iter().map(|pet_face| {
(
pet_face.file_id,
@@ -108,9 +107,8 @@ impl MlDb {
}
pub fn bulk_insert_pet_bodies(&self, pet_bodies: &[PetBodyRow]) -> Result<()> {
- self.write_batches_committing_each(
+ self.write_batch_atomic(
UPSERT_PET_BODY,
- const { NonZeroUsize::new(500).unwrap() },
pet_bodies.iter().map(|pet_body| {
(
pet_body.file_id,
@@ -134,9 +132,8 @@ impl MlDb {
if pet_face_id_to_vector_id.is_empty() {
return Ok(());
}
- self.write_batches_committing_each(
+ self.write_batch_atomic(
"UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",
- const { NonZeroUsize::new(500).unwrap() },
pet_face_id_to_vector_id
.iter()
.map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),
@@ -150,9 +147,8 @@ impl MlDb {
if pet_body_id_to_vector_id.is_empty() {
return Ok(());
}
- self.write_batches_committing_each(
+ self.write_batch_atomic(
"UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",
- const { NonZeroUsize::new(500).unwrap() },
pet_body_id_to_vector_id
.iter()
.map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),