π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust][photos] Simplify the SQLite wrapper
laurenspriemf7af7c1a4e3a
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rsindex e5dbb7fe24..e7897f4ddc 100644--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs@@ -1,7 +1,6 @@use std::collections::{HashMap, HashSet};use std::sync::OnceLock;-use ente_photos::db;use ente_photos::ml_db;pub use ente_photos::ml_db::{ClipEmbedding, ClipRow, ClusterCentroidRow, ClusterSummary, EmbeddingVector,@@ -27,7 +26,7 @@ impl From<ml_db::Error> for MlDbError {fn from(error: ml_db::Error) -> Self {let message = ente_core::error::chain(&error);match error {- ml_db::Error::Database(db::Error::Downgrade { .. }) => Self::Downgrade { message },+ ml_db::Error::Downgrade { .. } => Self::Downgrade { message },_ => Self::Other { message },}}@@ -847,3 +846,30 @@ impl MlDb {.collect())}}++#[cfg(test)]+mod tests {+ use super::{MlDbError, ml_db};++ #[test]+ fn downgrade_errors_keep_the_bridge_variant_and_message() {+ let error = MlDbError::from(ml_db::Error::Downgrade {+ current: 16,+ target: 15,+ });+ assert!(matches!(+ error,+ MlDbError::Downgrade { message }+ if message == "currentVersion(16) cannot be greater than toVersion(15)"+ ));+ }++ #[test]+ fn other_errors_keep_the_bridge_variant_and_message() {+ let error = MlDbError::from(ml_db::Error::InvalidArgument("invalid value".into()));+ assert!(matches!(+ error,+ MlDbError::Other { message } if message == "invalid value"+ ));+ }+}diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rsindex 54450b8175..66fe5cba7b 100644--- a/rust/crates/photos/src/db.rs+++ b/rust/crates/photos/src/db.rs@@ -1,83 +1,24 @@-use std::collections::HashMap;-use std::hash::Hash;use std::num::NonZeroUsize;-use std::ops::{Deref, DerefMut};use std::path::Path;use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};use std::time::Duration;-use rusqlite::{OpenFlags, TransactionBehavior, types::FromSql};+pub use rusqlite;-pub use rusqlite::types;-pub use rusqlite::{- Error as SqliteError, OptionalExtension, Params, Result as SqliteResult, Row, ToSql,- Transaction, params_from_iter,-};+use rusqlite::{Connection, OpenFlags, Result, Transaction, TransactionBehavior};-const WRITER_PRAGMAS: &str = "PRAGMA journal_mode = WAL;-PRAGMA synchronous = NORMAL;-PRAGMA journal_size_limit = 6291456;";const BUSY_TIMEOUT: Duration = Duration::from_secs(30);-pub(crate) const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;--#[derive(Debug, thiserror::Error)]-pub enum Error {- #[error(transparent)]- Sqlite(#[from] SqliteError),- #[error("currentVersion({current}) cannot be greater than toVersion({target})")]- Downgrade { current: i64, target: i64 },- #[error("reader_count must be at least 1")]- InvalidReaderCount,-}--pub type Result<T> = std::result::Result<T, Error>;#[derive(Clone, Copy, Debug)]pub struct OpenOptions {- pub reader_count: usize,+ pub reader_count: NonZeroUsize,}impl Default for OpenOptions {fn default() -> Self {- Self { reader_count: 1 }- }-}--pub struct Connection(rusqlite::Connection);--impl Connection {- pub fn open(path: impl AsRef<Path>) -> Result<Self> {- let connection = rusqlite::Connection::open(path)?;- connection.busy_timeout(BUSY_TIMEOUT)?;- connection.execute_batch(WRITER_PRAGMAS)?;- Ok(Self(connection))- }-- pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self> {- let flags = OpenFlags::SQLITE_OPEN_READ_ONLY- | OpenFlags::SQLITE_OPEN_URI- | OpenFlags::SQLITE_OPEN_NO_MUTEX;- let connection = rusqlite::Connection::open_with_flags(path, flags)?;- connection.busy_timeout(BUSY_TIMEOUT)?;- Ok(Self(connection))- }-- pub fn open_in_memory() -> Result<Self> {- Ok(Self(rusqlite::Connection::open_in_memory()?))- }-}--impl Deref for Connection {- type Target = rusqlite::Connection;-- fn deref(&self) -> &Self::Target {- &self.0- }-}--impl DerefMut for Connection {- fn deref_mut(&mut self) -> &mut Self::Target {- &mut self.0+ Self {+ reader_count: NonZeroUsize::MIN,+ }}}@@ -86,114 +27,37 @@ pub struct Database {}impl Database {- pub fn open(path: impl AsRef<Path>, migration_scripts: &[&str]) -> Result<Self> {- Self::open_with_options(path, migration_scripts, OpenOptions::default())- }-- pub fn open_with_options(+ pub fn open<E: From<rusqlite::Error>>(path: impl AsRef<Path>,- migration_scripts: &[&str],options: OpenOptions,- ) -> Result<Self> {- if options.reader_count == 0 {- return Err(Error::InvalidReaderCount);- }+ initialize: impl FnOnce(&mut Connection) -> std::result::Result<(), E>,+ ) -> std::result::Result<Self, E> {let path = path.as_ref();let mut writer = Connection::open(path)?;- migrate(&mut writer, migration_scripts)?;- let readers = (0..options.reader_count)- .map(|_| Connection::open_read_only(path))+ writer.busy_timeout(BUSY_TIMEOUT)?;+ writer.pragma_update(None, "journal_mode", "WAL")?;+ initialize(&mut writer)?;+ let readers = (0..options.reader_count.get())+ .map(|_| open_reader(path)).collect::<Result<Vec<_>>>()?;Ok(Self {pool: Pool::new(writer, readers),})}- pub fn read_all<C: FromIterator<T>, T, P: Params>(- &self,- sql: &str,- parameters: P,- map: impl FnMut(&Row<'_>) -> SqliteResult<T>,- ) -> Result<C> {- self.pool.read(|connection| {- let mut statement = connection.prepare_cached(sql)?;- let rows = statement.query_map(parameters, map)?;- Ok(rows.collect::<SqliteResult<C>>()?)- })- }-- pub fn read_column<C: FromIterator<T>, T: FromSql, P: Params>(- &self,- sql: &str,- parameters: P,- ) -> Result<C> {- self.read_all(sql, parameters, |row| row.get(0))- }-- pub fn read_grouped<K: FromSql + Eq + Hash, V: FromSql, C: FromIterator<V>, P: Params>(- &self,- sql: &str,- parameters: P,- ) -> Result<HashMap<K, C>> {- let rows: Vec<(K, V)> = self.read_all(sql, parameters, pair)?;- Ok(group_into(rows))+ pub fn read<T>(&self, read: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {+ self.pool.read(read)}- pub fn read_value<T: FromSql>(&self, sql: &str, parameters: impl Params) -> Result<T> {- self.pool.read(|connection| {- Ok(connection- .prepare_cached(sql)?- .query_row(parameters, |row| row.get(0))?)- })- }-- pub fn read_optional<T: FromSql>(- &self,- sql: &str,- parameters: impl Params,- ) -> Result<Option<T>> {- self.pool.read(|connection| {- Ok(connection- .prepare_cached(sql)?- .query_row(parameters, |row| row.get(0))- .optional()?)- })- }-- pub fn read_chunked_in<C: FromIterator<T>, T, I: ToSql>(- &self,- sql: &str,- ids: &[I],- chunk_size: NonZeroUsize,- mut map: impl FnMut(&Row<'_>) -> SqliteResult<T>,- ) -> Result<C> {- let mut rows = Vec::new();- for chunk in ids.chunks(chunk_size.get()) {- let sql = expand_in_clause(sql, chunk.len());- let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;- rows.extend(chunk_rows);- }- Ok(rows.into_iter().collect())- }-- pub fn read_transaction<T>(- &self,- read: impl FnOnce(&Transaction<'_>) -> Result<T>,- ) -> Result<T> {- self.pool.read(|connection| {- let transaction =- connection.transaction_with_behavior(TransactionBehavior::Deferred)?;- let result = read(&transaction)?;- transaction.commit()?;- Ok(result)- })+ pub fn write<T>(&self, write: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {+ self.pool.write(write)}pub fn write_transaction<T>(&self,write: impl FnOnce(&Transaction<'_>) -> Result<T>,) -> Result<T> {- self.pool.write(|connection| {+ self.write(|connection| {let transaction =connection.transaction_with_behavior(TransactionBehavior::Immediate)?;let result = write(&transaction)?;@@ -201,73 +65,15 @@ impl Database {Ok(result)})}+}- pub fn execute(&self, sql: &str, parameters: impl Params) -> Result<usize> {- self.pool- .write(|connection| Ok(connection.prepare_cached(sql)?.execute(parameters)?))- }-- pub fn execute_chunked_in<I: ToSql>(&self, sql: &str, ids: &[I]) -> Result<()> {- if ids.is_empty() {- return Ok(());- }- self.write_transaction(|transaction| {- for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {- let sql = expand_in_clause(sql, chunk.len());- transaction.execute(&sql, params_from_iter(chunk))?;- }- Ok(())- })- }-- pub fn execute_statements<'a>(- &self,- statements: impl IntoIterator<Item = &'a str>,- ) -> Result<()> {- self.write_transaction(|transaction| {- for statement in statements {- transaction.execute_batch(statement)?;- }- Ok(())- })- }-- pub fn write_batch_atomic<P: Params>(- &self,- sql: &str,- parameter_sets: impl IntoIterator<Item = P>,- ) -> Result<()> {- self.write_transaction(|transaction| {- let mut statement = transaction.prepare_cached(sql)?;- for parameters in parameter_sets {- statement.execute(parameters)?;- }- Ok(())- })- }-- pub fn write_batches_committing_each<P: Params>(- &self,- sql: &str,- batch_size: NonZeroUsize,- parameter_sets: impl IntoIterator<Item = P>,- ) -> Result<()> {- self.pool.write(|connection| {- let mut parameter_sets = parameter_sets.into_iter().peekable();- while parameter_sets.peek().is_some() {- let transaction =- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;- {- let mut statement = transaction.prepare_cached(sql)?;- for parameters in parameter_sets.by_ref().take(batch_size.get()) {- statement.execute(parameters)?;- }- }- transaction.commit()?;- }- Ok(())- })- }+fn open_reader(path: &Path) -> Result<Connection> {+ let flags = OpenFlags::SQLITE_OPEN_READ_ONLY+ | OpenFlags::SQLITE_OPEN_URI+ | OpenFlags::SQLITE_OPEN_NO_MUTEX;+ let connection = Connection::open_with_flags(path, flags)?;+ connection.busy_timeout(BUSY_TIMEOUT)?;+ Ok(connection)}struct Pool {@@ -338,183 +144,106 @@ fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {mutex.lock().unwrap_or_else(PoisonError::into_inner)}-fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<()> {- let target = scripts.len() as i64;- let probed: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?;- check_not_downgrade(probed, target)?;- if probed == target {- return Ok(());- }-- let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;- let current: i64 = transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;- check_not_downgrade(current, target)?;- if current == target {- return Ok(());- }- for script in &scripts[current as usize..] {- transaction.execute_batch(script)?;- }- transaction.pragma_update(None, "user_version", target)?;- transaction.commit()?;- Ok(())-}--fn check_not_downgrade(current: i64, target: i64) -> Result<()> {- if current > target {- return Err(Error::Downgrade { current, target });- }- Ok(())-}--pub fn bind_placeholders(count: usize) -> String {- vec!["?"; count].join(", ")-}--fn expand_in_clause(sql: &str, count: usize) -> String {- sql.replacen("{}", &bind_placeholders(count), 1)-}--pub(crate) fn pair<A: types::FromSql, B: types::FromSql>(row: &Row<'_>) -> SqliteResult<(A, B)> {- Ok((row.get(0)?, row.get(1)?))-}--pub(crate) fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql> {- value.iter().map(|value| value as &dyn ToSql).collect()-}--pub(crate) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(- rows: impl IntoIterator<Item = (K, V)>,-) -> HashMap<K, C> {- let mut groups: HashMap<K, Vec<V>> = HashMap::new();- for (key, value) in rows {- groups.entry(key).or_default().push(value);- }- groups- .into_iter()- .map(|(key, values)| (key, values.into_iter().collect()))- .collect()-}-#[cfg(test)]mod tests {- use std::collections::{HashMap, HashSet};use std::num::NonZeroUsize;use std::panic::{AssertUnwindSafe, catch_unwind};use std::sync::mpsc::{self, RecvTimeoutError};use std::thread;use std::time::Duration;- use super::{- Connection, Database, Error, MAX_SQL_BIND_PARAMS_PER_QUERY, OpenOptions, bind_placeholders,- lock, types,- };+ use rusqlite::{Connection, Result};- const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";- const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";+ use super::{Database, OpenOptions, lock, open_reader};fn open() -> (tempfile::TempDir, Database) {let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ let db = Database::open(+ directory.path().join("items.db"),+ OpenOptions::default(),+ |connection| connection.execute_batch("CREATE TABLE items (id INTEGER PRIMARY KEY)"),+ )+ .unwrap();(directory, db)}- fn pragma<T: types::FromSql>(connection: &Connection, name: &str) -> T {- connection- .pragma_query_value(None, name, |row| row.get(0))+ fn value(db: &Database, sql: &str) -> i64 {+ db.read(|connection| connection.query_row(sql, (), |row| row.get(0))).unwrap()}#[test]- fn open_applies_connection_pragmas() {- let directory = tempfile::tempdir().unwrap();- let connection = Connection::open(directory.path().join("test.db")).unwrap();- assert_eq!(pragma::<String>(&connection, "journal_mode"), "wal");- assert_eq!(pragma::<i64>(&connection, "synchronous"), 1);- assert_eq!(pragma::<i64>(&connection, "journal_size_limit"), 6291456);- assert_eq!(pragma::<i64>(&connection, "busy_timeout"), 30000);+ fn database_is_send_and_sync() {+ fn assert_send_sync<T: Send + Sync>() {}+ assert_send_sync::<Database>();}#[test]- fn read_only_connection_reads_wal_without_changing_writer_settings() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("test.db");- let writer = Connection::open(&path).unwrap();- writer- .execute_batch("CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (7);")- .unwrap();- let reader = Connection::open_read_only(&path).unwrap();- let id: i64 = reader- .query_row("SELECT id FROM t", (), |row| row.get(0))- .unwrap();- assert_eq!(id, 7);- assert!(reader.execute("INSERT INTO t VALUES (8)", ()).is_err());- assert_eq!(pragma::<String>(&reader, "journal_mode"), "wal");- assert_eq!(pragma::<i64>(&reader, "busy_timeout"), 30000);- assert_eq!(pragma::<i64>(&writer, "synchronous"), 1);- assert_eq!(pragma::<i64>(&writer, "journal_size_limit"), 6291456);+ fn open_initializes_before_readers_and_configures_connections() {+ let (_directory, db) = open();+ db.write(|connection| {+ let journal: String =+ connection.pragma_query_value(None, "journal_mode", |row| row.get(0))?;+ let timeout: i64 =+ connection.pragma_query_value(None, "busy_timeout", |row| row.get(0))?;+ assert_eq!(journal, "wal");+ assert_eq!(timeout, 30000);+ connection.execute("INSERT INTO items VALUES (7)", ())+ })+ .unwrap();+ assert_eq!(value(&db, "SELECT id FROM items"), 7);+ assert_eq!(value(&db, "PRAGMA busy_timeout"), 30000);+ assert!(+ db.read(|connection| connection.execute("DELETE FROM items", ()))+ .is_err()+ );+ assert_eq!(value(&db, "SELECT id FROM items"), 7);}#[test]- fn read_only_connection_does_not_create_database() {+ fn open_preserves_an_existing_schema_version() {let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("missing.db");- assert!(Connection::open_read_only(&path).is_err());- assert!(!path.exists());+ let path = directory.path().join("items.db");+ {+ let connection = Connection::open(&path).unwrap();+ connection.execute_batch("CREATE TABLE items (id INTEGER); INSERT INTO items VALUES (7); PRAGMA user_version = 42;").unwrap();+ }+ let db = Database::open(&path, OpenOptions::default(), |_| {+ Ok::<_, rusqlite::Error>(())+ })+ .unwrap();+ assert_eq!(value(&db, "PRAGMA user_version"), 42);+ assert_eq!(value(&db, "SELECT id FROM items"), 7);}#[test]- fn in_memory_connection_runs_statements() {- let connection = Connection::open_in_memory().unwrap();- connection- .execute_batch("CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (7);")- .unwrap();- let id: i64 = connection- .query_row("SELECT id FROM t", (), |row| row.get(0))- .unwrap();- assert_eq!(id, 7);- }+ fn open_propagates_the_initializers_error_type() {+ #[derive(Debug, PartialEq, Eq)]+ struct InitializationError(String);- #[test]- fn reader_queries_reject_writes() {- let (_directory, db) = open();- db.execute("INSERT INTO items (id) VALUES (?)", [7])- .unwrap();- let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());- assert!(matches!(deleted, Err(Error::Sqlite(_))));- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())- .unwrap(),- [7]+ impl From<rusqlite::Error> for InitializationError {+ fn from(error: rusqlite::Error) -> Self {+ Self(error.to_string())+ }+ }++ let directory = tempfile::tempdir().unwrap();+ let result = Database::open(+ directory.path().join("items.db"),+ OpenOptions::default(),+ |_| Err(InitializationError("schema initialization failed".into())),+ );+ assert!(+ matches!(result, Err(InitializationError(message)) if message == "schema initialization failed"));}#[test]- fn failed_batch_rolls_back_and_writer_recovers() {- let (_directory, db) = open();- assert!(- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]])- .is_err()- );- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );- assert_eq!(- db.execute("INSERT INTO items (id) VALUES (?)", [3])- .unwrap(),- 1- );- assert_eq!(- db.execute("DELETE FROM items WHERE id = ?", [4]).unwrap(),- 0- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())- .unwrap(),- [3]- );+ fn read_only_connection_does_not_create_database() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("missing.db");+ assert!(open_reader(&path).is_err());+ assert!(!path.exists());}#[test]@@ -526,51 +255,40 @@ mod tests {Ok(())});assert!(result.is_err());- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );+ assert_eq!(value(&db, "SELECT COUNT(*) FROM items"), 0);assert!(catch_unwind(AssertUnwindSafe(|| {- let _: super::Result<()> = db.write_transaction(|transaction| {+ let _: Result<()> = db.write_transaction(|transaction| {transaction.execute("INSERT INTO items VALUES (?)", [3])?;panic!("transaction callback failed");});})).is_err());- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );+ assert_eq!(value(&db, "SELECT COUNT(*) FROM items"), 0);let total = db.write_transaction(|transaction| {transaction.execute("INSERT INTO items VALUES (?)", [1])?;transaction.execute("INSERT INTO items VALUES (?)", [2])?;- Ok(transaction- .query_row("SELECT SUM(id) FROM items", (), |row| row.get::<_, i64>(0))?)+ transaction.query_row("SELECT SUM(id) FROM items", (), |row| row.get::<_, i64>(0))}).unwrap();assert_eq!(total, 3);- assert_eq!(- db.read_value::<i64>("SELECT SUM(id) FROM items", ())- .unwrap(),- 3- );+ assert_eq!(value(&db, "SELECT SUM(id) FROM items"), 3);}#[test]- fn read_transaction_keeps_a_snapshot_and_rejects_writes() {+ fn reader_can_use_a_snapshot_while_the_writer_commits() {let (_directory, db) = open();- db.execute("INSERT INTO items VALUES (?)", [1]).unwrap();+ db.write(|connection| connection.execute("INSERT INTO items VALUES (?)", [1]))+ .unwrap();let counts = db- .read_transaction(|transaction| {+ .read(|connection| {+ let transaction = connection.transaction()?;let before: i64 =transaction.query_row("SELECT COUNT(*) FROM items", (), |row| row.get(0))?;- db.execute("INSERT INTO items VALUES (?)", [2])?;+ db.write(|writer| writer.execute("INSERT INTO items VALUES (?)", [2]))?;let after: i64 =transaction.query_row("SELECT COUNT(*) FROM items", (), |row| row.get(0))?;assert!(@@ -578,55 +296,25 @@ mod tests {.execute("INSERT INTO items VALUES (?)", [3]).is_err());+ transaction.commit()?;Ok((before, after))}).unwrap();assert_eq!(counts, (1, 1));- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 2- );- }-- #[test]- fn statement_sequence_rolls_back_if_a_later_statement_fails() {- let (_directory, db) = open();- assert!(- db.execute_statements([- "INSERT INTO items VALUES (1)",- "INSERT INTO missing_table VALUES (2)",- ])- .is_err()- );- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );- }-- #[test]- fn zero_readers_is_rejected_before_creating_a_database() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("items.db");- assert!(matches!(- Database::open_with_options(&path, &[CREATE_ITEMS], OpenOptions { reader_count: 0 }),- Err(Error::InvalidReaderCount)- ));- assert!(!path.exists());+ assert_eq!(value(&db, "SELECT COUNT(*) FROM items"), 2);}#[test]fn configured_readers_bound_concurrency_and_wake_on_any_returned_connection() {for reader_count in [1, 2, 3] {let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("items.db");- let db = if reader_count == 1 {- Database::open(path, &[CREATE_ITEMS])- } else {- Database::open_with_options(path, &[CREATE_ITEMS], OpenOptions { reader_count })- }+ let db = Database::open(+ directory.path().join("items.db"),+ OpenOptions {+ reader_count: NonZeroUsize::new(reader_count).unwrap(),+ },+ |_| Ok::<_, rusqlite::Error>(()),+ ).unwrap();thread::scope(|scope| {@@ -637,8 +325,8 @@ mod tests {release_readers.push(release_tx);let db = &db;scope.spawn(move || {- db.read_transaction(|transaction| {- transaction.query_row("SELECT 1", (), |_| Ok(()))?;+ db.read(|connection| {+ connection.query_row("SELECT 1", (), |_| Ok(()))?;entered_tx.send(()).unwrap();release_rx.recv().unwrap();Ok(())@@ -651,9 +339,7 @@ mod tests {let (result_tx, result_rx) = mpsc::channel();let db = &db;scope.spawn(move || {- result_tx- .send(db.read_value::<i64>("SELECT 7", ()))- .unwrap();+ result_tx.send(value(db, "SELECT 7")).unwrap();});let was_blocked = matches!(result_rx.recv_timeout(Duration::from_millis(100)),@@ -665,7 +351,7 @@ mod tests {release.send(()).unwrap();}assert!(was_blocked);- assert_eq!(result.unwrap().unwrap(), 7);+ assert_eq!(result.unwrap(), 7);});}}@@ -675,284 +361,14 @@ mod tests {let (_directory, db) = open();assert!(catch_unwind(AssertUnwindSafe(|| {- let _: super::Result<()> = db.read_transaction(|transaction| {- transaction.query_row("SELECT 1", (), |_| Ok(()))?;+ let _: Result<()> = db.read(|connection| {+ connection.query_row("SELECT 1", (), |_| Ok(()))?;panic!("reader callback failed");});})).is_err());assert_eq!(lock(&db.pool.readers).len(), 1);- assert_eq!(db.read_value::<i64>("SELECT 7", ()).unwrap(), 7);- }-- #[test]- fn chunked_delete_handles_empty_lists_and_rolls_back_on_failure() {- let (_directory, db) = open();- let ids: Vec<i64> = (0..10_001).collect();- db.write_batch_atomic(- "INSERT INTO items (id) VALUES (?)",- ids.iter().map(|id| [id]),- )- .unwrap();- db.execute_chunked_in::<i64>("DELETE FROM items WHERE id IN ({})", &[])- .unwrap();- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 10_001- );- db.execute_statements(["CREATE TRIGGER reject_last_delete BEFORE DELETE ON items- WHEN OLD.id = 10000 BEGIN SELECT RAISE(ABORT, 'cannot delete'); END"])- .unwrap();- assert!(- db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)- .is_err()- );- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 10_001- );- db.execute_statements(["DROP TRIGGER reject_last_delete"])- .unwrap();- db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)- .unwrap();- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 0- );- }-- #[test]- fn batched_writes_commit_each_completed_batch() {- let (_directory, db) = open();- db.write_batches_committing_each(- "INSERT INTO missing_table VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- Vec::<[i64; 1]>::new(),- )- .unwrap();- assert!(- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- [[1], [2], [3], [4], [1]]- )- .is_err()- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- [1, 2, 3, 4]- );- assert!(- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- [[5], [6], [7], [5], [8]]- )- .is_err()- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- [1, 2, 3, 4, 5, 6]- );- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- const { NonZeroUsize::new(2).unwrap() },- [[7], [8], [9]],- )- .unwrap();- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- [1, 2, 3, 4, 5, 6, 7, 8, 9]- );- assert!(- db.write_batches_committing_each(- "INSERT INTO items (id) VALUES (?)",- NonZeroUsize::MIN,- [[10], [11], [10]],- )- .is_err()- );- assert_eq!(- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())- .unwrap(),- (1..=11).collect::<Vec<_>>()- );- }-- #[test]- fn chunked_reads_query_each_chunk_in_order() {- let (_directory, db) = open();- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))- .unwrap();- let ids: Vec<i64> = (1..=7).collect();- let one_at_a_time: Vec<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",- &ids,- NonZeroUsize::MIN,- |row| row.get(0),- )- .unwrap();- assert_eq!(one_at_a_time, ids);- let by_chunk: Vec<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",- &ids,- const { NonZeroUsize::new(3).unwrap() },- |row| row.get(0),- )- .unwrap();- assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);- let single_chunk: Vec<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",- &ids,- const { NonZeroUsize::new(10).unwrap() },- |row| row.get(0),- )- .unwrap();- assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);- let none: Vec<i64> = db- .read_chunked_in(- "SELECT id FROM missing_table WHERE id IN ({})",- &Vec::<i64>::new(),- const { NonZeroUsize::new(3).unwrap() },- |row| row.get(0),- )- .unwrap();- assert!(none.is_empty());- let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();- let found: HashSet<i64> = db- .read_chunked_in(- "SELECT id FROM items WHERE id IN ({})",- &many,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },- |row| row.get(0),- )- .unwrap();- assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));- }-- #[test]- fn grouped_reads_collect_values_per_key() {- let (_directory, db) = open();- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=5).map(|id| [id]))- .unwrap();- let by_parity: HashMap<i64, Vec<i64>> = db- .read_grouped("SELECT id % 2, id FROM items ORDER BY id", ())- .unwrap();- assert_eq!(- by_parity,- HashMap::from([(1, vec![1, 3, 5]), (0, vec![2, 4])])- );- let by_label: HashMap<String, HashSet<i64>> = db- .read_grouped(- "SELECT CASE WHEN id < 3 THEN 'low' ELSE 'high' END, id FROM items",- (),- )- .unwrap();- assert_eq!(- by_label,- HashMap::from([- ("low".to_string(), HashSet::from([1, 2])),- ("high".to_string(), HashSet::from([3, 4, 5]))- ])- );- let empty: HashMap<i64, Vec<i64>> = db- .read_grouped("SELECT id, id FROM items WHERE id > 5", ())- .unwrap();- assert!(empty.is_empty());- }-- #[test]- fn open_migrates_a_caller_schema_and_refuses_downgrade() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("items.db");- {- let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();- db.execute("INSERT INTO items (id) VALUES (?)", [7])- .unwrap();- }- {- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();- assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);- assert_eq!(- db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [7])- .unwrap(),- Some("item".to_string())- );- assert_eq!(- db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [8])- .unwrap(),- None- );- }- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();- assert_eq!(- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())- .unwrap(),- 1- );- assert!(matches!(- Database::open(&path, &[CREATE_ITEMS]),- Err(Error::Downgrade {- current: 2,- target: 1- })- ));- }-- #[test]- fn failed_migration_keeps_previous_schema_and_version() {- let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("items.db");- {- let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();- db.execute("INSERT INTO items (id) VALUES (?)", [7])- .unwrap();- }- assert!(- Database::open(- &path,- &[- CREATE_ITEMS,- ADD_LABEL,- "INSERT INTO missing_table VALUES (1)"- ]- )- .is_err()- );- {- let connection = Connection::open(&path).unwrap();- let version: i64 = connection- .pragma_query_value(None, "user_version", |row| row.get(0))- .unwrap();- assert_eq!(version, 1);- let columns: Vec<String> = connection- .prepare("SELECT name FROM pragma_table_info('items')")- .unwrap()- .query_map((), |row| row.get(0))- .unwrap()- .collect::<Result<_, _>>()- .unwrap();- assert_eq!(columns, ["id"]);- }- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();- assert_eq!(db.read_value::<i64>("SELECT id FROM items", ()).unwrap(), 7);- assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);- }-- #[test]- fn bind_placeholders_joins_question_marks() {- assert_eq!(bind_placeholders(0), "");- assert_eq!(bind_placeholders(1), "?");- assert_eq!(bind_placeholders(3), "?, ?, ?");+ assert_eq!(value(&db, "SELECT 7"), 7);}}diff --git a/rust/crates/photos/src/ml_db/migrations.rs b/rust/crates/photos/src/ml_db/migrations.rsnew file mode 100644index 0000000000..eed97c5b83--- /dev/null+++ b/rust/crates/photos/src/ml_db/migrations.rs@@ -0,0 +1,154 @@+use rusqlite::{Connection, TransactionBehavior};++use super::{Error, Result};++pub(super) fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<()> {+ let target = scripts.len() as i64;+ let probed: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?;+ check_not_downgrade(probed, target)?;+ if probed == target {+ return Ok(());+ }++ let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;+ let current: i64 = transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;+ check_not_downgrade(current, target)?;+ if current == target {+ return Ok(());+ }+ for script in &scripts[current as usize..] {+ transaction.execute_batch(script)?;+ }+ transaction.pragma_update(None, "user_version", target)?;+ transaction.commit()?;+ Ok(())+}++fn check_not_downgrade(current: i64, target: i64) -> Result<()> {+ if current > target {+ return Err(Error::Downgrade { current, target });+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use std::path::Path;+ use std::sync::Barrier;+ use std::thread;++ use crate::db::{Database, OpenOptions};+ use rusqlite::Connection;++ use super::migrate;+ use crate::ml_db::{Error, Result};++ const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";+ const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";++ fn open(path: &Path, scripts: &[&str]) -> Result<Database> {+ Database::open(path, OpenOptions::default(), |connection| {+ migrate(connection, scripts)+ })+ }++ #[test]+ fn upgrades_keep_data_and_refuse_downgrades() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("items.db");+ {+ let db = open(&path, &[CREATE_ITEMS]).unwrap();+ db.write(|connection| connection.execute("INSERT INTO items VALUES (7)", ()))+ .unwrap();+ }+ let db = open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();+ let (version, label): (i64, String) = db+ .read(|connection| {+ Ok((+ connection.pragma_query_value(None, "user_version", |row| row.get(0))?,+ connection+ .query_row("SELECT label FROM items WHERE id = 7", (), |row| row.get(0))?,+ ))+ })+ .unwrap();+ assert_eq!(version, 2);+ assert_eq!(label, "item");+ assert!(matches!(+ open(&path, &[CREATE_ITEMS]),+ Err(Error::Downgrade {+ current: 2,+ target: 1+ })+ ));+ }++ #[test]+ fn failed_migration_keeps_previous_schema_and_version() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("items.db");+ {+ let db = open(&path, &[CREATE_ITEMS]).unwrap();+ db.write(|connection| connection.execute("INSERT INTO items VALUES (7)", ()))+ .unwrap();+ }+ assert!(+ open(+ &path,+ &[+ CREATE_ITEMS,+ ADD_LABEL,+ "INSERT INTO missing_table VALUES (1)"+ ]+ )+ .is_err()+ );+ {+ let connection = Connection::open(&path).unwrap();+ let version: i64 = connection+ .pragma_query_value(None, "user_version", |row| row.get(0))+ .unwrap();+ let columns = connection+ .prepare("SELECT name FROM pragma_table_info('items')")+ .unwrap()+ .query_map((), |row| row.get::<_, String>(0))+ .unwrap()+ .collect::<rusqlite::Result<Vec<_>>>()+ .unwrap();+ assert_eq!(version, 1);+ assert_eq!(columns, ["id"]);+ }+ let db = open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();+ assert_eq!(+ db.read(|connection| connection+ .query_row("SELECT id FROM items", (), |row| row.get::<_, i64>(0)))+ .unwrap(),+ 7+ );+ }++ #[test]+ fn concurrent_open_applies_each_migration_once() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("items.db");+ drop(open(&path, &[CREATE_ITEMS]).unwrap());+ let barrier = Barrier::new(2);+ thread::scope(|scope| {+ for _ in 0..2 {+ let path = &path;+ let barrier = &barrier;+ scope.spawn(move || {+ barrier.wait();+ let db = open(path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();+ let version = db+ .read(|connection| {+ connection.pragma_query_value(None, "user_version", |row| {+ row.get::<_, i64>(0)+ })+ })+ .unwrap();+ assert_eq!(version, 2);+ });+ }+ });+ }+}diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rsindex 6a4a95f4ca..7426d3049b 100644--- a/rust/crates/photos/src/ml_db/mod.rs+++ b/rust/crates/photos/src/ml_db/mod.rs@@ -1,7 +1,9 @@+mod migrations;mod queries;mod schema;mod vector_encoding;+use std::num::NonZeroUsize;use std::path::Path;use crate::db::{Database, OpenOptions};@@ -26,7 +28,9 @@ pub use vector_encoding::{decode_evector, decode_f32, encode_evector, encode_f32#[derive(Debug, thiserror::Error)]pub enum Error {#[error(transparent)]- Database(#[from] crate::db::Error),+ Database(#[from] rusqlite::Error),+ #[error("currentVersion({current}) cannot be greater than toVersion({target})")]+ Downgrade { current: i64, target: i64 },#[error("{0}")]Codec(String),#[error(transparent)]@@ -46,38 +50,43 @@ pub struct MlDb {impl MlDb {pub fn open(path: impl AsRef<Path>) -> Result<Self> {Ok(Self {- db: Database::open_with_options(+ db: Database::open(path,- &schema::MIGRATION_SCRIPTS,- OpenOptions { reader_count: 2 },+ OpenOptions {+ reader_count: const { NonZeroUsize::new(2).unwrap() },+ },+ |connection| {+ connection.execute_batch(+ "PRAGMA synchronous = NORMAL; PRAGMA journal_size_limit = 6291456;",+ )?;+ migrations::migrate(connection, &schema::MIGRATION_SCRIPTS)+ },)?,})}pub fn clear_non_pet_tables(&self) -> Result<()> {- self.db- .execute_statements([- schema::DELETE_FACES,- schema::DELETE_FACE_CLUSTERS,- schema::DELETE_CLUSTER_PERSON,- schema::DELETE_CLUSTER_SUMMARY,- schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,- schema::DELETE_NOT_PERSON_FEEDBACK,- schema::DELETE_CLIP_EMBEDDINGS,- schema::DELETE_FILE_DATA,- ])- .map_err(Into::into)+ self.execute_statements([+ schema::DELETE_FACES,+ schema::DELETE_FACE_CLUSTERS,+ schema::DELETE_CLUSTER_PERSON,+ schema::DELETE_CLUSTER_SUMMARY,+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,+ schema::DELETE_NOT_PERSON_FEEDBACK,+ schema::DELETE_CLIP_EMBEDDINGS,+ schema::DELETE_FILE_DATA,+ ])+ .map_err(Into::into)}pub fn clear_pet_tables(&self) -> Result<()> {- self.db- .execute_statements([- schema::DELETE_PET_FACES,- schema::DELETE_PET_BODIES,- schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,- schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,- ])- .map_err(Into::into)+ self.execute_statements([+ schema::DELETE_PET_FACES,+ schema::DELETE_PET_BODIES,+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,+ ])+ .map_err(Into::into)}}@@ -90,7 +99,7 @@ mod tests {use super::queries::clip::tests::full_clip;use super::queries::{caches, clip, clusters, faces, filedata, persons, pets};use super::{Error, MlDb, TARGET_VERSION};- use crate::db::Connection;+ use rusqlite::Connection;use tempfile::TempDir;type Query<T> = fn(&MlDb) -> Result<T, Error>;@@ -254,6 +263,22 @@ mod tests {);}+ #[test]+ fn ml_connections_keep_existing_settings() {+ let (_directory, db) = open();+ let (synchronous, journal_size_limit): (i64, i64) = db+ .db+ .write(|connection| {+ Ok((+ connection.pragma_query_value(None, "synchronous", |row| row.get(0))?,+ connection.pragma_query_value(None, "journal_size_limit", |row| row.get(0))?,+ ))+ })+ .unwrap();+ assert_eq!(synchronous, 1);+ assert_eq!(journal_size_limit, 6291456);+ }+#[test]fn ml_db_is_send_and_sync() {fn assert_send_sync<T: Send + Sync>() {}@@ -319,7 +344,7 @@ mod tests {.pragma_update(None, "user_version", 16).unwrap();match MlDb::open(&path) {- Err(Error::Database(crate::db::Error::Downgrade { current, target })) => {+ Err(Error::Downgrade { current, target }) => {assert_eq!(current, 16);assert_eq!(target, 15);}diff --git a/rust/crates/photos/src/ml_db/queries/caches.rs b/rust/crates/photos/src/ml_db/queries/caches.rsindex 4e153dba4e..d9a913ffb1 100644--- a/rust/crates/photos/src/ml_db/queries/caches.rs+++ b/rust/crates/photos/src/ml_db/queries/caches.rs@@ -9,7 +9,7 @@ const THREE_MONTHS_MILLIS: i64 = 90 * 24 * 60 * 60 * 1000;impl MlDb {pub fn put_repeated_text_embedding_cache(&self, query: &str, embedding: &[f64]) -> Result<()> {let embedding_bytes = encode_f32(embedding.iter().map(|value| *value as f32));- self.db.execute(+ self.execute(r#"INSERT OR REPLACE INTO text_embeddings_cache (text_query, embedding, ml_version, created_at)VALUES (?, ?, ?, ?)@@ -20,7 +20,7 @@ impl MlDb {}pub fn get_repeated_text_embedding_cache(&self, query: &str) -> Result<Option<Vec<f32>>> {- let results: Vec<(Vec<u8>, i64, i64)> = self.db.read_all(+ let results: Vec<(Vec<u8>, i64, i64)> = self.read_all(r#"SELECT embedding, ml_version, created_atFROM text_embeddings_cache@@ -38,7 +38,7 @@ impl MlDb {return Ok(Some(decode_f32(embedding)));}}- self.db.execute(+ self.execute("DELETE FROM text_embeddings_cache WHERE text_query = ?",[query],)?;@@ -50,7 +50,7 @@ impl MlDb {person_or_cluster_id: &str,face_id: &str,) -> Result<()> {- self.db.execute(+ self.execute("INSERT OR REPLACE INTO face_cache (person_or_cluster_id, face_id) VALUES (?, ?)",[person_or_cluster_id, face_id],)?;@@ -61,19 +61,18 @@ impl MlDb {&self,person_or_cluster_id: &str,) -> Result<Option<String>> {- self.db- .read_optional(- "SELECT face_id FROM face_cache WHERE person_or_cluster_id = ?",- [person_or_cluster_id],- )- .map_err(Into::into)+ self.read_optional(+ "SELECT face_id FROM face_cache WHERE person_or_cluster_id = ?",+ [person_or_cluster_id],+ )+ .map_err(Into::into)}pub fn remove_face_id_cached_for_person_or_cluster(&self,person_or_cluster_id: &str,) -> Result<()> {- self.db.execute(+ self.execute("DELETE FROM face_cache WHERE person_or_cluster_id = ?",[person_or_cluster_id],)?;@@ -91,8 +90,8 @@ fn now_millis() -> i64 {#[cfg(test)]pub(in crate::ml_db) mod tests {use super::MlDb;- use crate::db::Connection;use crate::ml_db::tests::{cases, check, open};+ use rusqlite::Connection;use tempfile::TempDir;pub(in crate::ml_db) fn seed(db: &MlDb) {diff --git a/rust/crates/photos/src/ml_db/queries/clip.rs b/rust/crates/photos/src/ml_db/queries/clip.rsindex f170f37a68..aa6c28dd47 100644--- a/rust/crates/photos/src/ml_db/queries/clip.rs+++ b/rust/crates/photos/src/ml_db/queries/clip.rs@@ -1,6 +1,6 @@use std::collections::HashMap;-use crate::db::pair;+use super::helpers::pair;use crate::ml_db::schema;use crate::ml_db::vector_encoding::{decode_f32, encode_f32};use crate::ml_db::{MlDb, Result};@@ -32,8 +32,7 @@ pub struct ClipRow {impl MlDb {pub fn get_all_clip_vectors(&self) -> Result<Vec<EmbeddingVector>> {let rows: Vec<(i64, Vec<u8>)> =- self.db- .read_all("SELECT file_id, embedding FROM clip", (), pair)?;+ self.read_all("SELECT file_id, embedding FROM clip", (), pair)?;Ok(rows.into_iter().map(|(file_id, embedding)| EmbeddingVector {@@ -45,81 +44,73 @@ impl MlDb {}pub fn clip_indexed_file_with_version(&self) -> Result<HashMap<i64, i64>> {- self.db- .read_all("SELECT file_id , ml_version FROM clip", (), pair)+ self.read_all("SELECT file_id , ml_version FROM clip", (), pair).map_err(Into::into)}pub fn get_clip_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {- self.db- .read_value(- "SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ?",- [minimum_ml_version],- )- .map_err(Into::into)+ self.read_value(+ "SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ?",+ [minimum_ml_version],+ )+ .map_err(Into::into)}pub fn get_clip_vectorizable_file_count(&self, minimum_ml_version: i64) -> Result<i64> {- self.db- .read_value(- r#"+ self.read_value(+ r#"SELECT COUNT(DISTINCT file_id) as countFROM clipWHERE ml_version >= ?AND LENGTH(embedding) = ?"#,- [minimum_ml_version, CLIP_EMBEDDING_BYTES_LENGTH],- )- .map_err(Into::into)+ [minimum_ml_version, CLIP_EMBEDDING_BYTES_LENGTH],+ )+ .map_err(Into::into)}pub fn insert_clip_rows(&self, embeddings: &[ClipEmbedding]) -> Result<()> {if let [embedding] = embeddings {- self.db.execute(+ self.execute("INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) VALUES (?, ?, ?)",clip_row(embedding),)?;return Ok(());}- self.db- .write_batch_atomic(- "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) values(?, ?, ?)",- embeddings.iter().map(clip_row),- )- .map_err(Into::into)+ self.write_batch_atomic(+ "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) values(?, ?, ?)",+ embeddings.iter().map(clip_row),+ )+ .map_err(Into::into)}pub fn delete_clip_rows(&self, file_ids: &[i64]) -> Result<()> {- self.db- .execute_chunked_in("DELETE FROM clip WHERE file_id IN ({})", file_ids)+ self.execute_chunked_in("DELETE FROM clip WHERE file_id IN ({})", file_ids).map_err(Into::into)}pub fn delete_all_clip_rows(&self) -> Result<()> {- self.db- .execute_statements([schema::DELETE_CLIP_EMBEDDINGS])+ self.execute_statements([schema::DELETE_CLIP_EMBEDDINGS]).map_err(Into::into)}pub fn count_clip_rows(&self) -> Result<i64> {- self.db- .read_value("SELECT COUNT(file_id) as total FROM clip", ())+ self.read_value("SELECT COUNT(file_id) as total FROM clip", ()).map_err(Into::into)}pub fn get_clip_rows_page(&self, limit: i64, offset: i64) -> Result<Vec<ClipRow>> {- self.db- .read_all(- "SELECT file_id, embedding FROM clip ORDER BY file_id DESC LIMIT ? OFFSET ?",- [limit, offset],- |row| {- Ok(ClipRow {- file_id: row.get(0)?,- embedding: row.get(1)?,- })- },- )- .map_err(Into::into)+ self.read_all(+ "SELECT file_id, embedding FROM clip ORDER BY file_id DESC LIMIT ? OFFSET ?",+ [limit, offset],+ |row| {+ Ok(ClipRow {+ file_id: row.get(0)?,+ embedding: row.get(1)?,+ })+ },+ )+ .map_err(Into::into)}}diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rsindex 976a0fdebf..a4dc7a2195 100644--- a/rust/crates/photos/src/ml_db/queries/clusters.rs+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs@@ -1,10 +1,10 @@use std::collections::{HashMap, HashSet};use std::num::NonZeroUsize;-use crate::db::{- MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, bind_placeholders, group_into,- optional_parameter, pair, params_from_iter,+use super::helpers::{+ MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,};+use rusqlite::{Result as SqliteResult, Row, params_from_iter};use super::faces::{file_id_from_face_id, is_bad_face_for_clustering};use super::unique_in_order;@@ -48,32 +48,30 @@ impl MlDb {&self,face_id_to_cluster_id: &HashMap<String, String>,) -> Result<()> {- self.db- .write_batches_committing_each(- UPSERT_FACE_CLUSTER,- const { NonZeroUsize::new(500).unwrap() },- face_id_to_cluster_id.iter(),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ UPSERT_FACE_CLUSTER,+ const { NonZeroUsize::new(500).unwrap() },+ face_id_to_cluster_id.iter(),+ )+ .map_err(Into::into)}pub fn cluster_id_to_face_count(&self) -> Result<HashMap<String, i64>> {- self.db- .read_all(- r#"+ self.read_all(+ r#"SELECT cluster_id, COUNT(*) as countFROM face_clusterswhere cluster_id IS NOT NULLGROUP BY cluster_id"#,- (),- pair,- )- .map_err(Into::into)+ (),+ pair,+ )+ .map_err(Into::into)}pub fn get_bad_face_singleton_cluster_ids(&self) -> Result<HashSet<String>> {- let rows: Vec<(String, f64, f64, i64)> = self.db.read_all(+ let rows: Vec<(String, f64, f64, i64)> = self.read_all(r#"SELECT fc.cluster_id, f.score, f.blur, f.is_sidewaysFROM face_clusters fc@@ -102,7 +100,7 @@ impl MlDb {return Ok(HashMap::new());}let cluster_id_list: Vec<&String> = cluster_ids.iter().collect();- let rows: Vec<(String, String)> = self.db.read_chunked_in(+ let rows: Vec<(String, String)> = self.read_chunked_in("SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",&cluster_id_list,const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },@@ -112,27 +110,24 @@ impl MlDb {}pub fn get_cluster_id_for_face_id(&self, face_id: &str) -> Result<Option<String>> {- self.db- .read_optional(- "SELECT cluster_id FROM face_clusters WHERE face_id = ?",- [face_id],- )- .map_err(Into::into)+ self.read_optional(+ "SELECT cluster_id FROM face_clusters WHERE face_id = ?",+ [face_id],+ )+ .map_err(Into::into)}pub fn get_all_cluster_id_to_face_ids(&self) -> Result<HashMap<String, Vec<String>>> {- self.db- .read_grouped("SELECT cluster_id, face_id FROM face_clusters", ())+ self.read_grouped("SELECT cluster_id, face_id FROM face_clusters", ()).map_err(Into::into)}pub fn get_face_ids_for_cluster(&self, cluster_id: &str) -> Result<Vec<String>> {- self.db- .read_column(- "SELECT face_id FROM face_clusters WHERE face_clusters.cluster_id = ?",- [cluster_id],- )- .map_err(Into::into)+ self.read_column(+ "SELECT face_id FROM face_clusters WHERE face_clusters.cluster_id = ?",+ [cluster_id],+ )+ .map_err(Into::into)}pub fn get_face_ids_for_cluster_ordered_by_score(@@ -140,9 +135,8 @@ impl MlDb {cluster_id: &str,limit: i64,) -> Result<Vec<String>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT faces.face_idFROM facesJOIN face_clusters@@ -151,13 +145,13 @@ impl MlDb {ORDER BY faces.score DESCLIMIT ?"#,- (cluster_id, limit),- )- .map_err(Into::into)+ (cluster_id, limit),+ )+ .map_err(Into::into)}pub fn get_blur_values_for_cluster(&self, cluster_id: &str) -> Result<Vec<f64>> {- let blur_values: Vec<f64> = self.db.read_column(+ let blur_values: Vec<f64> = self.read_column(r#"SELECT faces.blurFROM faces@@ -178,19 +172,17 @@ impl MlDb {&self,face_ids: &[String],) -> Result<HashMap<String, Option<String>>> {- self.db- .read_chunked_in(- "SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",- face_ids,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },- pair,- )- .map_err(Into::into)+ self.read_chunked_in(+ "SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",+ face_ids,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },+ pair,+ )+ .map_err(Into::into)}pub fn get_file_id_to_cluster_ids(&self) -> Result<HashMap<i64, HashSet<String>>> {- self.db- .read_all("SELECT cluster_id, face_id FROM face_clusters", (), pair)+ self.read_all("SELECT cluster_id, face_id FROM face_clusters", (), pair).map_err(Into::into).and_then(file_id_to_cluster_ids)}@@ -199,8 +191,7 @@ impl MlDb {&self,face_id_to_cluster_id: &HashMap<String, String>,) -> Result<()> {- self.db- .write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())+ self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter()).map_err(Into::into)}@@ -208,12 +199,11 @@ impl MlDb {&self,face_id_to_cluster_id: &HashMap<String, String>,) -> Result<()> {- self.db- .write_batch_atomic(- "DELETE FROM face_clusters WHERE face_id = ? AND cluster_id = ?",- face_id_to_cluster_id.iter(),- )- .map_err(Into::into)+ self.write_batch_atomic(+ "DELETE FROM face_clusters WHERE face_id = ? AND cluster_id = ?",+ face_id_to_cluster_id.iter(),+ )+ .map_err(Into::into)}pub fn get_file_id_to_cluster_id_set_for_cluster(@@ -224,25 +214,23 @@ impl MlDb {"SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",bind_placeholders(cluster_ids.len()));- self.db- .read_all(&sql, params_from_iter(cluster_ids), pair)+ self.read_all(&sql, params_from_iter(cluster_ids), pair).map_err(Into::into).and_then(file_id_to_cluster_ids)}pub fn get_file_ids_of_cluster_id(&self, cluster_id: &str) -> Result<Vec<i64>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT DISTINCT faces.file_idFROM face_clustersJOIN facesON face_clusters.face_id = faces.face_idWHERE face_clusters.cluster_id = ?"#,- [cluster_id],- )- .map_err(Into::into)+ [cluster_id],+ )+ .map_err(Into::into)}pub fn get_cluster_centroid_vector_id_map(@@ -255,27 +243,26 @@ impl MlDb {return Ok(HashMap::new());}if create_if_missing {- self.db.write_batch_atomic(+ self.write_batch_atomic("INSERT OR IGNORE INTO cluster_centroid_vector_id_map (cluster_id) VALUES (?)",unique_cluster_ids.iter().map(|cluster_id| [cluster_id]),)?;}- self.db- .read_chunked_in(- r#"+ self.read_chunked_in(+ r#"SELECT cluster_id, cluster_vector_idFROM cluster_centroid_vector_id_mapWHERE cluster_id IN ({})"#,- &unique_cluster_ids,- const { NonZeroUsize::new(800).unwrap() },- pair,- )- .map_err(Into::into)+ &unique_cluster_ids,+ const { NonZeroUsize::new(800).unwrap() },+ pair,+ )+ .map_err(Into::into)}pub fn delete_cluster_centroid_vector_id_mapping(&self, cluster_id: &str) -> Result<()> {- self.db.execute(+ self.execute("DELETE FROM cluster_centroid_vector_id_map WHERE cluster_id = ?",[cluster_id],)?;@@ -283,8 +270,7 @@ impl MlDb {}pub fn clear_cluster_centroid_vector_id_mappings(&self) -> Result<()> {- self.db- .execute_statements([DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING])+ self.execute_statements([DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING]).map_err(Into::into)}@@ -292,19 +278,18 @@ impl MlDb {&self,summary: &HashMap<String, ClusterSummary>,) -> Result<()> {- self.db- .write_batches_committing_each(- UPSERT_CLUSTER_SUMMARY,- const { NonZeroUsize::new(400).unwrap() },- summary- .iter()- .map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ UPSERT_CLUSTER_SUMMARY,+ const { NonZeroUsize::new(400).unwrap() },+ summary+ .iter()+ .map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),+ )+ .map_err(Into::into)}pub fn delete_cluster_summary_row(&self, cluster_id: &str) -> Result<()> {- self.db.execute(+ self.execute("DELETE FROM cluster_summary WHERE cluster_id = ?",[cluster_id],)?;@@ -323,13 +308,12 @@ impl MlDb {""});- self.db- .read_all(- &sql,- optional_parameter(&min_cluster_size).as_slice(),- read_cluster_summary,- )- .map_err(Into::into)+ self.read_all(+ &sql,+ optional_parameter(&min_cluster_size).as_slice(),+ read_cluster_summary,+ )+ .map_err(Into::into)}pub fn get_cluster_to_cluster_summary(@@ -340,14 +324,12 @@ impl MlDb {"SELECT * FROM cluster_summary WHERE cluster_id IN ({})",bind_placeholders(cluster_ids.len()));- self.db- .read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary)+ self.read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary).map_err(Into::into)}pub fn count_cluster_summaries(&self) -> Result<i64> {- self.db- .read_value("SELECT COUNT(cluster_id) as total FROM cluster_summary", ())+ self.read_value("SELECT COUNT(cluster_id) as total FROM cluster_summary", ()).map_err(Into::into)}@@ -357,12 +339,12 @@ impl MlDb {limit: i64,) -> Result<Vec<ClusterCentroidRow>> {match before_cluster_id {- None => self.db.read_all(+ None => self.read_all("SELECT cluster_id, avg FROM cluster_summary ORDER BY cluster_id DESC LIMIT ?",[limit],read_cluster_centroid,),- Some(before_cluster_id) => self.db.read_all(+ Some(before_cluster_id) => self.read_all(r#"SELECT cluster_id, avgFROM cluster_summary@@ -389,21 +371,20 @@ impl MlDb {} else {&[]};- self.db- .execute_statements(face_statements.iter().copied().chain([- DELETE_CLUSTER_PERSON,- DELETE_NOT_PERSON_FEEDBACK,- DELETE_CLUSTER_SUMMARY,- DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,- DELETE_FACE_CLUSTERS,- CREATE_CLUSTER_PERSON_TABLE,- CREATE_NOT_PERSON_FEEDBACK_TABLE,- CREATE_CLUSTER_SUMMARY_TABLE,- CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE,- CREATE_FACE_CLUSTERS_TABLE,- FC_CLUSTER_ID_INDEX,- ]))- .map_err(Into::into)+ self.execute_statements(face_statements.iter().copied().chain([+ DELETE_CLUSTER_PERSON,+ DELETE_NOT_PERSON_FEEDBACK,+ DELETE_CLUSTER_SUMMARY,+ DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,+ DELETE_FACE_CLUSTERS,+ CREATE_CLUSTER_PERSON_TABLE,+ CREATE_NOT_PERSON_FEEDBACK_TABLE,+ CREATE_CLUSTER_SUMMARY_TABLE,+ CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE,+ CREATE_FACE_CLUSTERS_TABLE,+ FC_CLUSTER_ID_INDEX,+ ]))+ .map_err(Into::into)}pub fn get_clusters_for_memory_lane(@@ -414,7 +395,7 @@ impl MlDb {let mut clusters = HashSet::new();let mut offset: i64 = 0;while clusters.len() < 20 {- let batch: Vec<String> = self.db.read_column(+ let batch: Vec<String> = self.read_column(r#"SELECT cluster_id, COUNT(*) AS countFROM face_clustersdiff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rsindex aeea11fc45..2d983a2ce1 100644--- a/rust/crates/photos/src/ml_db/queries/faces.rs+++ b/rust/crates/photos/src/ml_db/queries/faces.rs@@ -1,10 +1,10 @@use std::collections::{HashMap, HashSet};use std::num::NonZeroUsize;-use crate::db::{- MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, ToSql, bind_placeholders, group_into,- optional_parameter, pair, params_from_iter,+use super::helpers::{+ MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,};+use rusqlite::{Result as SqliteResult, Row, ToSql, params_from_iter};use super::clip::CLIP_ML_VERSION;use super::pets::PET_ML_VERSION;@@ -75,45 +75,42 @@ const UPSERT_FACE: &str = r#"impl MlDb {pub fn bulk_insert_faces(&self, faces: &[FaceRow]) -> Result<()> {- self.db- .write_batches_committing_each(- UPSERT_FACE,- const { NonZeroUsize::new(500).unwrap() },- faces.iter().map(|face| {- (- face.file_id,- &face.face_id,- &face.detection_json,- encode_evector(&face.embedding),- face.score,- face.blur,- face.is_sideways,- face.image_height,- face.image_width,- face.ml_version,- )- }),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ UPSERT_FACE,+ const { NonZeroUsize::new(500).unwrap() },+ faces.iter().map(|face| {+ (+ face.file_id,+ &face.face_id,+ &face.detection_json,+ encode_evector(&face.embedding),+ face.score,+ face.blur,+ face.is_sideways,+ face.image_height,+ face.image_width,+ face.ml_version,+ )+ }),+ )+ .map_err(Into::into)}pub fn face_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {- self.db- .read_all(- "SELECT file_id, ml_version FROM faces WHERE ml_version >= ?",- [minimum_ml_version],- pair,- )- .map_err(Into::into)+ self.read_all(+ "SELECT file_id, ml_version FROM faces WHERE ml_version >= ?",+ [minimum_ml_version],+ pair,+ )+ .map_err(Into::into)}pub fn get_face_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {- self.db- .read_value(- "SELECT COUNT(DISTINCT file_id) as count FROM faces WHERE ml_version >= ?",- [minimum_ml_version],- )- .map_err(Into::into)+ self.read_value(+ "SELECT COUNT(DISTINCT file_id) as count FROM faces WHERE ml_version >= ?",+ [minimum_ml_version],+ )+ .map_err(Into::into)}pub fn get_face_embeddings_for_cluster(@@ -135,8 +132,7 @@ impl MlDb {);let mut parameters: Vec<&dyn ToSql> = vec![&cluster_id];parameters.extend(optional_parameter(&limit));- self.db- .read_column(&sql, parameters.as_slice())+ self.read_column(&sql, parameters.as_slice()).map_err(Into::into)}@@ -174,8 +170,7 @@ impl MlDb {let mut parameters: Vec<&dyn ToSql> =cluster_chunk.iter().map(|id| id as &dyn ToSql).collect();parameters.extend(optional_parameter(&remaining_limit));- let rows: Vec<(String, Vec<u8>)> =- self.db.read_all(&sql, parameters.as_slice(), pair)?;+ let rows: Vec<(String, Vec<u8>)> = self.read_all(&sql, parameters.as_slice(), pair)?;if let Some(remaining) = remaining_limit.as_mut() {*remaining -= rows.len() as i64;}@@ -199,7 +194,7 @@ impl MlDb {if let Some(avatar_file_id) = avatar_file_id {file_ids.push(avatar_file_id);}- let cluster_ids: Vec<String> = self.db.read_column(+ let cluster_ids: Vec<String> = self.read_column("SELECT cluster_id FROM cluster_person WHERE person_id = ?",[person_id],)?;@@ -222,8 +217,7 @@ impl MlDb {cluster_ids.iter().map(|id| id as &dyn ToSql).collect();parameters.extend(file_ids.iter().map(|id| id as &dyn ToSql));let faces: Vec<StoredFace> =- self.db- .read_all(&sql, parameters.as_slice(), read_stored_face)?;+ self.read_all(&sql, parameters.as_slice(), read_stored_face)?;if !faces.is_empty() {if let Some(avatar_file_id) = avatar_file_id&& let Some(face) = faces.iter().find(|face| face.row.file_id == avatar_file_id)@@ -234,7 +228,7 @@ impl MlDb {}}if let Some(cluster_id) = cluster_id {- let face_ids: Vec<String> = self.db.read_column(+ let face_ids: Vec<String> = self.read_column("SELECT face_id FROM face_clusters WHERE cluster_id = ?",[cluster_id],)?;@@ -253,7 +247,7 @@ impl MlDb {}pub fn get_faces_for_given_file_id(&self, file_upload_id: i64) -> Result<Vec<FaceRow>> {- let faces: Vec<StoredFace> = self.db.read_all(+ let faces: Vec<StoredFace> = self.read_all("SELECT * FROM faces WHERE file_id = ?",[file_upload_id],read_stored_face,@@ -264,7 +258,7 @@ impl MlDb {pub fn get_file_ids_to_faces_without_embedding(&self,) -> Result<HashMap<i64, Vec<FaceWithoutEmbedding>>> {- let faces: Vec<FaceWithoutEmbedding> = self.db.read_all(+ let faces: Vec<FaceWithoutEmbedding> = self.read_all("SELECT face_id, file_id, score, detection, blur FROM faces",(),|row| {@@ -291,7 +285,7 @@ impl MlDb {let mut offset = offset;let mut result = Vec::new();loop {- let rows: Vec<(String, Vec<u8>, f64, f64, i64)> = self.db.read_all(+ let rows: Vec<(String, Vec<u8>, f64, f64, i64)> = self.read_all(r#"SELECT face_id, embedding, score, blur, is_sidewaysFROM faces@@ -367,31 +361,27 @@ impl MlDb {);let mut parameters: Vec<&dyn ToSql> = chunk.iter().map(|id| id as &dyn ToSql).collect();parameters.push(&remaining_limit);- let rows: Vec<(String, Vec<u8>)> =- self.db.read_all(&sql, parameters.as_slice(), pair)?;+ let rows: Vec<(String, Vec<u8>)> = self.read_all(&sql, parameters.as_slice(), pair)?;result.extend(rows);}Ok(result)}pub fn get_total_face_count(&self) -> Result<i64> {- self.db- .read_value(- "SELECT COUNT(*) as count FROM faces WHERE score > ? AND blur > ?",- (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),- )- .map_err(Into::into)+ self.read_value(+ "SELECT COUNT(*) as count FROM faces WHERE score > ? AND blur > ?",+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),+ )+ .map_err(Into::into)}pub fn get_errored_face_count(&self) -> Result<i64> {- self.db- .read_value("SELECT COUNT(*) as count FROM faces WHERE score < 0", ())+ self.read_value("SELECT COUNT(*) as count FROM faces WHERE score < 0", ()).map_err(Into::into)}pub fn get_errored_file_ids(&self) -> Result<HashSet<i64>> {- self.db- .read_column("SELECT DISTINCT file_id FROM faces WHERE score < 0", ())+ self.read_column("SELECT DISTINCT file_id FROM faces WHERE score < 0", ()).map_err(Into::into)}@@ -399,9 +389,8 @@ impl MlDb {if file_ids.is_empty() {return Ok(());}- self.db- .execute_chunked_in(- r#"+ self.execute_chunked_in(+ r#"DELETE FROM facesWHERE file_id IN ({})AND score < 0@@ -413,9 +402,9 @@ impl MlDb {AND successful.ml_version >= faces.ml_version)"#,- file_ids,- )- .map_err(Into::into)+ file_ids,+ )+ .map_err(Into::into)}pub fn get_file_ids_with_error_results(&self, file_ids: &[i64]) -> Result<HashSet<i64>> {@@ -450,7 +439,7 @@ impl MlDb {AND score < 0"#);- let matches: Vec<i64> = self.db.read_column(+ let matches: Vec<i64> = self.read_column(&sql,params_from_iter(chunk.iter().chain(chunk).chain(chunk)),)?;@@ -460,8 +449,7 @@ impl MlDb {}pub fn delete_face_index_for_files(&self, file_ids: &[i64]) -> Result<()> {- self.db- .execute_chunked_in("DELETE FROM faces WHERE file_id IN ({})", file_ids)+ self.execute_chunked_in("DELETE FROM faces WHERE file_id IN ({})", file_ids).map_err(Into::into)}@@ -469,9 +457,8 @@ impl MlDb {if file_ids.is_empty() {return Ok(());}- self.db- .execute_chunked_in(- r#"+ self.execute_chunked_in(+ r#"DELETE FROM facesWHERE file_id IN ({})AND NOT EXISTS (@@ -480,25 +467,24 @@ impl MlDb {WHERE face_clusters.face_id = faces.face_id)"#,- file_ids,- )- .map_err(Into::into)+ file_ids,+ )+ .map_err(Into::into)}pub fn get_clustered_or_faceless_file_count(&self) -> Result<i64> {- let clustered_face_ids: Vec<String> = self- .db- .read_column("SELECT face_id FROM face_clusters", ())?;+ let clustered_face_ids: Vec<String> =+ self.read_column("SELECT face_id FROM face_clusters", ())?;let clustered_file_ids = clustered_face_ids.iter().map(|face_id| file_id_from_face_id(face_id)).collect::<Result<HashSet<i64>>>()?;- let bad_file_ids: HashSet<i64> = self.db.read_column(+ let bad_file_ids: HashSet<i64> = self.read_column("SELECT DISTINCT file_id FROM faces WHERE score <= ? OR blur <= ?",(MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),)?;- let good_file_ids: HashSet<i64> = self.db.read_column(+ let good_file_ids: HashSet<i64> = self.read_column("SELECT DISTINCT file_id FROM faces WHERE score > ? AND blur > ?",(MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),)?;@@ -507,9 +493,8 @@ impl MlDb {}pub fn get_unclustered_face_count(&self) -> Result<i64> {- self.db- .read_value(- r#"+ self.read_value(+ r#"SELECT COUNT(*) as countFROM faces fLEFT JOIN face_clusters fc@@ -518,24 +503,23 @@ impl MlDb {AND f.blur > ?AND fc.face_id IS NULL"#,- (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),- )- .map_err(Into::into)+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),+ )+ .map_err(Into::into)}pub fn get_all_file_ids_of_face_ids_not_in_any_cluster(&self) -> Result<HashSet<i64>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT DISTINCT file_idFROM facesLEFT JOIN face_clustersON faces.face_id = face_clusters.face_idWHERE face_clusters.face_id IS NULL"#,- (),- )- .map_err(Into::into)+ (),+ )+ .map_err(Into::into)}pub fn get_all_files_associated_with_all_clusters(@@ -553,8 +537,7 @@ impl MlDb {"#,bind_placeholders(except_clusters.len()));- self.db- .read_column(&sql, params_from_iter(except_clusters))+ self.read_column(&sql, params_from_iter(except_clusters)).map_err(Into::into)}@@ -575,8 +558,7 @@ impl MlDb {sql.push_str(" INTERSECT SELECT file_id FROM pet_faces WHERE ml_version >= ?");parameters.push(PET_ML_VERSION);}- self.db- .read_column(&sql, params_from_iter(parameters))+ self.read_column(&sql, params_from_iter(parameters)).map_err(Into::into)}}diff --git a/rust/crates/photos/src/ml_db/queries/filedata.rs b/rust/crates/photos/src/ml_db/queries/filedata.rsindex 856a226314..31b1ad4ab8 100644--- a/rust/crates/photos/src/ml_db/queries/filedata.rs+++ b/rust/crates/photos/src/ml_db/queries/filedata.rs@@ -1,6 +1,6 @@use std::collections::{HashMap, HashSet};-use crate::db::optional_parameter;+use super::helpers::optional_parameter;use crate::ml_db::{MlDb, Result};#[derive(Clone, Debug, PartialEq, Eq)]@@ -25,45 +25,43 @@ impl MlDb {if fd_status_list.is_empty() {return Ok(());}- self.db- .write_batch_atomic(- r#"+ self.write_batch_atomic(+ r#"INSERT OR REPLACE INTO filedata (file_id, user_id, type, size, obj_id, obj_nonce, updated_at)values (?, ?, ?, ?, ?, ?, ?)"#,- fd_status_list.iter().map(|status| {- (- status.file_id,- status.user_id,- &status.data_type,- status.size,- &status.object_id,- &status.object_nonce,- status.updated_at,- )- }),- )- .map_err(Into::into)+ fd_status_list.iter().map(|status| {+ (+ status.file_id,+ status.user_id,+ &status.data_type,+ status.size,+ &status.object_id,+ &status.object_nonce,+ status.updated_at,+ )+ }),+ )+ .map_err(Into::into)}pub fn get_file_ids_vid_preview(&self) -> Result<HashMap<i64, PreviewInfo>> {- self.db- .read_all(- "SELECT file_id, obj_id, size FROM filedata WHERE type='vid_preview'",- (),- |row| {- Ok((- row.get(0)?,- PreviewInfo {- object_id: row.get(1)?,- object_size: row.get(2)?,- },- ))- },- )- .map_err(Into::into)+ self.read_all(+ "SELECT file_id, obj_id, size FROM filedata WHERE type='vid_preview'",+ (),+ |row| {+ Ok((+ row.get(0)?,+ PreviewInfo {+ object_id: row.get(1)?,+ object_size: row.get(2)?,+ },+ ))+ },+ )+ .map_err(Into::into)}pub fn get_file_ids_with_fd_data(&self, data_type: Option<&str>) -> Result<HashSet<i64>> {@@ -71,8 +69,7 @@ impl MlDb {None => "SELECT file_id FROM filedata",Some(_) => "SELECT file_id FROM filedata WHERE type = ?",};- self.db- .read_column(sql, optional_parameter(&data_type).as_slice())+ self.read_column(sql, optional_parameter(&data_type).as_slice()).map_err(Into::into)}}diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rsnew file mode 100644index 0000000000..486bd8d577--- /dev/null+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs@@ -0,0 +1,518 @@+use std::collections::HashMap;+use std::hash::Hash;+use std::num::NonZeroUsize;++use rusqlite::{+ OptionalExtension, Params, Result, Row, ToSql, TransactionBehavior, params_from_iter,+ types::FromSql,+};++use crate::ml_db::MlDb;++pub(super) const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;++impl MlDb {+ pub(in crate::ml_db) fn read_all<C: FromIterator<T>, T, P: Params>(+ &self,+ sql: &str,+ parameters: P,+ map: impl FnMut(&Row<'_>) -> Result<T>,+ ) -> Result<C> {+ self.db.read(|connection| {+ let mut statement = connection.prepare_cached(sql)?;+ let rows = statement.query_map(parameters, map)?;+ rows.collect::<Result<C>>()+ })+ }++ pub(in crate::ml_db) fn read_column<C: FromIterator<T>, T: FromSql, P: Params>(+ &self,+ sql: &str,+ parameters: P,+ ) -> Result<C> {+ self.read_all(sql, parameters, |row| row.get(0))+ }++ pub(in crate::ml_db) fn read_grouped<+ K: FromSql + Eq + Hash,+ V: FromSql,+ C: FromIterator<V>,+ P: Params,+ >(+ &self,+ sql: &str,+ parameters: P,+ ) -> Result<HashMap<K, C>> {+ let rows: Vec<(K, V)> = self.read_all(sql, parameters, pair)?;+ Ok(group_into(rows))+ }++ pub(in crate::ml_db) fn read_value<T: FromSql>(+ &self,+ sql: &str,+ parameters: impl Params,+ ) -> Result<T> {+ self.db.read(|connection| {+ connection+ .prepare_cached(sql)?+ .query_row(parameters, |row| row.get(0))+ })+ }++ pub(in crate::ml_db) fn read_optional<T: FromSql>(+ &self,+ sql: &str,+ parameters: impl Params,+ ) -> Result<Option<T>> {+ self.db.read(|connection| {+ connection+ .prepare_cached(sql)?+ .query_row(parameters, |row| row.get(0))+ .optional()+ })+ }++ pub(in crate::ml_db) fn read_chunked_in<C: FromIterator<T>, T, I: ToSql>(+ &self,+ sql: &str,+ ids: &[I],+ chunk_size: NonZeroUsize,+ mut map: impl FnMut(&Row<'_>) -> Result<T>,+ ) -> Result<C> {+ let mut rows = Vec::new();+ for chunk in ids.chunks(chunk_size.get()) {+ let sql = expand_in_clause(sql, chunk.len());+ let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;+ rows.extend(chunk_rows);+ }+ Ok(rows.into_iter().collect())+ }++ pub(in crate::ml_db) fn execute(&self, sql: &str, parameters: impl Params) -> Result<usize> {+ self.db+ .write(|connection| connection.prepare_cached(sql)?.execute(parameters))+ }++ pub(in crate::ml_db) fn execute_chunked_in<I: ToSql>(+ &self,+ sql: &str,+ ids: &[I],+ ) -> Result<()> {+ if ids.is_empty() {+ return Ok(());+ }+ self.db.write_transaction(|transaction| {+ for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {+ let sql = expand_in_clause(sql, chunk.len());+ transaction.execute(&sql, params_from_iter(chunk))?;+ }+ Ok(())+ })+ }++ pub(in crate::ml_db) fn execute_statements<'a>(+ &self,+ statements: impl IntoIterator<Item = &'a str>,+ ) -> Result<()> {+ self.db.write_transaction(|transaction| {+ for statement in statements {+ transaction.execute_batch(statement)?;+ }+ Ok(())+ })+ }++ pub(in crate::ml_db) fn write_batch_atomic<P: Params>(+ &self,+ sql: &str,+ parameter_sets: impl IntoIterator<Item = P>,+ ) -> Result<()> {+ self.db.write_transaction(|transaction| {+ let mut statement = transaction.prepare_cached(sql)?;+ for parameters in parameter_sets {+ statement.execute(parameters)?;+ }+ Ok(())+ })+ }++ pub(in crate::ml_db) fn write_batches_committing_each<P: Params>(+ &self,+ sql: &str,+ batch_size: NonZeroUsize,+ parameter_sets: impl IntoIterator<Item = P>,+ ) -> Result<()> {+ self.db.write(|connection| {+ let mut parameter_sets = parameter_sets.into_iter().peekable();+ while parameter_sets.peek().is_some() {+ let transaction =+ connection.transaction_with_behavior(TransactionBehavior::Immediate)?;+ {+ let mut statement = transaction.prepare_cached(sql)?;+ for parameters in parameter_sets.by_ref().take(batch_size.get()) {+ statement.execute(parameters)?;+ }+ }+ transaction.commit()?;+ }+ Ok(())+ })+ }+}++pub(super) fn bind_placeholders(count: usize) -> String {+ vec!["?"; count].join(", ")+}++fn expand_in_clause(sql: &str, count: usize) -> String {+ sql.replacen("{}", &bind_placeholders(count), 1)+}++pub(super) fn pair<A: FromSql, B: FromSql>(row: &Row<'_>) -> Result<(A, B)> {+ Ok((row.get(0)?, row.get(1)?))+}++pub(super) fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql> {+ value.iter().map(|value| value as &dyn ToSql).collect()+}++pub(super) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(+ rows: impl IntoIterator<Item = (K, V)>,+) -> HashMap<K, C> {+ let mut groups: HashMap<K, Vec<V>> = HashMap::new();+ for (key, value) in rows {+ groups.entry(key).or_default().push(value);+ }+ groups+ .into_iter()+ .map(|(key, values)| (key, values.into_iter().collect()))+ .collect()+}++#[cfg(test)]+mod tests {+ use std::collections::{HashMap, HashSet};+ use std::num::NonZeroUsize;++ use crate::db::{Database, OpenOptions};++ use super::{MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders};+ use crate::ml_db::MlDb;++ fn open() -> (tempfile::TempDir, MlDb) {+ let directory = tempfile::tempdir().unwrap();+ let db = Database::open(+ directory.path().join("items.db"),+ OpenOptions::default(),+ |connection| connection.execute_batch("CREATE TABLE items (id INTEGER PRIMARY KEY)"),+ )+ .unwrap();+ (directory, MlDb { db })+ }++ #[test]+ fn reader_queries_reject_writes() {+ let (_directory, db) = open();+ db.execute("INSERT INTO items (id) VALUES (?)", [7])+ .unwrap();+ let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());+ assert!(deleted.is_err());+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())+ .unwrap(),+ [7]+ );+ }++ #[test]+ fn failed_batch_rolls_back_and_writer_recovers() {+ let (_directory, db) = open();+ assert!(+ db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]])+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ assert_eq!(+ db.execute("INSERT INTO items (id) VALUES (?)", [3])+ .unwrap(),+ 1+ );+ assert_eq!(+ db.execute("DELETE FROM items WHERE id = ?", [4]).unwrap(),+ 0+ );+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())+ .unwrap(),+ [3]+ );+ }++ #[test]+ fn statement_sequence_rolls_back_if_a_later_statement_fails() {+ let (_directory, db) = open();+ assert!(+ db.execute_statements([+ "INSERT INTO items VALUES (1)",+ "INSERT INTO missing_table VALUES (2)",+ ])+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ }++ #[test]+ fn chunked_delete_handles_empty_lists_and_rolls_back_on_failure() {+ let (_directory, db) = open();+ let ids: Vec<i64> = (0..10_001).collect();+ db.write_batch_atomic(+ "INSERT INTO items (id) VALUES (?)",+ ids.iter().map(|id| [id]),+ )+ .unwrap();+ db.execute_chunked_in::<i64>("DELETE FROM items WHERE id IN ({})", &[])+ .unwrap();+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 10_001+ );+ db.execute_statements(["CREATE TRIGGER reject_last_delete BEFORE DELETE ON items+ WHEN OLD.id = 10000 BEGIN SELECT RAISE(ABORT, 'cannot delete'); END"])+ .unwrap();+ assert!(+ db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 10_001+ );+ db.execute_statements(["DROP TRIGGER reject_last_delete"])+ .unwrap();+ db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)+ .unwrap();+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ }++ #[test]+ fn batched_writes_commit_each_completed_batch() {+ let (_directory, db) = open();+ db.write_batches_committing_each(+ "INSERT INTO missing_table VALUES (?)",+ const { NonZeroUsize::new(2).unwrap() },+ Vec::<[i64; 1]>::new(),+ )+ .unwrap();+ assert!(+ db.write_batches_committing_each(+ "INSERT INTO items (id) VALUES (?)",+ const { NonZeroUsize::new(2).unwrap() },+ [[1], [2], [3], [4], [1]]+ )+ .is_err()+ );+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())+ .unwrap(),+ [1, 2, 3, 4]+ );+ assert!(+ db.write_batches_committing_each(+ "INSERT INTO items (id) VALUES (?)",+ const { NonZeroUsize::new(2).unwrap() },+ [[5], [6], [7], [5], [8]]+ )+ .is_err()+ );+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())+ .unwrap(),+ [1, 2, 3, 4, 5, 6]+ );+ db.write_batches_committing_each(+ "INSERT INTO items (id) VALUES (?)",+ const { NonZeroUsize::new(2).unwrap() },+ [[7], [8], [9]],+ )+ .unwrap();+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())+ .unwrap(),+ [1, 2, 3, 4, 5, 6, 7, 8, 9]+ );+ assert!(+ db.write_batches_committing_each(+ "INSERT INTO items (id) VALUES (?)",+ NonZeroUsize::MIN,+ [[10], [11], [10]],+ )+ .is_err()+ );+ assert_eq!(+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())+ .unwrap(),+ (1..=11).collect::<Vec<_>>()+ );+ }++ #[test]+ fn batched_writes_hold_the_writer_between_commits() {+ use std::sync::mpsc::{self, RecvTimeoutError};+ use std::thread;+ use std::time::Duration;++ let (_directory, db) = open();+ thread::scope(|scope| {+ let (between_tx, between_rx) = mpsc::channel();+ let (release_tx, release_rx) = mpsc::channel();+ let db = &db;+ scope.spawn(move || {+ db.write_batches_committing_each(+ "INSERT INTO items VALUES (?)",+ NonZeroUsize::MIN,+ (1..=2).map(|id| {+ if id == 2 {+ between_tx.send(()).unwrap();+ release_rx.recv().unwrap();+ }+ [id]+ }),+ )+ .unwrap();+ });+ between_rx.recv_timeout(Duration::from_secs(5)).unwrap();+ let committed_count = db+ .read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap();+ let (started_tx, started_rx) = mpsc::channel();+ let (finished_tx, finished_rx) = mpsc::channel();+ scope.spawn(move || {+ started_tx.send(()).unwrap();+ finished_tx+ .send(db.execute("INSERT INTO items VALUES (3)", ()))+ .unwrap();+ });+ started_rx.recv_timeout(Duration::from_secs(5)).unwrap();+ let was_blocked = matches!(+ finished_rx.recv_timeout(Duration::from_millis(100)),+ Err(RecvTimeoutError::Timeout)+ );+ release_tx.send(()).unwrap();+ let result = finished_rx.recv_timeout(Duration::from_secs(5)).unwrap();+ assert_eq!(committed_count, 1);+ assert!(was_blocked);+ assert_eq!(result.unwrap(), 1);+ });+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 3+ );+ }++ #[test]+ fn chunked_reads_query_each_chunk_in_order() {+ let (_directory, db) = open();+ db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))+ .unwrap();+ let ids: Vec<i64> = (1..=7).collect();+ let one_at_a_time: Vec<i64> = db+ .read_chunked_in(+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",+ &ids,+ NonZeroUsize::MIN,+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(one_at_a_time, ids);+ let by_chunk: Vec<i64> = db+ .read_chunked_in(+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",+ &ids,+ const { NonZeroUsize::new(3).unwrap() },+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);+ let single_chunk: Vec<i64> = db+ .read_chunked_in(+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",+ &ids,+ const { NonZeroUsize::new(10).unwrap() },+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);+ let none: Vec<i64> = db+ .read_chunked_in(+ "SELECT id FROM missing_table WHERE id IN ({})",+ &Vec::<i64>::new(),+ const { NonZeroUsize::new(3).unwrap() },+ |row| row.get(0),+ )+ .unwrap();+ assert!(none.is_empty());+ let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();+ let found: HashSet<i64> = db+ .read_chunked_in(+ "SELECT id FROM items WHERE id IN ({})",+ &many,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));+ }++ #[test]+ fn grouped_reads_collect_values_per_key() {+ let (_directory, db) = open();+ db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=5).map(|id| [id]))+ .unwrap();+ let by_parity: HashMap<i64, Vec<i64>> = db+ .read_grouped("SELECT id % 2, id FROM items ORDER BY id", ())+ .unwrap();+ assert_eq!(+ by_parity,+ HashMap::from([(1, vec![1, 3, 5]), (0, vec![2, 4])])+ );+ let by_label: HashMap<String, HashSet<i64>> = db+ .read_grouped(+ "SELECT CASE WHEN id < 3 THEN 'low' ELSE 'high' END, id FROM items",+ (),+ )+ .unwrap();+ assert_eq!(+ by_label,+ HashMap::from([+ ("low".to_string(), HashSet::from([1, 2])),+ ("high".to_string(), HashSet::from([3, 4, 5]))+ ])+ );+ let empty: HashMap<i64, Vec<i64>> = db+ .read_grouped("SELECT id, id FROM items WHERE id > 5", ())+ .unwrap();+ assert!(empty.is_empty());+ }++ #[test]+ fn bind_placeholders_joins_question_marks() {+ assert_eq!(bind_placeholders(0), "");+ assert_eq!(bind_placeholders(1), "?");+ assert_eq!(bind_placeholders(3), "?, ?, ?");+ }+}diff --git a/rust/crates/photos/src/ml_db/queries/mod.rs b/rust/crates/photos/src/ml_db/queries/mod.rsindex a22b335eee..f49f7410a5 100644--- a/rust/crates/photos/src/ml_db/queries/mod.rs+++ b/rust/crates/photos/src/ml_db/queries/mod.rs@@ -3,6 +3,7 @@ pub(super) mod clip;pub(super) mod clusters;pub(super) mod faces;pub(super) mod filedata;+mod helpers;pub(super) mod persons;pub(super) mod pets;diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rsindex 6c43340720..b8768a95e1 100644--- a/rust/crates/photos/src/ml_db/queries/persons.rs+++ b/rust/crates/photos/src/ml_db/queries/persons.rs@@ -1,7 +1,7 @@use std::collections::{HashMap, HashSet};use std::num::NonZeroUsize;-use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};+use super::helpers::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};use super::clusters::file_id_to_cluster_ids;use crate::ml_db::schema::{@@ -25,21 +25,20 @@ const INSERT_NOT_PERSON_FEEDBACK: &str = r#"impl MlDb {pub fn get_clusters_with_three_or_more_not_person_feedback(&self) -> Result<HashSet<String>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT cluster_idFROM not_person_feedbackGROUP BY cluster_idHAVING COUNT(*) >= 3"#,- (),- )- .map_err(Into::into)+ (),+ )+ .map_err(Into::into)}pub fn get_person_ignored_clusters(&self, person_id: &str) -> Result<HashSet<String>> {- let mut ignored_cluster_ids: HashSet<String> = self.db.read_column(+ let mut ignored_cluster_ids: HashSet<String> = self.read_column(r#"SELECT cluster_idFROM cluster_person@@ -48,7 +47,7 @@ impl MlDb {"#,[person_id],)?;- let reject_cluster_ids: Vec<String> = self.db.read_column(+ let reject_cluster_ids: Vec<String> = self.read_column("SELECT cluster_id FROM not_person_feedback WHERE person_id = ?",[person_id],)?;@@ -57,33 +56,30 @@ impl MlDb {}pub fn get_person_to_rejected_suggestions(&self) -> Result<HashMap<String, HashSet<String>>> {- self.db- .read_grouped("SELECT person_id, cluster_id FROM not_person_feedback", ())+ self.read_grouped("SELECT person_id, cluster_id FROM not_person_feedback", ()).map_err(Into::into)}pub fn get_person_cluster_ids(&self, person_id: &str) -> Result<HashSet<String>> {- self.db- .read_column(- "SELECT cluster_id FROM cluster_person WHERE person_id = ?",- [person_id],- )- .map_err(Into::into)+ self.read_column(+ "SELECT cluster_id FROM cluster_person WHERE person_id = ?",+ [person_id],+ )+ .map_err(Into::into)}pub fn get_persons_cluster_ids(&self, person_ids: &[String]) -> Result<HashSet<String>> {- self.db- .read_chunked_in(- "SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",- person_ids,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },- |row| row.get(0),- )- .map_err(Into::into)+ self.read_chunked_in(+ "SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",+ person_ids,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },+ |row| row.get(0),+ )+ .map_err(Into::into)}pub fn get_person_to_cluster_id_to_face_ids(&self) -> Result<PersonToClusterIdToFaceIds> {- let rows: Vec<(String, String, String)> = self.db.read_all(+ let rows: Vec<(String, String, String)> = self.read_all(r#"SELECT person_id, face_clusters.cluster_id, face_idFROM cluster_person@@ -106,8 +102,7 @@ impl MlDb {}pub fn get_person_to_cluster_ids(&self) -> Result<HashMap<String, HashSet<String>>> {- self.db- .read_grouped("SELECT person_id, cluster_id FROM cluster_person", ())+ self.read_grouped("SELECT person_id, cluster_id FROM cluster_person", ()).map_err(Into::into)}@@ -115,53 +110,50 @@ impl MlDb {&self,face_ids: &[String],) -> Result<HashMap<String, String>> {- self.db- .read_chunked_in(- r#"+ self.read_chunked_in(+ r#"SELECT face_id, person_idFROM cluster_personINNER JOIN face_clustersON cluster_person.cluster_id = face_clusters.cluster_idWHERE face_id IN ({})"#,- face_ids,- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },- pair,- )- .map_err(Into::into)+ face_ids,+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },+ pair,+ )+ .map_err(Into::into)}pub fn get_cluster_id_to_face_ids_for_person(&self,person_id: &str,) -> Result<HashMap<String, HashSet<String>>> {- self.db- .read_grouped(- r#"+ self.read_grouped(+ r#"SELECT face_clusters.cluster_id, face_idFROM cluster_personINNER JOIN face_clustersON cluster_person.cluster_id = face_clusters.cluster_idWHERE person_id = ?"#,- [person_id],- )- .map_err(Into::into)+ [person_id],+ )+ .map_err(Into::into)}pub fn get_face_ids_for_person(&self, person_id: &str) -> Result<HashSet<String>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT face_idFROM face_clustersLEFT JOIN cluster_personON face_clusters.cluster_id = cluster_person.cluster_idWHERE cluster_person.person_id = ?"#,- [person_id],- )- .map_err(Into::into)+ [person_id],+ )+ .map_err(Into::into)}pub fn get_face_ids_for_person_ordered_by_score(@@ -169,9 +161,8 @@ impl MlDb {person_id: &str,limit: i64,) -> Result<Vec<String>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT faces.face_idFROM facesJOIN face_clusters@@ -182,9 +173,9 @@ impl MlDb {ORDER BY faces.score DESCLIMIT ?"#,- (person_id, limit),- )- .map_err(Into::into)+ (person_id, limit),+ )+ .map_err(Into::into)}pub fn remove_person(&self, person_id: &str) -> Result<()> {@@ -204,8 +195,7 @@ impl MlDb {}pub fn assign_cluster_to_person(&self, person_id: &str, cluster_id: &str) -> Result<()> {- self.db- .execute(INSERT_CLUSTER_PERSON, [person_id, cluster_id])?;+ self.execute(INSERT_CLUSTER_PERSON, [person_id, cluster_id])?;Ok(())}@@ -213,19 +203,17 @@ impl MlDb {&self,cluster_to_person_id: &HashMap<String, String>,) -> Result<()> {- self.db- .write_batch_atomic(- INSERT_CLUSTER_PERSON,- cluster_to_person_id- .iter()- .map(|(cluster_id, person_id)| (person_id, cluster_id)),- )- .map_err(Into::into)+ self.write_batch_atomic(+ INSERT_CLUSTER_PERSON,+ cluster_to_person_id+ .iter()+ .map(|(cluster_id, person_id)| (person_id, cluster_id)),+ )+ .map_err(Into::into)}pub fn capture_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {- self.db- .execute(INSERT_NOT_PERSON_FEEDBACK, [person_id, cluster_id])?;+ self.execute(INSERT_NOT_PERSON_FEEDBACK, [person_id, cluster_id])?;Ok(())}@@ -233,18 +221,17 @@ impl MlDb {&self,cluster_to_person_id: &HashMap<String, String>,) -> Result<()> {- self.db- .write_batch_atomic(- INSERT_NOT_PERSON_FEEDBACK,- cluster_to_person_id- .iter()- .map(|(cluster_id, person_id)| (person_id, cluster_id)),- )- .map_err(Into::into)+ self.write_batch_atomic(+ INSERT_NOT_PERSON_FEEDBACK,+ cluster_to_person_id+ .iter()+ .map(|(cluster_id, person_id)| (person_id, cluster_id)),+ )+ .map_err(Into::into)}pub fn remove_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {- self.db.execute(+ self.execute("DELETE FROM not_person_feedback WHERE person_id = ? AND cluster_id = ?",[person_id, cluster_id],)?;@@ -252,7 +239,7 @@ impl MlDb {}pub fn remove_cluster_to_person(&self, person_id: &str, cluster_id: &str) -> Result<()> {- self.db.execute(+ self.execute("DELETE FROM cluster_person WHERE person_id = ? AND cluster_id = ?",[person_id, cluster_id],)?;@@ -263,47 +250,43 @@ impl MlDb {&self,person_id: &str,) -> Result<HashMap<i64, HashSet<String>>> {- self.db- .read_all(- r#"+ self.read_all(+ r#"SELECT face_clusters.cluster_id, face_idFROM face_clustersINNER JOIN cluster_personON face_clusters.cluster_id = cluster_person.cluster_idWHERE cluster_person.person_id = ?"#,- [person_id],- pair,- )- .map_err(Into::into)- .and_then(file_id_to_cluster_ids)+ [person_id],+ pair,+ )+ .map_err(Into::into)+ .and_then(file_id_to_cluster_ids)}pub fn get_cluster_id_to_person_id(&self) -> Result<HashMap<String, String>> {- self.db- .read_all(- "SELECT person_id, cluster_id FROM cluster_person",- (),- |row| Ok((row.get(1)?, row.get(0)?)),- )- .map_err(Into::into)+ self.read_all(+ "SELECT person_id, cluster_id FROM cluster_person",+ (),+ |row| Ok((row.get(1)?, row.get(0)?)),+ )+ .map_err(Into::into)}pub fn drop_faces_feedback_tables(&self) -> Result<()> {- self.db- .execute_statements([- DELETE_CLUSTER_PERSON,- DELETE_NOT_PERSON_FEEDBACK,- CREATE_CLUSTER_PERSON_TABLE,- CREATE_NOT_PERSON_FEEDBACK_TABLE,- ])- .map_err(Into::into)+ self.execute_statements([+ DELETE_CLUSTER_PERSON,+ DELETE_NOT_PERSON_FEEDBACK,+ CREATE_CLUSTER_PERSON_TABLE,+ CREATE_NOT_PERSON_FEEDBACK_TABLE,+ ])+ .map_err(Into::into)}pub fn get_file_ids_of_person_id(&self, person_id: &str) -> Result<Vec<i64>> {- self.db- .read_column(- r#"+ self.read_column(+ r#"SELECT DISTINCT faces.file_idFROM cluster_personJOIN face_clusters@@ -312,9 +295,9 @@ impl MlDb {ON face_clusters.face_id = faces.face_idWHERE cluster_person.person_id = ?"#,- [person_id],- )- .map_err(Into::into)+ [person_id],+ )+ .map_err(Into::into)}}diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rsindex 5136fa9d95..c30b355afb 100644--- a/rust/crates/photos/src/ml_db/queries/pets.rs+++ b/rust/crates/photos/src/ml_db/queries/pets.rs@@ -1,7 +1,8 @@use std::collections::HashMap;use std::num::NonZeroUsize;-use crate::db::{Row, SqliteResult, bind_placeholders, pair, params_from_iter};+use super::helpers::{bind_placeholders, pair};+use rusqlite::{Result as SqliteResult, Row, params_from_iter};use super::unique_in_order;use crate::ml_db::{MlDb, Result};@@ -87,47 +88,45 @@ const UPSERT_PET_BODY: &str = r#"impl MlDb {pub fn bulk_insert_pet_faces(&self, pet_faces: &[PetFaceRow]) -> Result<()> {- self.db- .write_batches_committing_each(- UPSERT_PET_FACE,- const { NonZeroUsize::new(500).unwrap() },- pet_faces.iter().map(|pet_face| {- (- pet_face.file_id,- &pet_face.pet_face_id,- &pet_face.detection_json,- pet_face.face_vector_id,- pet_face.species,- pet_face.face_score,- pet_face.image_height,- pet_face.image_width,- pet_face.ml_version,- )- }),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ UPSERT_PET_FACE,+ const { NonZeroUsize::new(500).unwrap() },+ pet_faces.iter().map(|pet_face| {+ (+ pet_face.file_id,+ &pet_face.pet_face_id,+ &pet_face.detection_json,+ pet_face.face_vector_id,+ pet_face.species,+ pet_face.face_score,+ pet_face.image_height,+ pet_face.image_width,+ pet_face.ml_version,+ )+ }),+ )+ .map_err(Into::into)}pub fn bulk_insert_pet_bodies(&self, pet_bodies: &[PetBodyRow]) -> Result<()> {- self.db- .write_batches_committing_each(- UPSERT_PET_BODY,- const { NonZeroUsize::new(500).unwrap() },- pet_bodies.iter().map(|pet_body| {- (- pet_body.file_id,- &pet_body.pet_body_id,- &pet_body.detection_json,- pet_body.body_vector_id,- pet_body.species,- pet_body.score,- pet_body.image_height,- pet_body.image_width,- pet_body.ml_version,- )- }),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ UPSERT_PET_BODY,+ const { NonZeroUsize::new(500).unwrap() },+ pet_bodies.iter().map(|pet_body| {+ (+ pet_body.file_id,+ &pet_body.pet_body_id,+ &pet_body.detection_json,+ pet_body.body_vector_id,+ pet_body.species,+ pet_body.score,+ pet_body.image_height,+ pet_body.image_width,+ pet_body.ml_version,+ )+ }),+ )+ .map_err(Into::into)}pub fn update_pet_face_vector_ids(@@ -137,15 +136,14 @@ impl MlDb {if pet_face_id_to_vector_id.is_empty() {return Ok(());}- self.db- .write_batches_committing_each(- "UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",- const { NonZeroUsize::new(500).unwrap() },- pet_face_id_to_vector_id- .iter()- .map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ "UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",+ const { NonZeroUsize::new(500).unwrap() },+ pet_face_id_to_vector_id+ .iter()+ .map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),+ )+ .map_err(Into::into)}pub fn update_pet_body_vector_ids(@@ -155,54 +153,49 @@ impl MlDb {if pet_body_id_to_vector_id.is_empty() {return Ok(());}- self.db- .write_batches_committing_each(- "UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",- const { NonZeroUsize::new(500).unwrap() },- pet_body_id_to_vector_id- .iter()- .map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),- )- .map_err(Into::into)+ self.write_batches_committing_each(+ "UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",+ const { NonZeroUsize::new(500).unwrap() },+ pet_body_id_to_vector_id+ .iter()+ .map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),+ )+ .map_err(Into::into)}pub fn get_pet_faces_for_file_id(&self, file_upload_id: i64) -> Result<Vec<PetFaceRow>> {- self.db- .read_all(- "SELECT * FROM pet_faces WHERE file_id = ? AND species != -1",- [file_upload_id],- read_pet_face,- )- .map_err(Into::into)+ self.read_all(+ "SELECT * FROM pet_faces WHERE file_id = ? AND species != -1",+ [file_upload_id],+ read_pet_face,+ )+ .map_err(Into::into)}pub fn get_pet_bodies_for_file_id(&self, file_upload_id: i64) -> Result<Vec<PetBodyRow>> {- self.db- .read_all(- "SELECT * FROM pet_bodies WHERE file_id = ? AND species != -1",- [file_upload_id],- read_pet_body,- )- .map_err(Into::into)+ self.read_all(+ "SELECT * FROM pet_bodies WHERE file_id = ? AND species != -1",+ [file_upload_id],+ read_pet_body,+ )+ .map_err(Into::into)}pub fn pet_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {- self.db- .read_all(- "SELECT DISTINCT file_id, ml_version FROM pet_faces WHERE ml_version >= ?",- [minimum_ml_version],- pair,- )- .map_err(Into::into)+ self.read_all(+ "SELECT DISTINCT file_id, ml_version FROM pet_faces WHERE ml_version >= ?",+ [minimum_ml_version],+ pair,+ )+ .map_err(Into::into)}pub fn get_pet_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {- self.db- .read_value(- "SELECT COUNT(DISTINCT file_id) as count FROM pet_faces WHERE ml_version >= ?",- [minimum_ml_version],- )- .map_err(Into::into)+ self.read_value(+ "SELECT COUNT(DISTINCT file_id) as count FROM pet_faces WHERE ml_version >= ?",+ [minimum_ml_version],+ )+ .map_err(Into::into)}pub fn get_pet_rows_for_files(&self, file_ids: &[i64]) -> Result<PetRowsForFiles> {@@ -221,24 +214,20 @@ impl MlDb {WHERE file_id IN ({placeholders})"#);- let faces = self- .db- .read_all(&faces_sql, params_from_iter(file_ids), |row| {- Ok(PetFaceVectorRow {- pet_face_id: row.get(0)?,- face_vector_id: row.get(1)?,- species: row.get(2)?,- })- })?;- let bodies = self- .db- .read_all(&bodies_sql, params_from_iter(file_ids), |row| {- Ok(PetBodyVectorRow {- pet_body_id: row.get(0)?,- body_vector_id: row.get(1)?,- species: row.get(2)?,- })- })?;+ let faces = self.read_all(&faces_sql, params_from_iter(file_ids), |row| {+ Ok(PetFaceVectorRow {+ pet_face_id: row.get(0)?,+ face_vector_id: row.get(1)?,+ species: row.get(2)?,+ })+ })?;+ let bodies = self.read_all(&bodies_sql, params_from_iter(file_ids), |row| {+ Ok(PetBodyVectorRow {+ pet_body_id: row.get(0)?,+ body_vector_id: row.get(1)?,+ species: row.get(2)?,+ })+ })?;Ok(PetRowsForFiles { faces, bodies })}@@ -328,18 +317,16 @@ impl MlDb {return Ok(HashMap::new());}if create_if_missing {- self.db- .write_batch_atomic(insert_sql, unique_ids.iter().map(|id| [id]))?;+ self.write_batch_atomic(insert_sql, unique_ids.iter().map(|id| [id]))?;}let select_sql = select_prefix.to_owned() + " ({})";- self.db- .read_chunked_in(- &select_sql,- &unique_ids,- const { NonZeroUsize::new(800).unwrap() },- pair,- )- .map_err(Into::into)+ self.read_chunked_in(+ &select_sql,+ &unique_ids,+ const { NonZeroUsize::new(800).unwrap() },+ pair,+ )+ .map_err(Into::into)}}