Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][photos] Simplify the SQLite wrapper

laurenspriemf7af7c1a4e3a
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rs
index e5dbb7fe24..e7897f4ddc 100644
--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs
@@ -1,7 +1,6 @@
use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;
-use ente_photos::db;
use ente_photos::ml_db;
pub use ente_photos::ml_db::{
ClipEmbedding, ClipRow, ClusterCentroidRow, ClusterSummary, EmbeddingVector,
@@ -27,7 +26,7 @@ impl From<ml_db::Error> for MlDbError {
fn from(error: ml_db::Error) -> Self {
let message = ente_core::error::chain(&error);
match error {
- ml_db::Error::Database(db::Error::Downgrade { .. }) => Self::Downgrade { message },
+ ml_db::Error::Downgrade { .. } => Self::Downgrade { message },
_ => Self::Other { message },
}
}
@@ -847,3 +846,30 @@ impl MlDb {
.collect())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::{MlDbError, ml_db};
+
+ #[test]
+ fn downgrade_errors_keep_the_bridge_variant_and_message() {
+ let error = MlDbError::from(ml_db::Error::Downgrade {
+ current: 16,
+ target: 15,
+ });
+ assert!(matches!(
+ error,
+ MlDbError::Downgrade { message }
+ if message == "currentVersion(16) cannot be greater than toVersion(15)"
+ ));
+ }
+
+ #[test]
+ fn other_errors_keep_the_bridge_variant_and_message() {
+ let error = MlDbError::from(ml_db::Error::InvalidArgument("invalid value".into()));
+ assert!(matches!(
+ error,
+ MlDbError::Other { message } if message == "invalid value"
+ ));
+ }
+}
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rs
index 54450b8175..66fe5cba7b 100644
--- a/rust/crates/photos/src/db.rs
+++ b/rust/crates/photos/src/db.rs
@@ -1,83 +1,24 @@
-use std::collections::HashMap;
-use std::hash::Hash;
use std::num::NonZeroUsize;
-use std::ops::{Deref, DerefMut};
use std::path::Path;
use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
-use rusqlite::{OpenFlags, TransactionBehavior, types::FromSql};
+pub use rusqlite;
-pub use rusqlite::types;
-pub use rusqlite::{
- Error as SqliteError, OptionalExtension, Params, Result as SqliteResult, Row, ToSql,
- Transaction, params_from_iter,
-};
+use rusqlite::{Connection, OpenFlags, Result, Transaction, TransactionBehavior};
-const WRITER_PRAGMAS: &str = "PRAGMA journal_mode = WAL;
-PRAGMA synchronous = NORMAL;
-PRAGMA journal_size_limit = 6291456;";
const BUSY_TIMEOUT: Duration = Duration::from_secs(30);
-pub(crate) const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
- #[error(transparent)]
- Sqlite(#[from] SqliteError),
- #[error("currentVersion({current}) cannot be greater than toVersion({target})")]
- Downgrade { current: i64, target: i64 },
- #[error("reader_count must be at least 1")]
- InvalidReaderCount,
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
#[derive(Clone, Copy, Debug)]
pub struct OpenOptions {
- pub reader_count: usize,
+ pub reader_count: NonZeroUsize,
}
impl Default for OpenOptions {
fn default() -> Self {
- Self { reader_count: 1 }
- }
-}
-
-pub struct Connection(rusqlite::Connection);
-
-impl Connection {
- pub fn open(path: impl AsRef<Path>) -> Result<Self> {
- let connection = rusqlite::Connection::open(path)?;
- connection.busy_timeout(BUSY_TIMEOUT)?;
- connection.execute_batch(WRITER_PRAGMAS)?;
- Ok(Self(connection))
- }
-
- pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self> {
- let flags = OpenFlags::SQLITE_OPEN_READ_ONLY
- | OpenFlags::SQLITE_OPEN_URI
- | OpenFlags::SQLITE_OPEN_NO_MUTEX;
- let connection = rusqlite::Connection::open_with_flags(path, flags)?;
- connection.busy_timeout(BUSY_TIMEOUT)?;
- Ok(Self(connection))
- }
-
- pub fn open_in_memory() -> Result<Self> {
- Ok(Self(rusqlite::Connection::open_in_memory()?))
- }
-}
-
-impl Deref for Connection {
- type Target = rusqlite::Connection;
-
- fn deref(&self) -> &Self::Target {
- &self.0
- }
-}
-
-impl DerefMut for Connection {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.0
+ Self {
+ reader_count: NonZeroUsize::MIN,
+ }
}
}
@@ -86,114 +27,37 @@ pub struct Database {
}
impl Database {
- pub fn open(path: impl AsRef<Path>, migration_scripts: &[&str]) -> Result<Self> {
- Self::open_with_options(path, migration_scripts, OpenOptions::default())
- }
-
- pub fn open_with_options(
+ pub fn open<E: From<rusqlite::Error>>(
path: impl AsRef<Path>,
- migration_scripts: &[&str],
options: OpenOptions,
- ) -> Result<Self> {
- if options.reader_count == 0 {
- return Err(Error::InvalidReaderCount);
- }
+ initialize: impl FnOnce(&mut Connection) -> std::result::Result<(), E>,
+ ) -> std::result::Result<Self, E> {
let path = path.as_ref();
let mut writer = Connection::open(path)?;
- migrate(&mut writer, migration_scripts)?;
- let readers = (0..options.reader_count)
- .map(|_| Connection::open_read_only(path))
+ writer.busy_timeout(BUSY_TIMEOUT)?;
+ writer.pragma_update(None, "journal_mode", "WAL")?;
+ initialize(&mut writer)?;
+ let readers = (0..options.reader_count.get())
+ .map(|_| open_reader(path))
.collect::<Result<Vec<_>>>()?;
Ok(Self {
pool: Pool::new(writer, readers),
})
}
- pub fn read_all<C: FromIterator<T>, T, P: Params>(
- &self,
- sql: &str,
- parameters: P,
- map: impl FnMut(&Row<'_>) -> SqliteResult<T>,
- ) -> Result<C> {
- self.pool.read(|connection| {
- let mut statement = connection.prepare_cached(sql)?;
- let rows = statement.query_map(parameters, map)?;
- Ok(rows.collect::<SqliteResult<C>>()?)
- })
- }
-
- pub fn read_column<C: FromIterator<T>, T: FromSql, P: Params>(
- &self,
- sql: &str,
- parameters: P,
- ) -> Result<C> {
- self.read_all(sql, parameters, |row| row.get(0))
- }
-
- pub fn read_grouped<K: FromSql + Eq + Hash, V: FromSql, C: FromIterator<V>, P: Params>(
- &self,
- sql: &str,
- parameters: P,
- ) -> Result<HashMap<K, C>> {
- let rows: Vec<(K, V)> = self.read_all(sql, parameters, pair)?;
- Ok(group_into(rows))
+ pub fn read<T>(&self, read: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {
+ self.pool.read(read)
}
- pub fn read_value<T: FromSql>(&self, sql: &str, parameters: impl Params) -> Result<T> {
- self.pool.read(|connection| {
- Ok(connection
- .prepare_cached(sql)?
- .query_row(parameters, |row| row.get(0))?)
- })
- }
-
- pub fn read_optional<T: FromSql>(
- &self,
- sql: &str,
- parameters: impl Params,
- ) -> Result<Option<T>> {
- self.pool.read(|connection| {
- Ok(connection
- .prepare_cached(sql)?
- .query_row(parameters, |row| row.get(0))
- .optional()?)
- })
- }
-
- pub fn read_chunked_in<C: FromIterator<T>, T, I: ToSql>(
- &self,
- sql: &str,
- ids: &[I],
- chunk_size: NonZeroUsize,
- mut map: impl FnMut(&Row<'_>) -> SqliteResult<T>,
- ) -> Result<C> {
- let mut rows = Vec::new();
- for chunk in ids.chunks(chunk_size.get()) {
- let sql = expand_in_clause(sql, chunk.len());
- let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;
- rows.extend(chunk_rows);
- }
- Ok(rows.into_iter().collect())
- }
-
- pub fn read_transaction<T>(
- &self,
- read: impl FnOnce(&Transaction<'_>) -> Result<T>,
- ) -> Result<T> {
- self.pool.read(|connection| {
- let transaction =
- connection.transaction_with_behavior(TransactionBehavior::Deferred)?;
- let result = read(&transaction)?;
- transaction.commit()?;
- Ok(result)
- })
+ pub fn write<T>(&self, write: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {
+ self.pool.write(write)
}
pub fn write_transaction<T>(
&self,
write: impl FnOnce(&Transaction<'_>) -> Result<T>,
) -> Result<T> {
- self.pool.write(|connection| {
+ self.write(|connection| {
let transaction =
connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
let result = write(&transaction)?;
@@ -201,73 +65,15 @@ impl Database {
Ok(result)
})
}
+}
- pub fn execute(&self, sql: &str, parameters: impl Params) -> Result<usize> {
- self.pool
- .write(|connection| Ok(connection.prepare_cached(sql)?.execute(parameters)?))
- }
-
- pub fn execute_chunked_in<I: ToSql>(&self, sql: &str, ids: &[I]) -> Result<()> {
- if ids.is_empty() {
- return Ok(());
- }
- self.write_transaction(|transaction| {
- for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
- let sql = expand_in_clause(sql, chunk.len());
- transaction.execute(&sql, params_from_iter(chunk))?;
- }
- Ok(())
- })
- }
-
- pub fn execute_statements<'a>(
- &self,
- statements: impl IntoIterator<Item = &'a str>,
- ) -> Result<()> {
- self.write_transaction(|transaction| {
- for statement in statements {
- transaction.execute_batch(statement)?;
- }
- Ok(())
- })
- }
-
- pub fn write_batch_atomic<P: Params>(
- &self,
- sql: &str,
- parameter_sets: impl IntoIterator<Item = P>,
- ) -> Result<()> {
- self.write_transaction(|transaction| {
- let mut statement = transaction.prepare_cached(sql)?;
- for parameters in parameter_sets {
- statement.execute(parameters)?;
- }
- Ok(())
- })
- }
-
- pub fn write_batches_committing_each<P: Params>(
- &self,
- sql: &str,
- batch_size: NonZeroUsize,
- parameter_sets: impl IntoIterator<Item = P>,
- ) -> Result<()> {
- self.pool.write(|connection| {
- let mut parameter_sets = parameter_sets.into_iter().peekable();
- while parameter_sets.peek().is_some() {
- let transaction =
- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
- {
- let mut statement = transaction.prepare_cached(sql)?;
- for parameters in parameter_sets.by_ref().take(batch_size.get()) {
- statement.execute(parameters)?;
- }
- }
- transaction.commit()?;
- }
- Ok(())
- })
- }
+fn open_reader(path: &Path) -> Result<Connection> {
+ let flags = OpenFlags::SQLITE_OPEN_READ_ONLY
+ | OpenFlags::SQLITE_OPEN_URI
+ | OpenFlags::SQLITE_OPEN_NO_MUTEX;
+ let connection = Connection::open_with_flags(path, flags)?;
+ connection.busy_timeout(BUSY_TIMEOUT)?;
+ Ok(connection)
}
struct Pool {
@@ -338,183 +144,106 @@ fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(PoisonError::into_inner)
}
-fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<()> {
- let target = scripts.len() as i64;
- let probed: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?;
- check_not_downgrade(probed, target)?;
- if probed == target {
- return Ok(());
- }
-
- let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
- let current: i64 = transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;
- check_not_downgrade(current, target)?;
- if current == target {
- return Ok(());
- }
- for script in &scripts[current as usize..] {
- transaction.execute_batch(script)?;
- }
- transaction.pragma_update(None, "user_version", target)?;
- transaction.commit()?;
- Ok(())
-}
-
-fn check_not_downgrade(current: i64, target: i64) -> Result<()> {
- if current > target {
- return Err(Error::Downgrade { current, target });
- }
- Ok(())
-}
-
-pub fn bind_placeholders(count: usize) -> String {
- vec!["?"; count].join(", ")
-}
-
-fn expand_in_clause(sql: &str, count: usize) -> String {
- sql.replacen("{}", &bind_placeholders(count), 1)
-}
-
-pub(crate) fn pair<A: types::FromSql, B: types::FromSql>(row: &Row<'_>) -> SqliteResult<(A, B)> {
- Ok((row.get(0)?, row.get(1)?))
-}
-
-pub(crate) fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql> {
- value.iter().map(|value| value as &dyn ToSql).collect()
-}
-
-pub(crate) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(
- rows: impl IntoIterator<Item = (K, V)>,
-) -> HashMap<K, C> {
- let mut groups: HashMap<K, Vec<V>> = HashMap::new();
- for (key, value) in rows {
- groups.entry(key).or_default().push(value);
- }
- groups
- .into_iter()
- .map(|(key, values)| (key, values.into_iter().collect()))
- .collect()
-}
-
#[cfg(test)]
mod tests {
- use std::collections::{HashMap, HashSet};
use std::num::NonZeroUsize;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::mpsc::{self, RecvTimeoutError};
use std::thread;
use std::time::Duration;
- use super::{
- Connection, Database, Error, MAX_SQL_BIND_PARAMS_PER_QUERY, OpenOptions, bind_placeholders,
- lock, types,
- };
+ use rusqlite::{Connection, Result};
- const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";
- const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";
+ use super::{Database, OpenOptions, lock, open_reader};
fn open() -> (tempfile::TempDir, Database) {
let directory = tempfile::tempdir().unwrap();
- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();
+ let db = Database::open(
+ directory.path().join("items.db"),
+ OpenOptions::default(),
+ |connection| connection.execute_batch("CREATE TABLE items (id INTEGER PRIMARY KEY)"),
+ )
+ .unwrap();
(directory, db)
}
- fn pragma<T: types::FromSql>(connection: &Connection, name: &str) -> T {
- connection
- .pragma_query_value(None, name, |row| row.get(0))
+ fn value(db: &Database, sql: &str) -> i64 {
+ db.read(|connection| connection.query_row(sql, (), |row| row.get(0)))
.unwrap()
}
#[test]
- fn open_applies_connection_pragmas() {
- let directory = tempfile::tempdir().unwrap();
- let connection = Connection::open(directory.path().join("test.db")).unwrap();
- assert_eq!(pragma::<String>(&connection, "journal_mode"), "wal");
- assert_eq!(pragma::<i64>(&connection, "synchronous"), 1);
- assert_eq!(pragma::<i64>(&connection, "journal_size_limit"), 6291456);
- assert_eq!(pragma::<i64>(&connection, "busy_timeout"), 30000);
+ fn database_is_send_and_sync() {
+ fn assert_send_sync<T: Send + Sync>() {}
+ assert_send_sync::<Database>();
}
#[test]
- fn read_only_connection_reads_wal_without_changing_writer_settings() {
- let directory = tempfile::tempdir().unwrap();
- let path = directory.path().join("test.db");
- let writer = Connection::open(&path).unwrap();
- writer
- .execute_batch("CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (7);")
- .unwrap();
- let reader = Connection::open_read_only(&path).unwrap();
- let id: i64 = reader
- .query_row("SELECT id FROM t", (), |row| row.get(0))
- .unwrap();
- assert_eq!(id, 7);
- assert!(reader.execute("INSERT INTO t VALUES (8)", ()).is_err());
- assert_eq!(pragma::<String>(&reader, "journal_mode"), "wal");
- assert_eq!(pragma::<i64>(&reader, "busy_timeout"), 30000);
- assert_eq!(pragma::<i64>(&writer, "synchronous"), 1);
- assert_eq!(pragma::<i64>(&writer, "journal_size_limit"), 6291456);
+ fn open_initializes_before_readers_and_configures_connections() {
+ let (_directory, db) = open();
+ db.write(|connection| {
+ let journal: String =
+ connection.pragma_query_value(None, "journal_mode", |row| row.get(0))?;
+ let timeout: i64 =
+ connection.pragma_query_value(None, "busy_timeout", |row| row.get(0))?;
+ assert_eq!(journal, "wal");
+ assert_eq!(timeout, 30000);
+ connection.execute("INSERT INTO items VALUES (7)", ())
+ })
+ .unwrap();
+ assert_eq!(value(&db, "SELECT id FROM items"), 7);
+ assert_eq!(value(&db, "PRAGMA busy_timeout"), 30000);
+ assert!(
+ db.read(|connection| connection.execute("DELETE FROM items", ()))
+ .is_err()
+ );
+ assert_eq!(value(&db, "SELECT id FROM items"), 7);
}
#[test]
- fn read_only_connection_does_not_create_database() {
+ fn open_preserves_an_existing_schema_version() {
let directory = tempfile::tempdir().unwrap();
- let path = directory.path().join("missing.db");
- assert!(Connection::open_read_only(&path).is_err());
- assert!(!path.exists());
+ let path = directory.path().join("items.db");
+ {
+ let connection = Connection::open(&path).unwrap();
+ connection.execute_batch("CREATE TABLE items (id INTEGER); INSERT INTO items VALUES (7); PRAGMA user_version = 42;").unwrap();
+ }
+ let db = Database::open(&path, OpenOptions::default(), |_| {
+ Ok::<_, rusqlite::Error>(())
+ })
+ .unwrap();
+ assert_eq!(value(&db, "PRAGMA user_version"), 42);
+ assert_eq!(value(&db, "SELECT id FROM items"), 7);
}
#[test]
- fn in_memory_connection_runs_statements() {
- let connection = Connection::open_in_memory().unwrap();
- connection
- .execute_batch("CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (7);")
- .unwrap();
- let id: i64 = connection
- .query_row("SELECT id FROM t", (), |row| row.get(0))
- .unwrap();
- assert_eq!(id, 7);
- }
+ fn open_propagates_the_initializers_error_type() {
+ #[derive(Debug, PartialEq, Eq)]
+ struct InitializationError(String);
- #[test]
- fn reader_queries_reject_writes() {
- let (_directory, db) = open();
- db.execute("INSERT INTO items (id) VALUES (?)", [7])
- .unwrap();
- let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());
- assert!(matches!(deleted, Err(Error::Sqlite(_))));
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())
- .unwrap(),
- [7]
+ impl From<rusqlite::Error> for InitializationError {
+ fn from(error: rusqlite::Error) -> Self {
+ Self(error.to_string())
+ }
+ }
+
+ let directory = tempfile::tempdir().unwrap();
+ let result = Database::open(
+ directory.path().join("items.db"),
+ OpenOptions::default(),
+ |_| Err(InitializationError("schema initialization failed".into())),
+ );
+ assert!(
+ matches!(result, Err(InitializationError(message)) if message == "schema initialization failed")
);
}
#[test]
- fn failed_batch_rolls_back_and_writer_recovers() {
- let (_directory, db) = open();
- assert!(
- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]])
- .is_err()
- );
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 0
- );
- assert_eq!(
- db.execute("INSERT INTO items (id) VALUES (?)", [3])
- .unwrap(),
- 1
- );
- assert_eq!(
- db.execute("DELETE FROM items WHERE id = ?", [4]).unwrap(),
- 0
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())
- .unwrap(),
- [3]
- );
+ fn read_only_connection_does_not_create_database() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("missing.db");
+ assert!(open_reader(&path).is_err());
+ assert!(!path.exists());
}
#[test]
@@ -526,51 +255,40 @@ mod tests {
Ok(())
});
assert!(result.is_err());
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 0
- );
+ assert_eq!(value(&db, "SELECT COUNT(*) FROM items"), 0);
assert!(
catch_unwind(AssertUnwindSafe(|| {
- let _: super::Result<()> = db.write_transaction(|transaction| {
+ let _: Result<()> = db.write_transaction(|transaction| {
transaction.execute("INSERT INTO items VALUES (?)", [3])?;
panic!("transaction callback failed");
});
}))
.is_err()
);
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 0
- );
+ assert_eq!(value(&db, "SELECT COUNT(*) FROM items"), 0);
let total = db
.write_transaction(|transaction| {
transaction.execute("INSERT INTO items VALUES (?)", [1])?;
transaction.execute("INSERT INTO items VALUES (?)", [2])?;
- Ok(transaction
- .query_row("SELECT SUM(id) FROM items", (), |row| row.get::<_, i64>(0))?)
+ transaction.query_row("SELECT SUM(id) FROM items", (), |row| row.get::<_, i64>(0))
})
.unwrap();
assert_eq!(total, 3);
- assert_eq!(
- db.read_value::<i64>("SELECT SUM(id) FROM items", ())
- .unwrap(),
- 3
- );
+ assert_eq!(value(&db, "SELECT SUM(id) FROM items"), 3);
}
#[test]
- fn read_transaction_keeps_a_snapshot_and_rejects_writes() {
+ fn reader_can_use_a_snapshot_while_the_writer_commits() {
let (_directory, db) = open();
- db.execute("INSERT INTO items VALUES (?)", [1]).unwrap();
+ db.write(|connection| connection.execute("INSERT INTO items VALUES (?)", [1]))
+ .unwrap();
let counts = db
- .read_transaction(|transaction| {
+ .read(|connection| {
+ let transaction = connection.transaction()?;
let before: i64 =
transaction.query_row("SELECT COUNT(*) FROM items", (), |row| row.get(0))?;
- db.execute("INSERT INTO items VALUES (?)", [2])?;
+ db.write(|writer| writer.execute("INSERT INTO items VALUES (?)", [2]))?;
let after: i64 =
transaction.query_row("SELECT COUNT(*) FROM items", (), |row| row.get(0))?;
assert!(
@@ -578,55 +296,25 @@ mod tests {
.execute("INSERT INTO items VALUES (?)", [3])
.is_err()
);
+ transaction.commit()?;
Ok((before, after))
})
.unwrap();
assert_eq!(counts, (1, 1));
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 2
- );
- }
-
- #[test]
- fn statement_sequence_rolls_back_if_a_later_statement_fails() {
- let (_directory, db) = open();
- assert!(
- db.execute_statements([
- "INSERT INTO items VALUES (1)",
- "INSERT INTO missing_table VALUES (2)",
- ])
- .is_err()
- );
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 0
- );
- }
-
- #[test]
- fn zero_readers_is_rejected_before_creating_a_database() {
- let directory = tempfile::tempdir().unwrap();
- let path = directory.path().join("items.db");
- assert!(matches!(
- Database::open_with_options(&path, &[CREATE_ITEMS], OpenOptions { reader_count: 0 }),
- Err(Error::InvalidReaderCount)
- ));
- assert!(!path.exists());
+ assert_eq!(value(&db, "SELECT COUNT(*) FROM items"), 2);
}
#[test]
fn configured_readers_bound_concurrency_and_wake_on_any_returned_connection() {
for reader_count in [1, 2, 3] {
let directory = tempfile::tempdir().unwrap();
- let path = directory.path().join("items.db");
- let db = if reader_count == 1 {
- Database::open(path, &[CREATE_ITEMS])
- } else {
- Database::open_with_options(path, &[CREATE_ITEMS], OpenOptions { reader_count })
- }
+ let db = Database::open(
+ directory.path().join("items.db"),
+ OpenOptions {
+ reader_count: NonZeroUsize::new(reader_count).unwrap(),
+ },
+ |_| Ok::<_, rusqlite::Error>(()),
+ )
.unwrap();
thread::scope(|scope| {
@@ -637,8 +325,8 @@ mod tests {
release_readers.push(release_tx);
let db = &db;
scope.spawn(move || {
- db.read_transaction(|transaction| {
- transaction.query_row("SELECT 1", (), |_| Ok(()))?;
+ db.read(|connection| {
+ connection.query_row("SELECT 1", (), |_| Ok(()))?;
entered_tx.send(()).unwrap();
release_rx.recv().unwrap();
Ok(())
@@ -651,9 +339,7 @@ mod tests {
let (result_tx, result_rx) = mpsc::channel();
let db = &db;
scope.spawn(move || {
- result_tx
- .send(db.read_value::<i64>("SELECT 7", ()))
- .unwrap();
+ result_tx.send(value(db, "SELECT 7")).unwrap();
});
let was_blocked = matches!(
result_rx.recv_timeout(Duration::from_millis(100)),
@@ -665,7 +351,7 @@ mod tests {
release.send(()).unwrap();
}
assert!(was_blocked);
- assert_eq!(result.unwrap().unwrap(), 7);
+ assert_eq!(result.unwrap(), 7);
});
}
}
@@ -675,284 +361,14 @@ mod tests {
let (_directory, db) = open();
assert!(
catch_unwind(AssertUnwindSafe(|| {
- let _: super::Result<()> = db.read_transaction(|transaction| {
- transaction.query_row("SELECT 1", (), |_| Ok(()))?;
+ let _: Result<()> = db.read(|connection| {
+ connection.query_row("SELECT 1", (), |_| Ok(()))?;
panic!("reader callback failed");
});
}))
.is_err()
);
assert_eq!(lock(&db.pool.readers).len(), 1);
- assert_eq!(db.read_value::<i64>("SELECT 7", ()).unwrap(), 7);
- }
-
- #[test]
- fn chunked_delete_handles_empty_lists_and_rolls_back_on_failure() {
- let (_directory, db) = open();
- let ids: Vec<i64> = (0..10_001).collect();
- db.write_batch_atomic(
- "INSERT INTO items (id) VALUES (?)",
- ids.iter().map(|id| [id]),
- )
- .unwrap();
- db.execute_chunked_in::<i64>("DELETE FROM items WHERE id IN ({})", &[])
- .unwrap();
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 10_001
- );
- db.execute_statements(["CREATE TRIGGER reject_last_delete BEFORE DELETE ON items
- WHEN OLD.id = 10000 BEGIN SELECT RAISE(ABORT, 'cannot delete'); END"])
- .unwrap();
- assert!(
- db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)
- .is_err()
- );
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 10_001
- );
- db.execute_statements(["DROP TRIGGER reject_last_delete"])
- .unwrap();
- db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)
- .unwrap();
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 0
- );
- }
-
- #[test]
- fn batched_writes_commit_each_completed_batch() {
- let (_directory, db) = open();
- db.write_batches_committing_each(
- "INSERT INTO missing_table VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- Vec::<[i64; 1]>::new(),
- )
- .unwrap();
- assert!(
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- [[1], [2], [3], [4], [1]]
- )
- .is_err()
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- [1, 2, 3, 4]
- );
- assert!(
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- [[5], [6], [7], [5], [8]]
- )
- .is_err()
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- [1, 2, 3, 4, 5, 6]
- );
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- const { NonZeroUsize::new(2).unwrap() },
- [[7], [8], [9]],
- )
- .unwrap();
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- [1, 2, 3, 4, 5, 6, 7, 8, 9]
- );
- assert!(
- db.write_batches_committing_each(
- "INSERT INTO items (id) VALUES (?)",
- NonZeroUsize::MIN,
- [[10], [11], [10]],
- )
- .is_err()
- );
- assert_eq!(
- db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
- .unwrap(),
- (1..=11).collect::<Vec<_>>()
- );
- }
-
- #[test]
- fn chunked_reads_query_each_chunk_in_order() {
- let (_directory, db) = open();
- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))
- .unwrap();
- let ids: Vec<i64> = (1..=7).collect();
- let one_at_a_time: Vec<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
- &ids,
- NonZeroUsize::MIN,
- |row| row.get(0),
- )
- .unwrap();
- assert_eq!(one_at_a_time, ids);
- let by_chunk: Vec<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
- &ids,
- const { NonZeroUsize::new(3).unwrap() },
- |row| row.get(0),
- )
- .unwrap();
- assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);
- let single_chunk: Vec<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
- &ids,
- const { NonZeroUsize::new(10).unwrap() },
- |row| row.get(0),
- )
- .unwrap();
- assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);
- let none: Vec<i64> = db
- .read_chunked_in(
- "SELECT id FROM missing_table WHERE id IN ({})",
- &Vec::<i64>::new(),
- const { NonZeroUsize::new(3).unwrap() },
- |row| row.get(0),
- )
- .unwrap();
- assert!(none.is_empty());
- let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();
- let found: HashSet<i64> = db
- .read_chunked_in(
- "SELECT id FROM items WHERE id IN ({})",
- &many,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
- |row| row.get(0),
- )
- .unwrap();
- assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));
- }
-
- #[test]
- fn grouped_reads_collect_values_per_key() {
- let (_directory, db) = open();
- db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=5).map(|id| [id]))
- .unwrap();
- let by_parity: HashMap<i64, Vec<i64>> = db
- .read_grouped("SELECT id % 2, id FROM items ORDER BY id", ())
- .unwrap();
- assert_eq!(
- by_parity,
- HashMap::from([(1, vec![1, 3, 5]), (0, vec![2, 4])])
- );
- let by_label: HashMap<String, HashSet<i64>> = db
- .read_grouped(
- "SELECT CASE WHEN id < 3 THEN 'low' ELSE 'high' END, id FROM items",
- (),
- )
- .unwrap();
- assert_eq!(
- by_label,
- HashMap::from([
- ("low".to_string(), HashSet::from([1, 2])),
- ("high".to_string(), HashSet::from([3, 4, 5]))
- ])
- );
- let empty: HashMap<i64, Vec<i64>> = db
- .read_grouped("SELECT id, id FROM items WHERE id > 5", ())
- .unwrap();
- assert!(empty.is_empty());
- }
-
- #[test]
- fn open_migrates_a_caller_schema_and_refuses_downgrade() {
- let directory = tempfile::tempdir().unwrap();
- let path = directory.path().join("items.db");
- {
- let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();
- db.execute("INSERT INTO items (id) VALUES (?)", [7])
- .unwrap();
- }
- {
- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();
- assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);
- assert_eq!(
- db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [7])
- .unwrap(),
- Some("item".to_string())
- );
- assert_eq!(
- db.read_optional::<String>("SELECT label FROM items WHERE id = ?", [8])
- .unwrap(),
- None
- );
- }
- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();
- assert_eq!(
- db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
- .unwrap(),
- 1
- );
- assert!(matches!(
- Database::open(&path, &[CREATE_ITEMS]),
- Err(Error::Downgrade {
- current: 2,
- target: 1
- })
- ));
- }
-
- #[test]
- fn failed_migration_keeps_previous_schema_and_version() {
- let directory = tempfile::tempdir().unwrap();
- let path = directory.path().join("items.db");
- {
- let db = Database::open(&path, &[CREATE_ITEMS]).unwrap();
- db.execute("INSERT INTO items (id) VALUES (?)", [7])
- .unwrap();
- }
- assert!(
- Database::open(
- &path,
- &[
- CREATE_ITEMS,
- ADD_LABEL,
- "INSERT INTO missing_table VALUES (1)"
- ]
- )
- .is_err()
- );
- {
- let connection = Connection::open(&path).unwrap();
- let version: i64 = connection
- .pragma_query_value(None, "user_version", |row| row.get(0))
- .unwrap();
- assert_eq!(version, 1);
- let columns: Vec<String> = connection
- .prepare("SELECT name FROM pragma_table_info('items')")
- .unwrap()
- .query_map((), |row| row.get(0))
- .unwrap()
- .collect::<Result<_, _>>()
- .unwrap();
- assert_eq!(columns, ["id"]);
- }
- let db = Database::open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();
- assert_eq!(db.read_value::<i64>("SELECT id FROM items", ()).unwrap(), 7);
- assert_eq!(db.read_value::<i64>("PRAGMA user_version", ()).unwrap(), 2);
- }
-
- #[test]
- fn bind_placeholders_joins_question_marks() {
- assert_eq!(bind_placeholders(0), "");
- assert_eq!(bind_placeholders(1), "?");
- assert_eq!(bind_placeholders(3), "?, ?, ?");
+ assert_eq!(value(&db, "SELECT 7"), 7);
}
}
diff --git a/rust/crates/photos/src/ml_db/migrations.rs b/rust/crates/photos/src/ml_db/migrations.rs
new file mode 100644
index 0000000000..eed97c5b83
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/migrations.rs
@@ -0,0 +1,154 @@
+use rusqlite::{Connection, TransactionBehavior};
+
+use super::{Error, Result};
+
+pub(super) fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<()> {
+ let target = scripts.len() as i64;
+ let probed: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?;
+ check_not_downgrade(probed, target)?;
+ if probed == target {
+ return Ok(());
+ }
+
+ let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let current: i64 = transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;
+ check_not_downgrade(current, target)?;
+ if current == target {
+ return Ok(());
+ }
+ for script in &scripts[current as usize..] {
+ transaction.execute_batch(script)?;
+ }
+ transaction.pragma_update(None, "user_version", target)?;
+ transaction.commit()?;
+ Ok(())
+}
+
+fn check_not_downgrade(current: i64, target: i64) -> Result<()> {
+ if current > target {
+ return Err(Error::Downgrade { current, target });
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use std::path::Path;
+ use std::sync::Barrier;
+ use std::thread;
+
+ use crate::db::{Database, OpenOptions};
+ use rusqlite::Connection;
+
+ use super::migrate;
+ use crate::ml_db::{Error, Result};
+
+ const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";
+ const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";
+
+ fn open(path: &Path, scripts: &[&str]) -> Result<Database> {
+ Database::open(path, OpenOptions::default(), |connection| {
+ migrate(connection, scripts)
+ })
+ }
+
+ #[test]
+ fn upgrades_keep_data_and_refuse_downgrades() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("items.db");
+ {
+ let db = open(&path, &[CREATE_ITEMS]).unwrap();
+ db.write(|connection| connection.execute("INSERT INTO items VALUES (7)", ()))
+ .unwrap();
+ }
+ let db = open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();
+ let (version, label): (i64, String) = db
+ .read(|connection| {
+ Ok((
+ connection.pragma_query_value(None, "user_version", |row| row.get(0))?,
+ connection
+ .query_row("SELECT label FROM items WHERE id = 7", (), |row| row.get(0))?,
+ ))
+ })
+ .unwrap();
+ assert_eq!(version, 2);
+ assert_eq!(label, "item");
+ assert!(matches!(
+ open(&path, &[CREATE_ITEMS]),
+ Err(Error::Downgrade {
+ current: 2,
+ target: 1
+ })
+ ));
+ }
+
+ #[test]
+ fn failed_migration_keeps_previous_schema_and_version() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("items.db");
+ {
+ let db = open(&path, &[CREATE_ITEMS]).unwrap();
+ db.write(|connection| connection.execute("INSERT INTO items VALUES (7)", ()))
+ .unwrap();
+ }
+ assert!(
+ open(
+ &path,
+ &[
+ CREATE_ITEMS,
+ ADD_LABEL,
+ "INSERT INTO missing_table VALUES (1)"
+ ]
+ )
+ .is_err()
+ );
+ {
+ let connection = Connection::open(&path).unwrap();
+ let version: i64 = connection
+ .pragma_query_value(None, "user_version", |row| row.get(0))
+ .unwrap();
+ let columns = connection
+ .prepare("SELECT name FROM pragma_table_info('items')")
+ .unwrap()
+ .query_map((), |row| row.get::<_, String>(0))
+ .unwrap()
+ .collect::<rusqlite::Result<Vec<_>>>()
+ .unwrap();
+ assert_eq!(version, 1);
+ assert_eq!(columns, ["id"]);
+ }
+ let db = open(&path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();
+ assert_eq!(
+ db.read(|connection| connection
+ .query_row("SELECT id FROM items", (), |row| row.get::<_, i64>(0)))
+ .unwrap(),
+ 7
+ );
+ }
+
+ #[test]
+ fn concurrent_open_applies_each_migration_once() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("items.db");
+ drop(open(&path, &[CREATE_ITEMS]).unwrap());
+ let barrier = Barrier::new(2);
+ thread::scope(|scope| {
+ for _ in 0..2 {
+ let path = &path;
+ let barrier = &barrier;
+ scope.spawn(move || {
+ barrier.wait();
+ let db = open(path, &[CREATE_ITEMS, ADD_LABEL]).unwrap();
+ let version = db
+ .read(|connection| {
+ connection.pragma_query_value(None, "user_version", |row| {
+ row.get::<_, i64>(0)
+ })
+ })
+ .unwrap();
+ assert_eq!(version, 2);
+ });
+ }
+ });
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index 6a4a95f4ca..7426d3049b 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -1,7 +1,9 @@
+mod migrations;
mod queries;
mod schema;
mod vector_encoding;
+use std::num::NonZeroUsize;
use std::path::Path;
use crate::db::{Database, OpenOptions};
@@ -26,7 +28,9 @@ pub use vector_encoding::{decode_evector, decode_f32, encode_evector, encode_f32
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error(transparent)]
- Database(#[from] crate::db::Error),
+ Database(#[from] rusqlite::Error),
+ #[error("currentVersion({current}) cannot be greater than toVersion({target})")]
+ Downgrade { current: i64, target: i64 },
#[error("{0}")]
Codec(String),
#[error(transparent)]
@@ -46,38 +50,43 @@ pub struct MlDb {
impl MlDb {
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
Ok(Self {
- db: Database::open_with_options(
+ db: Database::open(
path,
- &schema::MIGRATION_SCRIPTS,
- OpenOptions { reader_count: 2 },
+ OpenOptions {
+ reader_count: const { NonZeroUsize::new(2).unwrap() },
+ },
+ |connection| {
+ connection.execute_batch(
+ "PRAGMA synchronous = NORMAL; PRAGMA journal_size_limit = 6291456;",
+ )?;
+ migrations::migrate(connection, &schema::MIGRATION_SCRIPTS)
+ },
)?,
})
}
pub fn clear_non_pet_tables(&self) -> Result<()> {
- self.db
- .execute_statements([
- schema::DELETE_FACES,
- schema::DELETE_FACE_CLUSTERS,
- schema::DELETE_CLUSTER_PERSON,
- schema::DELETE_CLUSTER_SUMMARY,
- schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
- schema::DELETE_NOT_PERSON_FEEDBACK,
- schema::DELETE_CLIP_EMBEDDINGS,
- schema::DELETE_FILE_DATA,
- ])
- .map_err(Into::into)
+ self.execute_statements([
+ schema::DELETE_FACES,
+ schema::DELETE_FACE_CLUSTERS,
+ schema::DELETE_CLUSTER_PERSON,
+ schema::DELETE_CLUSTER_SUMMARY,
+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ schema::DELETE_NOT_PERSON_FEEDBACK,
+ schema::DELETE_CLIP_EMBEDDINGS,
+ schema::DELETE_FILE_DATA,
+ ])
+ .map_err(Into::into)
}
pub fn clear_pet_tables(&self) -> Result<()> {
- self.db
- .execute_statements([
- schema::DELETE_PET_FACES,
- schema::DELETE_PET_BODIES,
- schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
- schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
- ])
- .map_err(Into::into)
+ self.execute_statements([
+ schema::DELETE_PET_FACES,
+ schema::DELETE_PET_BODIES,
+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
+ ])
+ .map_err(Into::into)
}
}
@@ -90,7 +99,7 @@ mod tests {
use super::queries::clip::tests::full_clip;
use super::queries::{caches, clip, clusters, faces, filedata, persons, pets};
use super::{Error, MlDb, TARGET_VERSION};
- use crate::db::Connection;
+ use rusqlite::Connection;
use tempfile::TempDir;
type Query<T> = fn(&MlDb) -> Result<T, Error>;
@@ -254,6 +263,22 @@ mod tests {
);
}
+ #[test]
+ fn ml_connections_keep_existing_settings() {
+ let (_directory, db) = open();
+ let (synchronous, journal_size_limit): (i64, i64) = db
+ .db
+ .write(|connection| {
+ Ok((
+ connection.pragma_query_value(None, "synchronous", |row| row.get(0))?,
+ connection.pragma_query_value(None, "journal_size_limit", |row| row.get(0))?,
+ ))
+ })
+ .unwrap();
+ assert_eq!(synchronous, 1);
+ assert_eq!(journal_size_limit, 6291456);
+ }
+
#[test]
fn ml_db_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
@@ -319,7 +344,7 @@ mod tests {
.pragma_update(None, "user_version", 16)
.unwrap();
match MlDb::open(&path) {
- Err(Error::Database(crate::db::Error::Downgrade { current, target })) => {
+ Err(Error::Downgrade { current, target }) => {
assert_eq!(current, 16);
assert_eq!(target, 15);
}
diff --git a/rust/crates/photos/src/ml_db/queries/caches.rs b/rust/crates/photos/src/ml_db/queries/caches.rs
index 4e153dba4e..d9a913ffb1 100644
--- a/rust/crates/photos/src/ml_db/queries/caches.rs
+++ b/rust/crates/photos/src/ml_db/queries/caches.rs
@@ -9,7 +9,7 @@ const THREE_MONTHS_MILLIS: i64 = 90 * 24 * 60 * 60 * 1000;
impl MlDb {
pub fn put_repeated_text_embedding_cache(&self, query: &str, embedding: &[f64]) -> Result<()> {
let embedding_bytes = encode_f32(embedding.iter().map(|value| *value as f32));
- self.db.execute(
+ self.execute(
r#"
INSERT OR REPLACE INTO text_embeddings_cache (text_query, embedding, ml_version, created_at)
VALUES (?, ?, ?, ?)
@@ -20,7 +20,7 @@ impl MlDb {
}
pub fn get_repeated_text_embedding_cache(&self, query: &str) -> Result<Option<Vec<f32>>> {
- let results: Vec<(Vec<u8>, i64, i64)> = self.db.read_all(
+ let results: Vec<(Vec<u8>, i64, i64)> = self.read_all(
r#"
SELECT embedding, ml_version, created_at
FROM text_embeddings_cache
@@ -38,7 +38,7 @@ impl MlDb {
return Ok(Some(decode_f32(embedding)));
}
}
- self.db.execute(
+ self.execute(
"DELETE FROM text_embeddings_cache WHERE text_query = ?",
[query],
)?;
@@ -50,7 +50,7 @@ impl MlDb {
person_or_cluster_id: &str,
face_id: &str,
) -> Result<()> {
- self.db.execute(
+ self.execute(
"INSERT OR REPLACE INTO face_cache (person_or_cluster_id, face_id) VALUES (?, ?)",
[person_or_cluster_id, face_id],
)?;
@@ -61,19 +61,18 @@ impl MlDb {
&self,
person_or_cluster_id: &str,
) -> Result<Option<String>> {
- self.db
- .read_optional(
- "SELECT face_id FROM face_cache WHERE person_or_cluster_id = ?",
- [person_or_cluster_id],
- )
- .map_err(Into::into)
+ self.read_optional(
+ "SELECT face_id FROM face_cache WHERE person_or_cluster_id = ?",
+ [person_or_cluster_id],
+ )
+ .map_err(Into::into)
}
pub fn remove_face_id_cached_for_person_or_cluster(
&self,
person_or_cluster_id: &str,
) -> Result<()> {
- self.db.execute(
+ self.execute(
"DELETE FROM face_cache WHERE person_or_cluster_id = ?",
[person_or_cluster_id],
)?;
@@ -91,8 +90,8 @@ fn now_millis() -> i64 {
#[cfg(test)]
pub(in crate::ml_db) mod tests {
use super::MlDb;
- use crate::db::Connection;
use crate::ml_db::tests::{cases, check, open};
+ use rusqlite::Connection;
use tempfile::TempDir;
pub(in crate::ml_db) fn seed(db: &MlDb) {
diff --git a/rust/crates/photos/src/ml_db/queries/clip.rs b/rust/crates/photos/src/ml_db/queries/clip.rs
index f170f37a68..aa6c28dd47 100644
--- a/rust/crates/photos/src/ml_db/queries/clip.rs
+++ b/rust/crates/photos/src/ml_db/queries/clip.rs
@@ -1,6 +1,6 @@
use std::collections::HashMap;
-use crate::db::pair;
+use super::helpers::pair;
use crate::ml_db::schema;
use crate::ml_db::vector_encoding::{decode_f32, encode_f32};
use crate::ml_db::{MlDb, Result};
@@ -32,8 +32,7 @@ pub struct ClipRow {
impl MlDb {
pub fn get_all_clip_vectors(&self) -> Result<Vec<EmbeddingVector>> {
let rows: Vec<(i64, Vec<u8>)> =
- self.db
- .read_all("SELECT file_id, embedding FROM clip", (), pair)?;
+ self.read_all("SELECT file_id, embedding FROM clip", (), pair)?;
Ok(rows
.into_iter()
.map(|(file_id, embedding)| EmbeddingVector {
@@ -45,81 +44,73 @@ impl MlDb {
}
pub fn clip_indexed_file_with_version(&self) -> Result<HashMap<i64, i64>> {
- self.db
- .read_all("SELECT file_id , ml_version FROM clip", (), pair)
+ self.read_all("SELECT file_id , ml_version FROM clip", (), pair)
.map_err(Into::into)
}
pub fn get_clip_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
- self.db
- .read_value(
- "SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ?",
- [minimum_ml_version],
- )
- .map_err(Into::into)
+ self.read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ?",
+ [minimum_ml_version],
+ )
+ .map_err(Into::into)
}
pub fn get_clip_vectorizable_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
- self.db
- .read_value(
- r#"
+ self.read_value(
+ r#"
SELECT COUNT(DISTINCT file_id) as count
FROM clip
WHERE ml_version >= ?
AND LENGTH(embedding) = ?
"#,
- [minimum_ml_version, CLIP_EMBEDDING_BYTES_LENGTH],
- )
- .map_err(Into::into)
+ [minimum_ml_version, CLIP_EMBEDDING_BYTES_LENGTH],
+ )
+ .map_err(Into::into)
}
pub fn insert_clip_rows(&self, embeddings: &[ClipEmbedding]) -> Result<()> {
if let [embedding] = embeddings {
- self.db.execute(
+ self.execute(
"INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) VALUES (?, ?, ?)",
clip_row(embedding),
)?;
return Ok(());
}
- self.db
- .write_batch_atomic(
- "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) values(?, ?, ?)",
- embeddings.iter().map(clip_row),
- )
- .map_err(Into::into)
+ self.write_batch_atomic(
+ "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) values(?, ?, ?)",
+ embeddings.iter().map(clip_row),
+ )
+ .map_err(Into::into)
}
pub fn delete_clip_rows(&self, file_ids: &[i64]) -> Result<()> {
- self.db
- .execute_chunked_in("DELETE FROM clip WHERE file_id IN ({})", file_ids)
+ self.execute_chunked_in("DELETE FROM clip WHERE file_id IN ({})", file_ids)
.map_err(Into::into)
}
pub fn delete_all_clip_rows(&self) -> Result<()> {
- self.db
- .execute_statements([schema::DELETE_CLIP_EMBEDDINGS])
+ self.execute_statements([schema::DELETE_CLIP_EMBEDDINGS])
.map_err(Into::into)
}
pub fn count_clip_rows(&self) -> Result<i64> {
- self.db
- .read_value("SELECT COUNT(file_id) as total FROM clip", ())
+ self.read_value("SELECT COUNT(file_id) as total FROM clip", ())
.map_err(Into::into)
}
pub fn get_clip_rows_page(&self, limit: i64, offset: i64) -> Result<Vec<ClipRow>> {
- self.db
- .read_all(
- "SELECT file_id, embedding FROM clip ORDER BY file_id DESC LIMIT ? OFFSET ?",
- [limit, offset],
- |row| {
- Ok(ClipRow {
- file_id: row.get(0)?,
- embedding: row.get(1)?,
- })
- },
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT file_id, embedding FROM clip ORDER BY file_id DESC LIMIT ? OFFSET ?",
+ [limit, offset],
+ |row| {
+ Ok(ClipRow {
+ file_id: row.get(0)?,
+ embedding: row.get(1)?,
+ })
+ },
+ )
+ .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index 976a0fdebf..a4dc7a2195 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -1,10 +1,10 @@
use std::collections::{HashMap, HashSet};
use std::num::NonZeroUsize;
-use crate::db::{
- MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, bind_placeholders, group_into,
- optional_parameter, pair, params_from_iter,
+use super::helpers::{
+ MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,
};
+use rusqlite::{Result as SqliteResult, Row, params_from_iter};
use super::faces::{file_id_from_face_id, is_bad_face_for_clustering};
use super::unique_in_order;
@@ -48,32 +48,30 @@ impl MlDb {
&self,
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
- self.db
- .write_batches_committing_each(
- UPSERT_FACE_CLUSTER,
- const { NonZeroUsize::new(500).unwrap() },
- face_id_to_cluster_id.iter(),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ UPSERT_FACE_CLUSTER,
+ const { NonZeroUsize::new(500).unwrap() },
+ face_id_to_cluster_id.iter(),
+ )
+ .map_err(Into::into)
}
pub fn cluster_id_to_face_count(&self) -> Result<HashMap<String, i64>> {
- self.db
- .read_all(
- r#"
+ self.read_all(
+ r#"
SELECT cluster_id, COUNT(*) as count
FROM face_clusters
where cluster_id IS NOT NULL
GROUP BY cluster_id
"#,
- (),
- pair,
- )
- .map_err(Into::into)
+ (),
+ pair,
+ )
+ .map_err(Into::into)
}
pub fn get_bad_face_singleton_cluster_ids(&self) -> Result<HashSet<String>> {
- let rows: Vec<(String, f64, f64, i64)> = self.db.read_all(
+ let rows: Vec<(String, f64, f64, i64)> = self.read_all(
r#"
SELECT fc.cluster_id, f.score, f.blur, f.is_sideways
FROM face_clusters fc
@@ -102,7 +100,7 @@ impl MlDb {
return Ok(HashMap::new());
}
let cluster_id_list: Vec<&String> = cluster_ids.iter().collect();
- let rows: Vec<(String, String)> = self.db.read_chunked_in(
+ let rows: Vec<(String, String)> = self.read_chunked_in(
"SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",
&cluster_id_list,
const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
@@ -112,27 +110,24 @@ impl MlDb {
}
pub fn get_cluster_id_for_face_id(&self, face_id: &str) -> Result<Option<String>> {
- self.db
- .read_optional(
- "SELECT cluster_id FROM face_clusters WHERE face_id = ?",
- [face_id],
- )
- .map_err(Into::into)
+ self.read_optional(
+ "SELECT cluster_id FROM face_clusters WHERE face_id = ?",
+ [face_id],
+ )
+ .map_err(Into::into)
}
pub fn get_all_cluster_id_to_face_ids(&self) -> Result<HashMap<String, Vec<String>>> {
- self.db
- .read_grouped("SELECT cluster_id, face_id FROM face_clusters", ())
+ self.read_grouped("SELECT cluster_id, face_id FROM face_clusters", ())
.map_err(Into::into)
}
pub fn get_face_ids_for_cluster(&self, cluster_id: &str) -> Result<Vec<String>> {
- self.db
- .read_column(
- "SELECT face_id FROM face_clusters WHERE face_clusters.cluster_id = ?",
- [cluster_id],
- )
- .map_err(Into::into)
+ self.read_column(
+ "SELECT face_id FROM face_clusters WHERE face_clusters.cluster_id = ?",
+ [cluster_id],
+ )
+ .map_err(Into::into)
}
pub fn get_face_ids_for_cluster_ordered_by_score(
@@ -140,9 +135,8 @@ impl MlDb {
cluster_id: &str,
limit: i64,
) -> Result<Vec<String>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT faces.face_id
FROM faces
JOIN face_clusters
@@ -151,13 +145,13 @@ impl MlDb {
ORDER BY faces.score DESC
LIMIT ?
"#,
- (cluster_id, limit),
- )
- .map_err(Into::into)
+ (cluster_id, limit),
+ )
+ .map_err(Into::into)
}
pub fn get_blur_values_for_cluster(&self, cluster_id: &str) -> Result<Vec<f64>> {
- let blur_values: Vec<f64> = self.db.read_column(
+ let blur_values: Vec<f64> = self.read_column(
r#"
SELECT faces.blur
FROM faces
@@ -178,19 +172,17 @@ impl MlDb {
&self,
face_ids: &[String],
) -> Result<HashMap<String, Option<String>>> {
- self.db
- .read_chunked_in(
- "SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",
- face_ids,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
- pair,
- )
- .map_err(Into::into)
+ self.read_chunked_in(
+ "SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",
+ face_ids,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
+ pair,
+ )
+ .map_err(Into::into)
}
pub fn get_file_id_to_cluster_ids(&self) -> Result<HashMap<i64, HashSet<String>>> {
- self.db
- .read_all("SELECT cluster_id, face_id FROM face_clusters", (), pair)
+ self.read_all("SELECT cluster_id, face_id FROM face_clusters", (), pair)
.map_err(Into::into)
.and_then(file_id_to_cluster_ids)
}
@@ -199,8 +191,7 @@ impl MlDb {
&self,
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
- self.db
- .write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())
+ self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())
.map_err(Into::into)
}
@@ -208,12 +199,11 @@ impl MlDb {
&self,
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
- self.db
- .write_batch_atomic(
- "DELETE FROM face_clusters WHERE face_id = ? AND cluster_id = ?",
- face_id_to_cluster_id.iter(),
- )
- .map_err(Into::into)
+ self.write_batch_atomic(
+ "DELETE FROM face_clusters WHERE face_id = ? AND cluster_id = ?",
+ face_id_to_cluster_id.iter(),
+ )
+ .map_err(Into::into)
}
pub fn get_file_id_to_cluster_id_set_for_cluster(
@@ -224,25 +214,23 @@ impl MlDb {
"SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",
bind_placeholders(cluster_ids.len())
);
- self.db
- .read_all(&sql, params_from_iter(cluster_ids), pair)
+ self.read_all(&sql, params_from_iter(cluster_ids), pair)
.map_err(Into::into)
.and_then(file_id_to_cluster_ids)
}
pub fn get_file_ids_of_cluster_id(&self, cluster_id: &str) -> Result<Vec<i64>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT DISTINCT faces.file_id
FROM face_clusters
JOIN faces
ON face_clusters.face_id = faces.face_id
WHERE face_clusters.cluster_id = ?
"#,
- [cluster_id],
- )
- .map_err(Into::into)
+ [cluster_id],
+ )
+ .map_err(Into::into)
}
pub fn get_cluster_centroid_vector_id_map(
@@ -255,27 +243,26 @@ impl MlDb {
return Ok(HashMap::new());
}
if create_if_missing {
- self.db.write_batch_atomic(
+ self.write_batch_atomic(
"INSERT OR IGNORE INTO cluster_centroid_vector_id_map (cluster_id) VALUES (?)",
unique_cluster_ids.iter().map(|cluster_id| [cluster_id]),
)?;
}
- self.db
- .read_chunked_in(
- r#"
+ self.read_chunked_in(
+ r#"
SELECT cluster_id, cluster_vector_id
FROM cluster_centroid_vector_id_map
WHERE cluster_id IN ({})
"#,
- &unique_cluster_ids,
- const { NonZeroUsize::new(800).unwrap() },
- pair,
- )
- .map_err(Into::into)
+ &unique_cluster_ids,
+ const { NonZeroUsize::new(800).unwrap() },
+ pair,
+ )
+ .map_err(Into::into)
}
pub fn delete_cluster_centroid_vector_id_mapping(&self, cluster_id: &str) -> Result<()> {
- self.db.execute(
+ self.execute(
"DELETE FROM cluster_centroid_vector_id_map WHERE cluster_id = ?",
[cluster_id],
)?;
@@ -283,8 +270,7 @@ impl MlDb {
}
pub fn clear_cluster_centroid_vector_id_mappings(&self) -> Result<()> {
- self.db
- .execute_statements([DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING])
+ self.execute_statements([DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING])
.map_err(Into::into)
}
@@ -292,19 +278,18 @@ impl MlDb {
&self,
summary: &HashMap<String, ClusterSummary>,
) -> Result<()> {
- self.db
- .write_batches_committing_each(
- UPSERT_CLUSTER_SUMMARY,
- const { NonZeroUsize::new(400).unwrap() },
- summary
- .iter()
- .map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ UPSERT_CLUSTER_SUMMARY,
+ const { NonZeroUsize::new(400).unwrap() },
+ summary
+ .iter()
+ .map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),
+ )
+ .map_err(Into::into)
}
pub fn delete_cluster_summary_row(&self, cluster_id: &str) -> Result<()> {
- self.db.execute(
+ self.execute(
"DELETE FROM cluster_summary WHERE cluster_id = ?",
[cluster_id],
)?;
@@ -323,13 +308,12 @@ impl MlDb {
""
}
);
- self.db
- .read_all(
- &sql,
- optional_parameter(&min_cluster_size).as_slice(),
- read_cluster_summary,
- )
- .map_err(Into::into)
+ self.read_all(
+ &sql,
+ optional_parameter(&min_cluster_size).as_slice(),
+ read_cluster_summary,
+ )
+ .map_err(Into::into)
}
pub fn get_cluster_to_cluster_summary(
@@ -340,14 +324,12 @@ impl MlDb {
"SELECT * FROM cluster_summary WHERE cluster_id IN ({})",
bind_placeholders(cluster_ids.len())
);
- self.db
- .read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary)
+ self.read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary)
.map_err(Into::into)
}
pub fn count_cluster_summaries(&self) -> Result<i64> {
- self.db
- .read_value("SELECT COUNT(cluster_id) as total FROM cluster_summary", ())
+ self.read_value("SELECT COUNT(cluster_id) as total FROM cluster_summary", ())
.map_err(Into::into)
}
@@ -357,12 +339,12 @@ impl MlDb {
limit: i64,
) -> Result<Vec<ClusterCentroidRow>> {
match before_cluster_id {
- None => self.db.read_all(
+ None => self.read_all(
"SELECT cluster_id, avg FROM cluster_summary ORDER BY cluster_id DESC LIMIT ?",
[limit],
read_cluster_centroid,
),
- Some(before_cluster_id) => self.db.read_all(
+ Some(before_cluster_id) => self.read_all(
r#"
SELECT cluster_id, avg
FROM cluster_summary
@@ -389,21 +371,20 @@ impl MlDb {
} else {
&[]
};
- self.db
- .execute_statements(face_statements.iter().copied().chain([
- DELETE_CLUSTER_PERSON,
- DELETE_NOT_PERSON_FEEDBACK,
- DELETE_CLUSTER_SUMMARY,
- DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
- DELETE_FACE_CLUSTERS,
- CREATE_CLUSTER_PERSON_TABLE,
- CREATE_NOT_PERSON_FEEDBACK_TABLE,
- CREATE_CLUSTER_SUMMARY_TABLE,
- CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE,
- CREATE_FACE_CLUSTERS_TABLE,
- FC_CLUSTER_ID_INDEX,
- ]))
- .map_err(Into::into)
+ self.execute_statements(face_statements.iter().copied().chain([
+ DELETE_CLUSTER_PERSON,
+ DELETE_NOT_PERSON_FEEDBACK,
+ DELETE_CLUSTER_SUMMARY,
+ DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ DELETE_FACE_CLUSTERS,
+ CREATE_CLUSTER_PERSON_TABLE,
+ CREATE_NOT_PERSON_FEEDBACK_TABLE,
+ CREATE_CLUSTER_SUMMARY_TABLE,
+ CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE,
+ CREATE_FACE_CLUSTERS_TABLE,
+ FC_CLUSTER_ID_INDEX,
+ ]))
+ .map_err(Into::into)
}
pub fn get_clusters_for_memory_lane(
@@ -414,7 +395,7 @@ impl MlDb {
let mut clusters = HashSet::new();
let mut offset: i64 = 0;
while clusters.len() < 20 {
- let batch: Vec<String> = self.db.read_column(
+ let batch: Vec<String> = self.read_column(
r#"
SELECT cluster_id, COUNT(*) AS count
FROM face_clusters
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index aeea11fc45..2d983a2ce1 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -1,10 +1,10 @@
use std::collections::{HashMap, HashSet};
use std::num::NonZeroUsize;
-use crate::db::{
- MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, ToSql, bind_placeholders, group_into,
- optional_parameter, pair, params_from_iter,
+use super::helpers::{
+ MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders, group_into, optional_parameter, pair,
};
+use rusqlite::{Result as SqliteResult, Row, ToSql, params_from_iter};
use super::clip::CLIP_ML_VERSION;
use super::pets::PET_ML_VERSION;
@@ -75,45 +75,42 @@ const UPSERT_FACE: &str = r#"
impl MlDb {
pub fn bulk_insert_faces(&self, faces: &[FaceRow]) -> Result<()> {
- self.db
- .write_batches_committing_each(
- UPSERT_FACE,
- const { NonZeroUsize::new(500).unwrap() },
- faces.iter().map(|face| {
- (
- face.file_id,
- &face.face_id,
- &face.detection_json,
- encode_evector(&face.embedding),
- face.score,
- face.blur,
- face.is_sideways,
- face.image_height,
- face.image_width,
- face.ml_version,
- )
- }),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ UPSERT_FACE,
+ const { NonZeroUsize::new(500).unwrap() },
+ faces.iter().map(|face| {
+ (
+ face.file_id,
+ &face.face_id,
+ &face.detection_json,
+ encode_evector(&face.embedding),
+ face.score,
+ face.blur,
+ face.is_sideways,
+ face.image_height,
+ face.image_width,
+ face.ml_version,
+ )
+ }),
+ )
+ .map_err(Into::into)
}
pub fn face_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {
- self.db
- .read_all(
- "SELECT file_id, ml_version FROM faces WHERE ml_version >= ?",
- [minimum_ml_version],
- pair,
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT file_id, ml_version FROM faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ pair,
+ )
+ .map_err(Into::into)
}
pub fn get_face_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
- self.db
- .read_value(
- "SELECT COUNT(DISTINCT file_id) as count FROM faces WHERE ml_version >= ?",
- [minimum_ml_version],
- )
- .map_err(Into::into)
+ self.read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ )
+ .map_err(Into::into)
}
pub fn get_face_embeddings_for_cluster(
@@ -135,8 +132,7 @@ impl MlDb {
);
let mut parameters: Vec<&dyn ToSql> = vec![&cluster_id];
parameters.extend(optional_parameter(&limit));
- self.db
- .read_column(&sql, parameters.as_slice())
+ self.read_column(&sql, parameters.as_slice())
.map_err(Into::into)
}
@@ -174,8 +170,7 @@ impl MlDb {
let mut parameters: Vec<&dyn ToSql> =
cluster_chunk.iter().map(|id| id as &dyn ToSql).collect();
parameters.extend(optional_parameter(&remaining_limit));
- let rows: Vec<(String, Vec<u8>)> =
- self.db.read_all(&sql, parameters.as_slice(), pair)?;
+ let rows: Vec<(String, Vec<u8>)> = self.read_all(&sql, parameters.as_slice(), pair)?;
if let Some(remaining) = remaining_limit.as_mut() {
*remaining -= rows.len() as i64;
}
@@ -199,7 +194,7 @@ impl MlDb {
if let Some(avatar_file_id) = avatar_file_id {
file_ids.push(avatar_file_id);
}
- let cluster_ids: Vec<String> = self.db.read_column(
+ let cluster_ids: Vec<String> = self.read_column(
"SELECT cluster_id FROM cluster_person WHERE person_id = ?",
[person_id],
)?;
@@ -222,8 +217,7 @@ impl MlDb {
cluster_ids.iter().map(|id| id as &dyn ToSql).collect();
parameters.extend(file_ids.iter().map(|id| id as &dyn ToSql));
let faces: Vec<StoredFace> =
- self.db
- .read_all(&sql, parameters.as_slice(), read_stored_face)?;
+ self.read_all(&sql, parameters.as_slice(), read_stored_face)?;
if !faces.is_empty() {
if let Some(avatar_file_id) = avatar_file_id
&& let Some(face) = faces.iter().find(|face| face.row.file_id == avatar_file_id)
@@ -234,7 +228,7 @@ impl MlDb {
}
}
if let Some(cluster_id) = cluster_id {
- let face_ids: Vec<String> = self.db.read_column(
+ let face_ids: Vec<String> = self.read_column(
"SELECT face_id FROM face_clusters WHERE cluster_id = ?",
[cluster_id],
)?;
@@ -253,7 +247,7 @@ impl MlDb {
}
pub fn get_faces_for_given_file_id(&self, file_upload_id: i64) -> Result<Vec<FaceRow>> {
- let faces: Vec<StoredFace> = self.db.read_all(
+ let faces: Vec<StoredFace> = self.read_all(
"SELECT * FROM faces WHERE file_id = ?",
[file_upload_id],
read_stored_face,
@@ -264,7 +258,7 @@ impl MlDb {
pub fn get_file_ids_to_faces_without_embedding(
&self,
) -> Result<HashMap<i64, Vec<FaceWithoutEmbedding>>> {
- let faces: Vec<FaceWithoutEmbedding> = self.db.read_all(
+ let faces: Vec<FaceWithoutEmbedding> = self.read_all(
"SELECT face_id, file_id, score, detection, blur FROM faces",
(),
|row| {
@@ -291,7 +285,7 @@ impl MlDb {
let mut offset = offset;
let mut result = Vec::new();
loop {
- let rows: Vec<(String, Vec<u8>, f64, f64, i64)> = self.db.read_all(
+ let rows: Vec<(String, Vec<u8>, f64, f64, i64)> = self.read_all(
r#"
SELECT face_id, embedding, score, blur, is_sideways
FROM faces
@@ -367,31 +361,27 @@ impl MlDb {
);
let mut parameters: Vec<&dyn ToSql> = chunk.iter().map(|id| id as &dyn ToSql).collect();
parameters.push(&remaining_limit);
- let rows: Vec<(String, Vec<u8>)> =
- self.db.read_all(&sql, parameters.as_slice(), pair)?;
+ let rows: Vec<(String, Vec<u8>)> = self.read_all(&sql, parameters.as_slice(), pair)?;
result.extend(rows);
}
Ok(result)
}
pub fn get_total_face_count(&self) -> Result<i64> {
- self.db
- .read_value(
- "SELECT COUNT(*) as count FROM faces WHERE score > ? AND blur > ?",
- (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
- )
- .map_err(Into::into)
+ self.read_value(
+ "SELECT COUNT(*) as count FROM faces WHERE score > ? AND blur > ?",
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ )
+ .map_err(Into::into)
}
pub fn get_errored_face_count(&self) -> Result<i64> {
- self.db
- .read_value("SELECT COUNT(*) as count FROM faces WHERE score < 0", ())
+ self.read_value("SELECT COUNT(*) as count FROM faces WHERE score < 0", ())
.map_err(Into::into)
}
pub fn get_errored_file_ids(&self) -> Result<HashSet<i64>> {
- self.db
- .read_column("SELECT DISTINCT file_id FROM faces WHERE score < 0", ())
+ self.read_column("SELECT DISTINCT file_id FROM faces WHERE score < 0", ())
.map_err(Into::into)
}
@@ -399,9 +389,8 @@ impl MlDb {
if file_ids.is_empty() {
return Ok(());
}
- self.db
- .execute_chunked_in(
- r#"
+ self.execute_chunked_in(
+ r#"
DELETE FROM faces
WHERE file_id IN ({})
AND score < 0
@@ -413,9 +402,9 @@ impl MlDb {
AND successful.ml_version >= faces.ml_version
)
"#,
- file_ids,
- )
- .map_err(Into::into)
+ file_ids,
+ )
+ .map_err(Into::into)
}
pub fn get_file_ids_with_error_results(&self, file_ids: &[i64]) -> Result<HashSet<i64>> {
@@ -450,7 +439,7 @@ impl MlDb {
AND score < 0
"#
);
- let matches: Vec<i64> = self.db.read_column(
+ let matches: Vec<i64> = self.read_column(
&sql,
params_from_iter(chunk.iter().chain(chunk).chain(chunk)),
)?;
@@ -460,8 +449,7 @@ impl MlDb {
}
pub fn delete_face_index_for_files(&self, file_ids: &[i64]) -> Result<()> {
- self.db
- .execute_chunked_in("DELETE FROM faces WHERE file_id IN ({})", file_ids)
+ self.execute_chunked_in("DELETE FROM faces WHERE file_id IN ({})", file_ids)
.map_err(Into::into)
}
@@ -469,9 +457,8 @@ impl MlDb {
if file_ids.is_empty() {
return Ok(());
}
- self.db
- .execute_chunked_in(
- r#"
+ self.execute_chunked_in(
+ r#"
DELETE FROM faces
WHERE file_id IN ({})
AND NOT EXISTS (
@@ -480,25 +467,24 @@ impl MlDb {
WHERE face_clusters.face_id = faces.face_id
)
"#,
- file_ids,
- )
- .map_err(Into::into)
+ file_ids,
+ )
+ .map_err(Into::into)
}
pub fn get_clustered_or_faceless_file_count(&self) -> Result<i64> {
- let clustered_face_ids: Vec<String> = self
- .db
- .read_column("SELECT face_id FROM face_clusters", ())?;
+ let clustered_face_ids: Vec<String> =
+ self.read_column("SELECT face_id FROM face_clusters", ())?;
let clustered_file_ids = clustered_face_ids
.iter()
.map(|face_id| file_id_from_face_id(face_id))
.collect::<Result<HashSet<i64>>>()?;
- let bad_file_ids: HashSet<i64> = self.db.read_column(
+ let bad_file_ids: HashSet<i64> = self.read_column(
"SELECT DISTINCT file_id FROM faces WHERE score <= ? OR blur <= ?",
(MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
)?;
- let good_file_ids: HashSet<i64> = self.db.read_column(
+ let good_file_ids: HashSet<i64> = self.read_column(
"SELECT DISTINCT file_id FROM faces WHERE score > ? AND blur > ?",
(MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
)?;
@@ -507,9 +493,8 @@ impl MlDb {
}
pub fn get_unclustered_face_count(&self) -> Result<i64> {
- self.db
- .read_value(
- r#"
+ self.read_value(
+ r#"
SELECT COUNT(*) as count
FROM faces f
LEFT JOIN face_clusters fc
@@ -518,24 +503,23 @@ impl MlDb {
AND f.blur > ?
AND fc.face_id IS NULL
"#,
- (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
- )
- .map_err(Into::into)
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ )
+ .map_err(Into::into)
}
pub fn get_all_file_ids_of_face_ids_not_in_any_cluster(&self) -> Result<HashSet<i64>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT DISTINCT file_id
FROM faces
LEFT JOIN face_clusters
ON faces.face_id = face_clusters.face_id
WHERE face_clusters.face_id IS NULL
"#,
- (),
- )
- .map_err(Into::into)
+ (),
+ )
+ .map_err(Into::into)
}
pub fn get_all_files_associated_with_all_clusters(
@@ -553,8 +537,7 @@ impl MlDb {
"#,
bind_placeholders(except_clusters.len())
);
- self.db
- .read_column(&sql, params_from_iter(except_clusters))
+ self.read_column(&sql, params_from_iter(except_clusters))
.map_err(Into::into)
}
@@ -575,8 +558,7 @@ impl MlDb {
sql.push_str(" INTERSECT SELECT file_id FROM pet_faces WHERE ml_version >= ?");
parameters.push(PET_ML_VERSION);
}
- self.db
- .read_column(&sql, params_from_iter(parameters))
+ self.read_column(&sql, params_from_iter(parameters))
.map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/filedata.rs b/rust/crates/photos/src/ml_db/queries/filedata.rs
index 856a226314..31b1ad4ab8 100644
--- a/rust/crates/photos/src/ml_db/queries/filedata.rs
+++ b/rust/crates/photos/src/ml_db/queries/filedata.rs
@@ -1,6 +1,6 @@
use std::collections::{HashMap, HashSet};
-use crate::db::optional_parameter;
+use super::helpers::optional_parameter;
use crate::ml_db::{MlDb, Result};
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -25,45 +25,43 @@ impl MlDb {
if fd_status_list.is_empty() {
return Ok(());
}
- self.db
- .write_batch_atomic(
- r#"
+ self.write_batch_atomic(
+ r#"
INSERT OR REPLACE INTO filedata (
file_id, user_id, type, size, obj_id, obj_nonce, updated_at
)
values (?, ?, ?, ?, ?, ?, ?)
"#,
- fd_status_list.iter().map(|status| {
- (
- status.file_id,
- status.user_id,
- &status.data_type,
- status.size,
- &status.object_id,
- &status.object_nonce,
- status.updated_at,
- )
- }),
- )
- .map_err(Into::into)
+ fd_status_list.iter().map(|status| {
+ (
+ status.file_id,
+ status.user_id,
+ &status.data_type,
+ status.size,
+ &status.object_id,
+ &status.object_nonce,
+ status.updated_at,
+ )
+ }),
+ )
+ .map_err(Into::into)
}
pub fn get_file_ids_vid_preview(&self) -> Result<HashMap<i64, PreviewInfo>> {
- self.db
- .read_all(
- "SELECT file_id, obj_id, size FROM filedata WHERE type='vid_preview'",
- (),
- |row| {
- Ok((
- row.get(0)?,
- PreviewInfo {
- object_id: row.get(1)?,
- object_size: row.get(2)?,
- },
- ))
- },
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT file_id, obj_id, size FROM filedata WHERE type='vid_preview'",
+ (),
+ |row| {
+ Ok((
+ row.get(0)?,
+ PreviewInfo {
+ object_id: row.get(1)?,
+ object_size: row.get(2)?,
+ },
+ ))
+ },
+ )
+ .map_err(Into::into)
}
pub fn get_file_ids_with_fd_data(&self, data_type: Option<&str>) -> Result<HashSet<i64>> {
@@ -71,8 +69,7 @@ impl MlDb {
None => "SELECT file_id FROM filedata",
Some(_) => "SELECT file_id FROM filedata WHERE type = ?",
};
- self.db
- .read_column(sql, optional_parameter(&data_type).as_slice())
+ self.read_column(sql, optional_parameter(&data_type).as_slice())
.map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rs
new file mode 100644
index 0000000000..486bd8d577
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs
@@ -0,0 +1,518 @@
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::num::NonZeroUsize;
+
+use rusqlite::{
+ OptionalExtension, Params, Result, Row, ToSql, TransactionBehavior, params_from_iter,
+ types::FromSql,
+};
+
+use crate::ml_db::MlDb;
+
+pub(super) const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;
+
+impl MlDb {
+ pub(in crate::ml_db) fn read_all<C: FromIterator<T>, T, P: Params>(
+ &self,
+ sql: &str,
+ parameters: P,
+ map: impl FnMut(&Row<'_>) -> Result<T>,
+ ) -> Result<C> {
+ self.db.read(|connection| {
+ let mut statement = connection.prepare_cached(sql)?;
+ let rows = statement.query_map(parameters, map)?;
+ rows.collect::<Result<C>>()
+ })
+ }
+
+ pub(in crate::ml_db) fn read_column<C: FromIterator<T>, T: FromSql, P: Params>(
+ &self,
+ sql: &str,
+ parameters: P,
+ ) -> Result<C> {
+ self.read_all(sql, parameters, |row| row.get(0))
+ }
+
+ pub(in crate::ml_db) fn read_grouped<
+ K: FromSql + Eq + Hash,
+ V: FromSql,
+ C: FromIterator<V>,
+ P: Params,
+ >(
+ &self,
+ sql: &str,
+ parameters: P,
+ ) -> Result<HashMap<K, C>> {
+ let rows: Vec<(K, V)> = self.read_all(sql, parameters, pair)?;
+ Ok(group_into(rows))
+ }
+
+ pub(in crate::ml_db) fn read_value<T: FromSql>(
+ &self,
+ sql: &str,
+ parameters: impl Params,
+ ) -> Result<T> {
+ self.db.read(|connection| {
+ connection
+ .prepare_cached(sql)?
+ .query_row(parameters, |row| row.get(0))
+ })
+ }
+
+ pub(in crate::ml_db) fn read_optional<T: FromSql>(
+ &self,
+ sql: &str,
+ parameters: impl Params,
+ ) -> Result<Option<T>> {
+ self.db.read(|connection| {
+ connection
+ .prepare_cached(sql)?
+ .query_row(parameters, |row| row.get(0))
+ .optional()
+ })
+ }
+
+ pub(in crate::ml_db) fn read_chunked_in<C: FromIterator<T>, T, I: ToSql>(
+ &self,
+ sql: &str,
+ ids: &[I],
+ chunk_size: NonZeroUsize,
+ mut map: impl FnMut(&Row<'_>) -> Result<T>,
+ ) -> Result<C> {
+ let mut rows = Vec::new();
+ for chunk in ids.chunks(chunk_size.get()) {
+ let sql = expand_in_clause(sql, chunk.len());
+ let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;
+ rows.extend(chunk_rows);
+ }
+ Ok(rows.into_iter().collect())
+ }
+
+ pub(in crate::ml_db) fn execute(&self, sql: &str, parameters: impl Params) -> Result<usize> {
+ self.db
+ .write(|connection| connection.prepare_cached(sql)?.execute(parameters))
+ }
+
+ pub(in crate::ml_db) fn execute_chunked_in<I: ToSql>(
+ &self,
+ sql: &str,
+ ids: &[I],
+ ) -> Result<()> {
+ if ids.is_empty() {
+ return Ok(());
+ }
+ self.db.write_transaction(|transaction| {
+ for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
+ let sql = expand_in_clause(sql, chunk.len());
+ transaction.execute(&sql, params_from_iter(chunk))?;
+ }
+ Ok(())
+ })
+ }
+
+ pub(in crate::ml_db) fn execute_statements<'a>(
+ &self,
+ statements: impl IntoIterator<Item = &'a str>,
+ ) -> Result<()> {
+ self.db.write_transaction(|transaction| {
+ for statement in statements {
+ transaction.execute_batch(statement)?;
+ }
+ Ok(())
+ })
+ }
+
+ pub(in crate::ml_db) fn write_batch_atomic<P: Params>(
+ &self,
+ sql: &str,
+ parameter_sets: impl IntoIterator<Item = P>,
+ ) -> Result<()> {
+ self.db.write_transaction(|transaction| {
+ let mut statement = transaction.prepare_cached(sql)?;
+ for parameters in parameter_sets {
+ statement.execute(parameters)?;
+ }
+ Ok(())
+ })
+ }
+
+ pub(in crate::ml_db) fn write_batches_committing_each<P: Params>(
+ &self,
+ sql: &str,
+ batch_size: NonZeroUsize,
+ parameter_sets: impl IntoIterator<Item = P>,
+ ) -> Result<()> {
+ self.db.write(|connection| {
+ let mut parameter_sets = parameter_sets.into_iter().peekable();
+ while parameter_sets.peek().is_some() {
+ let transaction =
+ connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ {
+ let mut statement = transaction.prepare_cached(sql)?;
+ for parameters in parameter_sets.by_ref().take(batch_size.get()) {
+ statement.execute(parameters)?;
+ }
+ }
+ transaction.commit()?;
+ }
+ Ok(())
+ })
+ }
+}
+
+pub(super) fn bind_placeholders(count: usize) -> String {
+ vec!["?"; count].join(", ")
+}
+
+fn expand_in_clause(sql: &str, count: usize) -> String {
+ sql.replacen("{}", &bind_placeholders(count), 1)
+}
+
+pub(super) fn pair<A: FromSql, B: FromSql>(row: &Row<'_>) -> Result<(A, B)> {
+ Ok((row.get(0)?, row.get(1)?))
+}
+
+pub(super) fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql> {
+ value.iter().map(|value| value as &dyn ToSql).collect()
+}
+
+pub(super) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(
+ rows: impl IntoIterator<Item = (K, V)>,
+) -> HashMap<K, C> {
+ let mut groups: HashMap<K, Vec<V>> = HashMap::new();
+ for (key, value) in rows {
+ groups.entry(key).or_default().push(value);
+ }
+ groups
+ .into_iter()
+ .map(|(key, values)| (key, values.into_iter().collect()))
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::{HashMap, HashSet};
+ use std::num::NonZeroUsize;
+
+ use crate::db::{Database, OpenOptions};
+
+ use super::{MAX_SQL_BIND_PARAMS_PER_QUERY, bind_placeholders};
+ use crate::ml_db::MlDb;
+
+ fn open() -> (tempfile::TempDir, MlDb) {
+ let directory = tempfile::tempdir().unwrap();
+ let db = Database::open(
+ directory.path().join("items.db"),
+ OpenOptions::default(),
+ |connection| connection.execute_batch("CREATE TABLE items (id INTEGER PRIMARY KEY)"),
+ )
+ .unwrap();
+ (directory, MlDb { db })
+ }
+
+ #[test]
+ fn reader_queries_reject_writes() {
+ let (_directory, db) = open();
+ db.execute("INSERT INTO items (id) VALUES (?)", [7])
+ .unwrap();
+ let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());
+ assert!(deleted.is_err());
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())
+ .unwrap(),
+ [7]
+ );
+ }
+
+ #[test]
+ fn failed_batch_rolls_back_and_writer_recovers() {
+ let (_directory, db) = open();
+ assert!(
+ db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]])
+ .is_err()
+ );
+ assert_eq!(
+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ db.execute("INSERT INTO items (id) VALUES (?)", [3])
+ .unwrap(),
+ 1
+ );
+ assert_eq!(
+ db.execute("DELETE FROM items WHERE id = ?", [4]).unwrap(),
+ 0
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ())
+ .unwrap(),
+ [3]
+ );
+ }
+
+ #[test]
+ fn statement_sequence_rolls_back_if_a_later_statement_fails() {
+ let (_directory, db) = open();
+ assert!(
+ db.execute_statements([
+ "INSERT INTO items VALUES (1)",
+ "INSERT INTO missing_table VALUES (2)",
+ ])
+ .is_err()
+ );
+ assert_eq!(
+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap(),
+ 0
+ );
+ }
+
+ #[test]
+ fn chunked_delete_handles_empty_lists_and_rolls_back_on_failure() {
+ let (_directory, db) = open();
+ let ids: Vec<i64> = (0..10_001).collect();
+ db.write_batch_atomic(
+ "INSERT INTO items (id) VALUES (?)",
+ ids.iter().map(|id| [id]),
+ )
+ .unwrap();
+ db.execute_chunked_in::<i64>("DELETE FROM items WHERE id IN ({})", &[])
+ .unwrap();
+ assert_eq!(
+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap(),
+ 10_001
+ );
+ db.execute_statements(["CREATE TRIGGER reject_last_delete BEFORE DELETE ON items
+ WHEN OLD.id = 10000 BEGIN SELECT RAISE(ABORT, 'cannot delete'); END"])
+ .unwrap();
+ assert!(
+ db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)
+ .is_err()
+ );
+ assert_eq!(
+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap(),
+ 10_001
+ );
+ db.execute_statements(["DROP TRIGGER reject_last_delete"])
+ .unwrap();
+ db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)
+ .unwrap();
+ assert_eq!(
+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap(),
+ 0
+ );
+ }
+
+ #[test]
+ fn batched_writes_commit_each_completed_batch() {
+ let (_directory, db) = open();
+ db.write_batches_committing_each(
+ "INSERT INTO missing_table VALUES (?)",
+ const { NonZeroUsize::new(2).unwrap() },
+ Vec::<[i64; 1]>::new(),
+ )
+ .unwrap();
+ assert!(
+ db.write_batches_committing_each(
+ "INSERT INTO items (id) VALUES (?)",
+ const { NonZeroUsize::new(2).unwrap() },
+ [[1], [2], [3], [4], [1]]
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ [1, 2, 3, 4]
+ );
+ assert!(
+ db.write_batches_committing_each(
+ "INSERT INTO items (id) VALUES (?)",
+ const { NonZeroUsize::new(2).unwrap() },
+ [[5], [6], [7], [5], [8]]
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ [1, 2, 3, 4, 5, 6]
+ );
+ db.write_batches_committing_each(
+ "INSERT INTO items (id) VALUES (?)",
+ const { NonZeroUsize::new(2).unwrap() },
+ [[7], [8], [9]],
+ )
+ .unwrap();
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ [1, 2, 3, 4, 5, 6, 7, 8, 9]
+ );
+ assert!(
+ db.write_batches_committing_each(
+ "INSERT INTO items (id) VALUES (?)",
+ NonZeroUsize::MIN,
+ [[10], [11], [10]],
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ (1..=11).collect::<Vec<_>>()
+ );
+ }
+
+ #[test]
+ fn batched_writes_hold_the_writer_between_commits() {
+ use std::sync::mpsc::{self, RecvTimeoutError};
+ use std::thread;
+ use std::time::Duration;
+
+ let (_directory, db) = open();
+ thread::scope(|scope| {
+ let (between_tx, between_rx) = mpsc::channel();
+ let (release_tx, release_rx) = mpsc::channel();
+ let db = &db;
+ scope.spawn(move || {
+ db.write_batches_committing_each(
+ "INSERT INTO items VALUES (?)",
+ NonZeroUsize::MIN,
+ (1..=2).map(|id| {
+ if id == 2 {
+ between_tx.send(()).unwrap();
+ release_rx.recv().unwrap();
+ }
+ [id]
+ }),
+ )
+ .unwrap();
+ });
+ between_rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ let committed_count = db
+ .read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap();
+ let (started_tx, started_rx) = mpsc::channel();
+ let (finished_tx, finished_rx) = mpsc::channel();
+ scope.spawn(move || {
+ started_tx.send(()).unwrap();
+ finished_tx
+ .send(db.execute("INSERT INTO items VALUES (3)", ()))
+ .unwrap();
+ });
+ started_rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ let was_blocked = matches!(
+ finished_rx.recv_timeout(Duration::from_millis(100)),
+ Err(RecvTimeoutError::Timeout)
+ );
+ release_tx.send(()).unwrap();
+ let result = finished_rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ assert_eq!(committed_count, 1);
+ assert!(was_blocked);
+ assert_eq!(result.unwrap(), 1);
+ });
+ assert_eq!(
+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())
+ .unwrap(),
+ 3
+ );
+ }
+
+ #[test]
+ fn chunked_reads_query_each_chunk_in_order() {
+ let (_directory, db) = open();
+ db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))
+ .unwrap();
+ let ids: Vec<i64> = (1..=7).collect();
+ let one_at_a_time: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
+ &ids,
+ NonZeroUsize::MIN,
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(one_at_a_time, ids);
+ let by_chunk: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
+ &ids,
+ const { NonZeroUsize::new(3).unwrap() },
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);
+ let single_chunk: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
+ &ids,
+ const { NonZeroUsize::new(10).unwrap() },
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);
+ let none: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM missing_table WHERE id IN ({})",
+ &Vec::<i64>::new(),
+ const { NonZeroUsize::new(3).unwrap() },
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert!(none.is_empty());
+ let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();
+ let found: HashSet<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({})",
+ &many,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));
+ }
+
+ #[test]
+ fn grouped_reads_collect_values_per_key() {
+ let (_directory, db) = open();
+ db.write_batch_atomic("INSERT INTO items (id) VALUES (?)", (1..=5).map(|id| [id]))
+ .unwrap();
+ let by_parity: HashMap<i64, Vec<i64>> = db
+ .read_grouped("SELECT id % 2, id FROM items ORDER BY id", ())
+ .unwrap();
+ assert_eq!(
+ by_parity,
+ HashMap::from([(1, vec![1, 3, 5]), (0, vec![2, 4])])
+ );
+ let by_label: HashMap<String, HashSet<i64>> = db
+ .read_grouped(
+ "SELECT CASE WHEN id < 3 THEN 'low' ELSE 'high' END, id FROM items",
+ (),
+ )
+ .unwrap();
+ assert_eq!(
+ by_label,
+ HashMap::from([
+ ("low".to_string(), HashSet::from([1, 2])),
+ ("high".to_string(), HashSet::from([3, 4, 5]))
+ ])
+ );
+ let empty: HashMap<i64, Vec<i64>> = db
+ .read_grouped("SELECT id, id FROM items WHERE id > 5", ())
+ .unwrap();
+ assert!(empty.is_empty());
+ }
+
+ #[test]
+ fn bind_placeholders_joins_question_marks() {
+ assert_eq!(bind_placeholders(0), "");
+ assert_eq!(bind_placeholders(1), "?");
+ assert_eq!(bind_placeholders(3), "?, ?, ?");
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/queries/mod.rs b/rust/crates/photos/src/ml_db/queries/mod.rs
index a22b335eee..f49f7410a5 100644
--- a/rust/crates/photos/src/ml_db/queries/mod.rs
+++ b/rust/crates/photos/src/ml_db/queries/mod.rs
@@ -3,6 +3,7 @@ pub(super) mod clip;
pub(super) mod clusters;
pub(super) mod faces;
pub(super) mod filedata;
+mod helpers;
pub(super) mod persons;
pub(super) mod pets;
diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rs
index 6c43340720..b8768a95e1 100644
--- a/rust/crates/photos/src/ml_db/queries/persons.rs
+++ b/rust/crates/photos/src/ml_db/queries/persons.rs
@@ -1,7 +1,7 @@
use std::collections::{HashMap, HashSet};
use std::num::NonZeroUsize;
-use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};
+use super::helpers::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};
use super::clusters::file_id_to_cluster_ids;
use crate::ml_db::schema::{
@@ -25,21 +25,20 @@ const INSERT_NOT_PERSON_FEEDBACK: &str = r#"
impl MlDb {
pub fn get_clusters_with_three_or_more_not_person_feedback(&self) -> Result<HashSet<String>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT cluster_id
FROM not_person_feedback
GROUP BY cluster_id
HAVING COUNT(*) >= 3
"#,
- (),
- )
- .map_err(Into::into)
+ (),
+ )
+ .map_err(Into::into)
}
pub fn get_person_ignored_clusters(&self, person_id: &str) -> Result<HashSet<String>> {
- let mut ignored_cluster_ids: HashSet<String> = self.db.read_column(
+ let mut ignored_cluster_ids: HashSet<String> = self.read_column(
r#"
SELECT cluster_id
FROM cluster_person
@@ -48,7 +47,7 @@ impl MlDb {
"#,
[person_id],
)?;
- let reject_cluster_ids: Vec<String> = self.db.read_column(
+ let reject_cluster_ids: Vec<String> = self.read_column(
"SELECT cluster_id FROM not_person_feedback WHERE person_id = ?",
[person_id],
)?;
@@ -57,33 +56,30 @@ impl MlDb {
}
pub fn get_person_to_rejected_suggestions(&self) -> Result<HashMap<String, HashSet<String>>> {
- self.db
- .read_grouped("SELECT person_id, cluster_id FROM not_person_feedback", ())
+ self.read_grouped("SELECT person_id, cluster_id FROM not_person_feedback", ())
.map_err(Into::into)
}
pub fn get_person_cluster_ids(&self, person_id: &str) -> Result<HashSet<String>> {
- self.db
- .read_column(
- "SELECT cluster_id FROM cluster_person WHERE person_id = ?",
- [person_id],
- )
- .map_err(Into::into)
+ self.read_column(
+ "SELECT cluster_id FROM cluster_person WHERE person_id = ?",
+ [person_id],
+ )
+ .map_err(Into::into)
}
pub fn get_persons_cluster_ids(&self, person_ids: &[String]) -> Result<HashSet<String>> {
- self.db
- .read_chunked_in(
- "SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",
- person_ids,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
- |row| row.get(0),
- )
- .map_err(Into::into)
+ self.read_chunked_in(
+ "SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",
+ person_ids,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
+ |row| row.get(0),
+ )
+ .map_err(Into::into)
}
pub fn get_person_to_cluster_id_to_face_ids(&self) -> Result<PersonToClusterIdToFaceIds> {
- let rows: Vec<(String, String, String)> = self.db.read_all(
+ let rows: Vec<(String, String, String)> = self.read_all(
r#"
SELECT person_id, face_clusters.cluster_id, face_id
FROM cluster_person
@@ -106,8 +102,7 @@ impl MlDb {
}
pub fn get_person_to_cluster_ids(&self) -> Result<HashMap<String, HashSet<String>>> {
- self.db
- .read_grouped("SELECT person_id, cluster_id FROM cluster_person", ())
+ self.read_grouped("SELECT person_id, cluster_id FROM cluster_person", ())
.map_err(Into::into)
}
@@ -115,53 +110,50 @@ impl MlDb {
&self,
face_ids: &[String],
) -> Result<HashMap<String, String>> {
- self.db
- .read_chunked_in(
- r#"
+ self.read_chunked_in(
+ r#"
SELECT face_id, person_id
FROM cluster_person
INNER JOIN face_clusters
ON cluster_person.cluster_id = face_clusters.cluster_id
WHERE face_id IN ({})
"#,
- face_ids,
- const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
- pair,
- )
- .map_err(Into::into)
+ face_ids,
+ const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
+ pair,
+ )
+ .map_err(Into::into)
}
pub fn get_cluster_id_to_face_ids_for_person(
&self,
person_id: &str,
) -> Result<HashMap<String, HashSet<String>>> {
- self.db
- .read_grouped(
- r#"
+ self.read_grouped(
+ r#"
SELECT face_clusters.cluster_id, face_id
FROM cluster_person
INNER JOIN face_clusters
ON cluster_person.cluster_id = face_clusters.cluster_id
WHERE person_id = ?
"#,
- [person_id],
- )
- .map_err(Into::into)
+ [person_id],
+ )
+ .map_err(Into::into)
}
pub fn get_face_ids_for_person(&self, person_id: &str) -> Result<HashSet<String>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT face_id
FROM face_clusters
LEFT JOIN cluster_person
ON face_clusters.cluster_id = cluster_person.cluster_id
WHERE cluster_person.person_id = ?
"#,
- [person_id],
- )
- .map_err(Into::into)
+ [person_id],
+ )
+ .map_err(Into::into)
}
pub fn get_face_ids_for_person_ordered_by_score(
@@ -169,9 +161,8 @@ impl MlDb {
person_id: &str,
limit: i64,
) -> Result<Vec<String>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT faces.face_id
FROM faces
JOIN face_clusters
@@ -182,9 +173,9 @@ impl MlDb {
ORDER BY faces.score DESC
LIMIT ?
"#,
- (person_id, limit),
- )
- .map_err(Into::into)
+ (person_id, limit),
+ )
+ .map_err(Into::into)
}
pub fn remove_person(&self, person_id: &str) -> Result<()> {
@@ -204,8 +195,7 @@ impl MlDb {
}
pub fn assign_cluster_to_person(&self, person_id: &str, cluster_id: &str) -> Result<()> {
- self.db
- .execute(INSERT_CLUSTER_PERSON, [person_id, cluster_id])?;
+ self.execute(INSERT_CLUSTER_PERSON, [person_id, cluster_id])?;
Ok(())
}
@@ -213,19 +203,17 @@ impl MlDb {
&self,
cluster_to_person_id: &HashMap<String, String>,
) -> Result<()> {
- self.db
- .write_batch_atomic(
- INSERT_CLUSTER_PERSON,
- cluster_to_person_id
- .iter()
- .map(|(cluster_id, person_id)| (person_id, cluster_id)),
- )
- .map_err(Into::into)
+ self.write_batch_atomic(
+ INSERT_CLUSTER_PERSON,
+ cluster_to_person_id
+ .iter()
+ .map(|(cluster_id, person_id)| (person_id, cluster_id)),
+ )
+ .map_err(Into::into)
}
pub fn capture_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {
- self.db
- .execute(INSERT_NOT_PERSON_FEEDBACK, [person_id, cluster_id])?;
+ self.execute(INSERT_NOT_PERSON_FEEDBACK, [person_id, cluster_id])?;
Ok(())
}
@@ -233,18 +221,17 @@ impl MlDb {
&self,
cluster_to_person_id: &HashMap<String, String>,
) -> Result<()> {
- self.db
- .write_batch_atomic(
- INSERT_NOT_PERSON_FEEDBACK,
- cluster_to_person_id
- .iter()
- .map(|(cluster_id, person_id)| (person_id, cluster_id)),
- )
- .map_err(Into::into)
+ self.write_batch_atomic(
+ INSERT_NOT_PERSON_FEEDBACK,
+ cluster_to_person_id
+ .iter()
+ .map(|(cluster_id, person_id)| (person_id, cluster_id)),
+ )
+ .map_err(Into::into)
}
pub fn remove_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {
- self.db.execute(
+ self.execute(
"DELETE FROM not_person_feedback WHERE person_id = ? AND cluster_id = ?",
[person_id, cluster_id],
)?;
@@ -252,7 +239,7 @@ impl MlDb {
}
pub fn remove_cluster_to_person(&self, person_id: &str, cluster_id: &str) -> Result<()> {
- self.db.execute(
+ self.execute(
"DELETE FROM cluster_person WHERE person_id = ? AND cluster_id = ?",
[person_id, cluster_id],
)?;
@@ -263,47 +250,43 @@ impl MlDb {
&self,
person_id: &str,
) -> Result<HashMap<i64, HashSet<String>>> {
- self.db
- .read_all(
- r#"
+ self.read_all(
+ r#"
SELECT face_clusters.cluster_id, face_id
FROM face_clusters
INNER JOIN cluster_person
ON face_clusters.cluster_id = cluster_person.cluster_id
WHERE cluster_person.person_id = ?
"#,
- [person_id],
- pair,
- )
- .map_err(Into::into)
- .and_then(file_id_to_cluster_ids)
+ [person_id],
+ pair,
+ )
+ .map_err(Into::into)
+ .and_then(file_id_to_cluster_ids)
}
pub fn get_cluster_id_to_person_id(&self) -> Result<HashMap<String, String>> {
- self.db
- .read_all(
- "SELECT person_id, cluster_id FROM cluster_person",
- (),
- |row| Ok((row.get(1)?, row.get(0)?)),
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT person_id, cluster_id FROM cluster_person",
+ (),
+ |row| Ok((row.get(1)?, row.get(0)?)),
+ )
+ .map_err(Into::into)
}
pub fn drop_faces_feedback_tables(&self) -> Result<()> {
- self.db
- .execute_statements([
- DELETE_CLUSTER_PERSON,
- DELETE_NOT_PERSON_FEEDBACK,
- CREATE_CLUSTER_PERSON_TABLE,
- CREATE_NOT_PERSON_FEEDBACK_TABLE,
- ])
- .map_err(Into::into)
+ self.execute_statements([
+ DELETE_CLUSTER_PERSON,
+ DELETE_NOT_PERSON_FEEDBACK,
+ CREATE_CLUSTER_PERSON_TABLE,
+ CREATE_NOT_PERSON_FEEDBACK_TABLE,
+ ])
+ .map_err(Into::into)
}
pub fn get_file_ids_of_person_id(&self, person_id: &str) -> Result<Vec<i64>> {
- self.db
- .read_column(
- r#"
+ self.read_column(
+ r#"
SELECT DISTINCT faces.file_id
FROM cluster_person
JOIN face_clusters
@@ -312,9 +295,9 @@ impl MlDb {
ON face_clusters.face_id = faces.face_id
WHERE cluster_person.person_id = ?
"#,
- [person_id],
- )
- .map_err(Into::into)
+ [person_id],
+ )
+ .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rs
index 5136fa9d95..c30b355afb 100644
--- a/rust/crates/photos/src/ml_db/queries/pets.rs
+++ b/rust/crates/photos/src/ml_db/queries/pets.rs
@@ -1,7 +1,8 @@
use std::collections::HashMap;
use std::num::NonZeroUsize;
-use crate::db::{Row, SqliteResult, bind_placeholders, pair, params_from_iter};
+use super::helpers::{bind_placeholders, pair};
+use rusqlite::{Result as SqliteResult, Row, params_from_iter};
use super::unique_in_order;
use crate::ml_db::{MlDb, Result};
@@ -87,47 +88,45 @@ const UPSERT_PET_BODY: &str = r#"
impl MlDb {
pub fn bulk_insert_pet_faces(&self, pet_faces: &[PetFaceRow]) -> Result<()> {
- self.db
- .write_batches_committing_each(
- UPSERT_PET_FACE,
- const { NonZeroUsize::new(500).unwrap() },
- pet_faces.iter().map(|pet_face| {
- (
- pet_face.file_id,
- &pet_face.pet_face_id,
- &pet_face.detection_json,
- pet_face.face_vector_id,
- pet_face.species,
- pet_face.face_score,
- pet_face.image_height,
- pet_face.image_width,
- pet_face.ml_version,
- )
- }),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ UPSERT_PET_FACE,
+ const { NonZeroUsize::new(500).unwrap() },
+ pet_faces.iter().map(|pet_face| {
+ (
+ pet_face.file_id,
+ &pet_face.pet_face_id,
+ &pet_face.detection_json,
+ pet_face.face_vector_id,
+ pet_face.species,
+ pet_face.face_score,
+ pet_face.image_height,
+ pet_face.image_width,
+ pet_face.ml_version,
+ )
+ }),
+ )
+ .map_err(Into::into)
}
pub fn bulk_insert_pet_bodies(&self, pet_bodies: &[PetBodyRow]) -> Result<()> {
- self.db
- .write_batches_committing_each(
- UPSERT_PET_BODY,
- const { NonZeroUsize::new(500).unwrap() },
- pet_bodies.iter().map(|pet_body| {
- (
- pet_body.file_id,
- &pet_body.pet_body_id,
- &pet_body.detection_json,
- pet_body.body_vector_id,
- pet_body.species,
- pet_body.score,
- pet_body.image_height,
- pet_body.image_width,
- pet_body.ml_version,
- )
- }),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ UPSERT_PET_BODY,
+ const { NonZeroUsize::new(500).unwrap() },
+ pet_bodies.iter().map(|pet_body| {
+ (
+ pet_body.file_id,
+ &pet_body.pet_body_id,
+ &pet_body.detection_json,
+ pet_body.body_vector_id,
+ pet_body.species,
+ pet_body.score,
+ pet_body.image_height,
+ pet_body.image_width,
+ pet_body.ml_version,
+ )
+ }),
+ )
+ .map_err(Into::into)
}
pub fn update_pet_face_vector_ids(
@@ -137,15 +136,14 @@ impl MlDb {
if pet_face_id_to_vector_id.is_empty() {
return Ok(());
}
- self.db
- .write_batches_committing_each(
- "UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",
- const { NonZeroUsize::new(500).unwrap() },
- pet_face_id_to_vector_id
- .iter()
- .map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ "UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",
+ const { NonZeroUsize::new(500).unwrap() },
+ pet_face_id_to_vector_id
+ .iter()
+ .map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),
+ )
+ .map_err(Into::into)
}
pub fn update_pet_body_vector_ids(
@@ -155,54 +153,49 @@ impl MlDb {
if pet_body_id_to_vector_id.is_empty() {
return Ok(());
}
- self.db
- .write_batches_committing_each(
- "UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",
- const { NonZeroUsize::new(500).unwrap() },
- pet_body_id_to_vector_id
- .iter()
- .map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),
- )
- .map_err(Into::into)
+ self.write_batches_committing_each(
+ "UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",
+ const { NonZeroUsize::new(500).unwrap() },
+ pet_body_id_to_vector_id
+ .iter()
+ .map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),
+ )
+ .map_err(Into::into)
}
pub fn get_pet_faces_for_file_id(&self, file_upload_id: i64) -> Result<Vec<PetFaceRow>> {
- self.db
- .read_all(
- "SELECT * FROM pet_faces WHERE file_id = ? AND species != -1",
- [file_upload_id],
- read_pet_face,
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT * FROM pet_faces WHERE file_id = ? AND species != -1",
+ [file_upload_id],
+ read_pet_face,
+ )
+ .map_err(Into::into)
}
pub fn get_pet_bodies_for_file_id(&self, file_upload_id: i64) -> Result<Vec<PetBodyRow>> {
- self.db
- .read_all(
- "SELECT * FROM pet_bodies WHERE file_id = ? AND species != -1",
- [file_upload_id],
- read_pet_body,
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT * FROM pet_bodies WHERE file_id = ? AND species != -1",
+ [file_upload_id],
+ read_pet_body,
+ )
+ .map_err(Into::into)
}
pub fn pet_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {
- self.db
- .read_all(
- "SELECT DISTINCT file_id, ml_version FROM pet_faces WHERE ml_version >= ?",
- [minimum_ml_version],
- pair,
- )
- .map_err(Into::into)
+ self.read_all(
+ "SELECT DISTINCT file_id, ml_version FROM pet_faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ pair,
+ )
+ .map_err(Into::into)
}
pub fn get_pet_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
- self.db
- .read_value(
- "SELECT COUNT(DISTINCT file_id) as count FROM pet_faces WHERE ml_version >= ?",
- [minimum_ml_version],
- )
- .map_err(Into::into)
+ self.read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM pet_faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ )
+ .map_err(Into::into)
}
pub fn get_pet_rows_for_files(&self, file_ids: &[i64]) -> Result<PetRowsForFiles> {
@@ -221,24 +214,20 @@ impl MlDb {
WHERE file_id IN ({placeholders})
"#
);
- let faces = self
- .db
- .read_all(&faces_sql, params_from_iter(file_ids), |row| {
- Ok(PetFaceVectorRow {
- pet_face_id: row.get(0)?,
- face_vector_id: row.get(1)?,
- species: row.get(2)?,
- })
- })?;
- let bodies = self
- .db
- .read_all(&bodies_sql, params_from_iter(file_ids), |row| {
- Ok(PetBodyVectorRow {
- pet_body_id: row.get(0)?,
- body_vector_id: row.get(1)?,
- species: row.get(2)?,
- })
- })?;
+ let faces = self.read_all(&faces_sql, params_from_iter(file_ids), |row| {
+ Ok(PetFaceVectorRow {
+ pet_face_id: row.get(0)?,
+ face_vector_id: row.get(1)?,
+ species: row.get(2)?,
+ })
+ })?;
+ let bodies = self.read_all(&bodies_sql, params_from_iter(file_ids), |row| {
+ Ok(PetBodyVectorRow {
+ pet_body_id: row.get(0)?,
+ body_vector_id: row.get(1)?,
+ species: row.get(2)?,
+ })
+ })?;
Ok(PetRowsForFiles { faces, bodies })
}
@@ -328,18 +317,16 @@ impl MlDb {
return Ok(HashMap::new());
}
if create_if_missing {
- self.db
- .write_batch_atomic(insert_sql, unique_ids.iter().map(|id| [id]))?;
+ self.write_batch_atomic(insert_sql, unique_ids.iter().map(|id| [id]))?;
}
let select_sql = select_prefix.to_owned() + " ({})";
- self.db
- .read_chunked_in(
- &select_sql,
- &unique_ids,
- const { NonZeroUsize::new(800).unwrap() },
- pair,
- )
- .map_err(Into::into)
+ self.read_chunked_in(
+ &select_sql,
+ &unique_ids,
+ const { NonZeroUsize::new(800).unwrap() },
+ pair,
+ )
+ .map_err(Into::into)
}
}