π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Add configurable SQLite readers and transaction callbacks
laurenspriemd15043a680b5
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rsindex 1f74b5cb44..71cb216736 100644--- a/rust/crates/photos/src/db.rs+++ b/rust/crates/photos/src/db.rs@@ -1,14 +1,14 @@use std::ops::{Deref, DerefMut};use std::path::Path;-use std::sync::{Mutex, MutexGuard, PoisonError};+use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};use std::time::Duration;-use rusqlite::{OpenFlags, types::FromSql};+use rusqlite::{OpenFlags, TransactionBehavior, params_from_iter, types::FromSql};pub use rusqlite::types;pub use rusqlite::{- Error as SqliteError, OptionalExtension, Params, Result as SqliteResult, Row, Rows, Statement,- ToSql, Transaction, TransactionBehavior, params, params_from_iter,+ Error as SqliteError, OptionalExtension, Params, Result as SqliteResult, Row, ToSql,+ Transaction,};const WRITER_PRAGMAS: &str = "PRAGMA journal_mode = WAL;@@ -23,10 +23,23 @@ pub enum Error {Sqlite(#[from] SqliteError),#[error("currentVersion({current}) cannot be greater than toVersion({target})")]Downgrade { current: i64, target: i64 },+ #[error("reader_count must be at least 1")]+ InvalidReaderCount,}pub type Result<T> = std::result::Result<T, Error>;+#[derive(Clone, Copy, Debug)]+pub struct OpenOptions {+ pub reader_count: usize,+}++impl Default for OpenOptions {+ fn default() -> Self {+ Self { reader_count: 1 }+ }+}+pub struct Connection(rusqlite::Connection);impl Connection {@@ -71,13 +84,23 @@ pub struct Database {impl Database {pub fn open(path: impl AsRef<Path>, migration_scripts: &[&str]) -> Result<Self> {+ Self::open_with_options(path, migration_scripts, OpenOptions::default())+ }++ pub fn open_with_options(+ path: impl AsRef<Path>,+ migration_scripts: &[&str],+ options: OpenOptions,+ ) -> Result<Self> {+ if options.reader_count == 0 {+ return Err(Error::InvalidReaderCount);+ }let path = path.as_ref();let mut writer = Connection::open(path)?;migrate(&mut writer, migration_scripts)?;- let readers = [- Connection::open_read_only(path)?,- Connection::open_read_only(path)?,- ];+ let readers = (0..options.reader_count)+ .map(|_| Connection::open_read_only(path))+ .collect::<Result<Vec<_>>>()?;Ok(Self {pool: Pool::new(writer, readers),})@@ -90,7 +113,7 @@ impl Database {map: impl FnMut(&Row<'_>) -> SqliteResult<T>,) -> Result<C> {self.pool.read(|connection| {- let mut statement = connection.prepare(sql)?;+ let mut statement = connection.prepare_cached(sql)?;let rows = statement.query_map(parameters, map)?;Ok(rows.collect::<SqliteResult<C>>()?)})@@ -105,8 +128,11 @@ impl Database {}pub fn read_value<T: FromSql>(&self, sql: &str, parameters: impl Params) -> Result<T> {- self.pool- .read(|connection| Ok(connection.query_row(sql, parameters, |row| row.get(0))?))+ self.pool.read(|connection| {+ Ok(connection+ .prepare_cached(sql)?+ .query_row(parameters, |row| row.get(0))?)+ })}pub fn read_optional<T: FromSql>(@@ -116,23 +142,51 @@ impl Database {) -> Result<Option<T>> {self.pool.read(|connection| {Ok(connection- .query_row(sql, parameters, |row| row.get(0))+ .prepare_cached(sql)?+ .query_row(parameters, |row| row.get(0)).optional()?)})}- pub fn execute(&self, sql: &str, parameters: impl Params) -> Result<()> {+ pub fn read_transaction<T>(+ &self,+ read: impl FnOnce(&Transaction<'_>) -> Result<T>,+ ) -> Result<T> {+ self.pool.read(|connection| {+ let transaction =+ connection.transaction_with_behavior(TransactionBehavior::Deferred)?;+ let result = read(&transaction)?;+ transaction.commit()?;+ Ok(result)+ })+ }++ pub fn write_transaction<T>(+ &self,+ write: impl FnOnce(&Transaction<'_>) -> Result<T>,+ ) -> Result<T> {self.pool.write(|connection| {- connection.execute(sql, parameters)?;- Ok(())+ let transaction =+ connection.transaction_with_behavior(TransactionBehavior::Immediate)?;+ let result = write(&transaction)?;+ transaction.commit()?;+ Ok(result)})}+ pub fn execute(&self, sql: &str, parameters: impl Params) -> Result<usize> {+ self.pool+ .write(|connection| Ok(connection.prepare_cached(sql)?.execute(parameters)?))+ }+pub fn execute_chunked_in<I: ToSql>(&self, sql: &str, ids: &[I]) -> Result<()> {- self.pool.write(|connection| {+ if ids.is_empty() {+ return Ok(());+ }+ self.write_transaction(|transaction| {for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {let sql = sql.replacen("{}", &bind_placeholders(chunk.len()), 1);- connection.execute(&sql, params_from_iter(chunk))?;+ transaction.execute(&sql, params_from_iter(chunk))?;}Ok(())})@@ -142,9 +196,9 @@ impl Database {&self,statements: impl IntoIterator<Item = &'a str>,) -> Result<()> {- self.pool.write(|connection| {+ self.write_transaction(|transaction| {for statement in statements {- connection.execute_batch(statement)?;+ transaction.execute_batch(statement)?;}Ok(())})@@ -155,16 +209,11 @@ impl Database {sql: &str,parameter_sets: impl IntoIterator<Item = P>,) -> Result<()> {- self.pool.write(|connection| {- let transaction =- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;- {- let mut statement = transaction.prepare(sql)?;- for parameters in parameter_sets {- statement.execute(parameters)?;- }+ self.write_transaction(|transaction| {+ let mut statement = transaction.prepare_cached(sql)?;+ for parameters in parameter_sets {+ statement.execute(parameters)?;}- transaction.commit()?;Ok(())})}@@ -172,25 +221,38 @@ impl Database {struct Pool {writer: Mutex<Connection>,- readers: [Mutex<Connection>; 2],+ readers: Mutex<Vec<Connection>>,+ reader_available: Condvar,}impl Pool {- fn new(writer: Connection, readers: [Connection; 2]) -> Self {- let [first, second] = readers;+ fn new(writer: Connection, readers: Vec<Connection>) -> Self {Self {writer: Mutex::new(writer),- readers: [Mutex::new(first), Mutex::new(second)],+ readers: Mutex::new(readers),+ reader_available: Condvar::new(),}}- fn read<T>(&self, query: impl FnOnce(&Connection) -> Result<T>) -> Result<T> {- for reader in &self.readers {- if let Ok(connection) = reader.try_lock() {- return query(&connection);+ fn acquire_reader(&self) -> Reader<'_> {+ let mut readers = lock(&self.readers);+ loop {+ if let Some(connection) = readers.pop() {+ return Reader {+ pool: self,+ connection: Some(connection),+ };}+ readers = self+ .reader_available+ .wait(readers)+ .unwrap_or_else(PoisonError::into_inner);}- query(&lock(&self.readers[0]))+ }++ fn read<T>(&self, query: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {+ let mut reader = self.acquire_reader();+ query(reader.connection.as_mut().unwrap())}fn write<T>(&self, statement: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {@@ -198,8 +260,22 @@ impl Pool {}}-fn lock(connection: &Mutex<Connection>) -> MutexGuard<'_, Connection> {- connection.lock().unwrap_or_else(PoisonError::into_inner)+struct Reader<'a> {+ pool: &'a Pool,+ connection: Option<Connection>,+}++impl Drop for Reader<'_> {+ fn drop(&mut self) {+ if let Some(connection) = self.connection.take() {+ lock(&self.pool.readers).push(connection);+ self.pool.reader_available.notify_one();+ }+ }+}++fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+ mutex.lock().unwrap_or_else(PoisonError::into_inner)}fn migrate(connection: &mut Connection, scripts: &[&str]) -> Result<()> {@@ -245,11 +321,22 @@ pub(crate) fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql>#[cfg(test)]mod tests {- use super::{Connection, Database, Error, bind_placeholders, types};+ use std::panic::{AssertUnwindSafe, catch_unwind};+ use std::sync::mpsc::{self, RecvTimeoutError};+ use std::thread;+ use std::time::Duration;++ use super::{Connection, Database, Error, OpenOptions, bind_placeholders, lock, types};const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";+ fn open() -> (tempfile::TempDir, Database) {+ let directory = tempfile::tempdir().unwrap();+ let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ (directory, db)+ }+fn pragma<T: types::FromSql>(connection: &Connection, name: &str) -> T {connection.pragma_query_value(None, name, |row| row.get(0))@@ -308,8 +395,7 @@ mod tests {#[test]fn reader_queries_reject_writes() {- let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ let (_directory, db) = open();db.execute("INSERT INTO items (id) VALUES (?)", [7]).unwrap();let deleted: Result<Vec<i64>, _> = db.read_column("DELETE FROM items RETURNING id", ());@@ -323,8 +409,7 @@ mod tests {#[test]fn failed_batch_rolls_back_and_writer_recovers() {- let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ let (_directory, db) = open();assert!(db.write_batch("INSERT INTO items (id) VALUES (?)", [[1], [2], [1]]).is_err()@@ -334,8 +419,15 @@ mod tests {.unwrap(),0);- db.execute("INSERT INTO items (id) VALUES (?)", [3])- .unwrap();+ assert_eq!(+ db.execute("INSERT INTO items (id) VALUES (?)", [3])+ .unwrap(),+ 1+ );+ assert_eq!(+ db.execute("DELETE FROM items WHERE id = ?", [4]).unwrap(),+ 0+ );assert_eq!(db.read_column::<Vec<i64>, _, _>("SELECT id FROM items", ()).unwrap(),@@ -344,9 +436,177 @@ mod tests {}#[test]- fn chunked_delete_handles_large_and_empty_id_lists() {+ fn write_transactions_commit_or_rollback_and_writer_recovers() {+ let (_directory, db) = open();+ let result = db.write_transaction(|transaction| {+ transaction.execute("INSERT INTO items VALUES (?)", [1])?;+ transaction.execute("INSERT INTO missing_table VALUES (?)", [2])?;+ Ok(())+ });+ assert!(result.is_err());+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );++ assert!(+ catch_unwind(AssertUnwindSafe(|| {+ let _: super::Result<()> = db.write_transaction(|transaction| {+ transaction.execute("INSERT INTO items VALUES (?)", [3])?;+ panic!("transaction callback failed");+ });+ }))+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ let total = db+ .write_transaction(|transaction| {+ transaction.execute("INSERT INTO items VALUES (?)", [1])?;+ transaction.execute("INSERT INTO items VALUES (?)", [2])?;+ Ok(transaction+ .query_row("SELECT SUM(id) FROM items", (), |row| row.get::<_, i64>(0))?)+ })+ .unwrap();+ assert_eq!(total, 3);+ assert_eq!(+ db.read_value::<i64>("SELECT SUM(id) FROM items", ())+ .unwrap(),+ 3+ );+ }++ #[test]+ fn read_transaction_keeps_a_snapshot_and_rejects_writes() {+ let (_directory, db) = open();+ db.execute("INSERT INTO items VALUES (?)", [1]).unwrap();+ let counts = db+ .read_transaction(|transaction| {+ let before: i64 =+ transaction.query_row("SELECT COUNT(*) FROM items", (), |row| row.get(0))?;+ db.execute("INSERT INTO items VALUES (?)", [2])?;+ let after: i64 =+ transaction.query_row("SELECT COUNT(*) FROM items", (), |row| row.get(0))?;+ assert!(+ transaction+ .execute("INSERT INTO items VALUES (?)", [3])+ .is_err()+ );+ Ok((before, after))+ })+ .unwrap();+ assert_eq!(counts, (1, 1));+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 2+ );+ }++ #[test]+ fn statement_sequence_rolls_back_if_a_later_statement_fails() {+ let (_directory, db) = open();+ assert!(+ db.execute_statements([+ "INSERT INTO items VALUES (1)",+ "INSERT INTO missing_table VALUES (2)",+ ])+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 0+ );+ }++ #[test]+ fn zero_readers_is_rejected_before_creating_a_database() {let directory = tempfile::tempdir().unwrap();- let db = Database::open(directory.path().join("items.db"), &[CREATE_ITEMS]).unwrap();+ let path = directory.path().join("items.db");+ assert!(matches!(+ Database::open_with_options(&path, &[CREATE_ITEMS], OpenOptions { reader_count: 0 }),+ Err(Error::InvalidReaderCount)+ ));+ assert!(!path.exists());+ }++ #[test]+ fn configured_readers_bound_concurrency_and_wake_on_any_returned_connection() {+ for reader_count in [1, 2, 3] {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("items.db");+ let db = if reader_count == 1 {+ Database::open(path, &[CREATE_ITEMS])+ } else {+ Database::open_with_options(path, &[CREATE_ITEMS], OpenOptions { reader_count })+ }+ .unwrap();++ thread::scope(|scope| {+ let mut release_readers = Vec::new();+ for _ in 0..reader_count {+ let (entered_tx, entered_rx) = mpsc::channel();+ let (release_tx, release_rx) = mpsc::channel();+ release_readers.push(release_tx);+ let db = &db;+ scope.spawn(move || {+ db.read_transaction(|transaction| {+ transaction.query_row("SELECT 1", (), |_| Ok(()))?;+ entered_tx.send(()).unwrap();+ release_rx.recv().unwrap();+ Ok(())+ })+ .unwrap();+ });+ entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();+ }++ let (result_tx, result_rx) = mpsc::channel();+ let db = &db;+ scope.spawn(move || {+ result_tx+ .send(db.read_value::<i64>("SELECT 7", ()))+ .unwrap();+ });+ let was_blocked = matches!(+ result_rx.recv_timeout(Duration::from_millis(100)),+ Err(RecvTimeoutError::Timeout)+ );+ release_readers.pop().unwrap().send(()).unwrap();+ let result = result_rx.recv_timeout(Duration::from_secs(5));+ for release in release_readers {+ release.send(()).unwrap();+ }+ assert!(was_blocked);+ assert_eq!(result.unwrap().unwrap(), 7);+ });+ }+ }++ #[test]+ fn reader_returns_to_pool_after_a_callback_panics() {+ let (_directory, db) = open();+ assert!(+ catch_unwind(AssertUnwindSafe(|| {+ let _: super::Result<()> = db.read_transaction(|transaction| {+ transaction.query_row("SELECT 1", (), |_| Ok(()))?;+ panic!("reader callback failed");+ });+ }))+ .is_err()+ );+ assert_eq!(lock(&db.pool.readers).len(), 1);+ assert_eq!(db.read_value::<i64>("SELECT 7", ()).unwrap(), 7);+ }++ #[test]+ fn chunked_delete_handles_empty_lists_and_rolls_back_on_failure() {+ let (_directory, db) = open();let ids: Vec<i64> = (0..10_001).collect();db.write_batch("INSERT INTO items (id) VALUES (?)",@@ -360,6 +620,20 @@ mod tests {.unwrap(),10_001);+ db.execute_statements(["CREATE TRIGGER reject_last_delete BEFORE DELETE ON items+ WHEN OLD.id = 10000 BEGIN SELECT RAISE(ABORT, 'cannot delete'); END"])+ .unwrap();+ assert!(+ db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids)+ .is_err()+ );+ assert_eq!(+ db.read_value::<i64>("SELECT COUNT(*) FROM items", ())+ .unwrap(),+ 10_001+ );+ db.execute_statements(["DROP TRIGGER reject_last_delete"])+ .unwrap();db.execute_chunked_in("DELETE FROM items WHERE id IN ({})", &ids).unwrap();assert_eq!(diff --git a/rust/crates/photos/src/ml_db/caches.rs b/rust/crates/photos/src/ml_db/caches.rsindex 9bdb18e3bb..dab60f85e7 100644--- a/rust/crates/photos/src/ml_db/caches.rs+++ b/rust/crates/photos/src/ml_db/caches.rs@@ -12,7 +12,8 @@ impl MlDb {self.db.execute("INSERT OR REPLACE INTO text_embeddings_cache (text_query, embedding, ml_version, created_at) VALUES (?, ?, ?, ?)",(query, embedding_bytes, CLIP_ML_VERSION, now_millis()),- ).map_err(Into::into)+ )?;+ Ok(())}pub fn get_repeated_text_embedding_cache(&self, query: &str) -> Result<Option<Vec<f32>>> {@@ -42,12 +43,11 @@ impl MlDb {person_or_cluster_id: &str,face_id: &str,) -> Result<()> {- self.db- .execute(- "INSERT OR REPLACE INTO face_cache (person_or_cluster_id, face_id) VALUES (?, ?)",- [person_or_cluster_id, face_id],- )- .map_err(Into::into)+ self.db.execute(+ "INSERT OR REPLACE INTO face_cache (person_or_cluster_id, face_id) VALUES (?, ?)",+ [person_or_cluster_id, face_id],+ )?;+ Ok(())}pub fn get_face_id_used_for_person_or_cluster(@@ -66,12 +66,11 @@ impl MlDb {&self,person_or_cluster_id: &str,) -> Result<()> {- self.db- .execute(- "DELETE FROM face_cache WHERE person_or_cluster_id = ?",- [person_or_cluster_id],- )- .map_err(Into::into)+ self.db.execute(+ "DELETE FROM face_cache WHERE person_or_cluster_id = ?",+ [person_or_cluster_id],+ )?;+ Ok(())}}diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rsindex 12fc4d893c..7c1645c0a9 100644--- a/rust/crates/photos/src/ml_db/mod.rs+++ b/rust/crates/photos/src/ml_db/mod.rs@@ -6,7 +6,7 @@ mod vector_encoding;use std::path::Path;-use crate::db::Database;+use crate::db::{Database, OpenOptions};pub use filedata::{FdStatus, PreviewInfo};pub use queries::{@@ -42,7 +42,11 @@ pub struct MlDb {impl MlDb {pub fn open(path: impl AsRef<Path>) -> Result<Self> {Ok(Self {- db: Database::open(path, &schema::MIGRATION_SCRIPTS)?,+ db: Database::open_with_options(+ path,+ &schema::MIGRATION_SCRIPTS,+ OpenOptions { reader_count: 2 },+ )?,})}}diff --git a/rust/crates/photos/src/ml_db/queries.rs b/rust/crates/photos/src/ml_db/queries.rsindex 53842d13b3..499d97a95c 100644--- a/rust/crates/photos/src/ml_db/queries.rs+++ b/rust/crates/photos/src/ml_db/queries.rs@@ -194,13 +194,11 @@ impl MlDb {pub fn insert_clip_rows(&self, embeddings: &[ClipEmbedding]) -> Result<()> {if let [embedding] = embeddings {- return self- .db- .execute(- "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) VALUES (?, ?, ?)",- clip_row(embedding),- )- .map_err(Into::into);+ self.db.execute(+ "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) VALUES (?, ?, ?)",+ clip_row(embedding),+ )?;+ return Ok(());}self.db.write_batch(