Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Add Rust SQLite wrapper and ML DB foundation to ente-photos

laurenspriem9b9853756710
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 40836b1c7d..8dd9a3153c 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2710,6 +2710,7 @@ dependencies = [
"ente-assets",
"ente-location",
"quick-xml",
+ "rusqlite",
"tempfile",
"thiserror 2.0.18",
]
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 060e93d86f..7157d61d93 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2024"
ente-assets.workspace = true
ente-location.workspace = true
quick-xml = "0.41.0"
+rusqlite = { workspace = true, default-features = true }
thiserror.workspace = true
[dev-dependencies]
diff --git a/rust/crates/photos/src/db/mod.rs b/rust/crates/photos/src/db/mod.rs
new file mode 100644
index 0000000000..feef412f24
--- /dev/null
+++ b/rust/crates/photos/src/db/mod.rs
@@ -0,0 +1,85 @@
+use std::ops::{Deref, DerefMut};
+use std::path::Path;
+
+pub use rusqlite::types;
+pub use rusqlite::{
+ Error, OptionalExtension, Params, Result, Row, Rows, Statement, ToSql, Transaction,
+ TransactionBehavior, params, params_from_iter,
+};
+
+const OPEN_PRAGMAS: &str = "PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+PRAGMA journal_size_limit = 6291456;
+PRAGMA busy_timeout = 30000;";
+
+pub struct Connection(rusqlite::Connection);
+
+impl Connection {
+ pub fn open(path: impl AsRef<Path>) -> Result<Self> {
+ let connection = rusqlite::Connection::open(path)?;
+ connection.execute_batch(OPEN_PRAGMAS)?;
+ Ok(Self(connection))
+ }
+
+ pub fn open_in_memory() -> Result<Self> {
+ Ok(Self(rusqlite::Connection::open_in_memory()?))
+ }
+}
+
+impl Deref for Connection {
+ type Target = rusqlite::Connection;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+impl DerefMut for Connection {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.0
+ }
+}
+
+pub fn bind_placeholders(count: usize) -> String {
+ vec!["?"; count].join(", ")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn pragma<T: types::FromSql>(connection: &Connection, name: &str) -> T {
+ connection
+ .pragma_query_value(None, name, |row| row.get(0))
+ .unwrap()
+ }
+
+ #[test]
+ fn open_applies_connection_pragmas() {
+ let directory = tempfile::tempdir().unwrap();
+ let connection = Connection::open(directory.path().join("test.db")).unwrap();
+ assert_eq!(pragma::<String>(&connection, "journal_mode"), "wal");
+ assert_eq!(pragma::<i64>(&connection, "synchronous"), 1);
+ assert_eq!(pragma::<i64>(&connection, "journal_size_limit"), 6291456);
+ assert_eq!(pragma::<i64>(&connection, "busy_timeout"), 30000);
+ }
+
+ #[test]
+ fn in_memory_connection_runs_statements() {
+ let connection = Connection::open_in_memory().unwrap();
+ connection
+ .execute_batch("CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (7);")
+ .unwrap();
+ let id: i64 = connection
+ .query_row("SELECT id FROM t", (), |row| row.get(0))
+ .unwrap();
+ assert_eq!(id, 7);
+ }
+
+ #[test]
+ fn bind_placeholders_joins_question_marks() {
+ assert_eq!(bind_placeholders(0), "");
+ assert_eq!(bind_placeholders(1), "?");
+ assert_eq!(bind_placeholders(3), "?, ?, ?");
+ }
+}
diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rs
index 1e46f67699..893827368c 100644
--- a/rust/crates/photos/src/lib.rs
+++ b/rust/crates/photos/src/lib.rs
@@ -1,4 +1,6 @@
+pub mod db;
pub mod location;
+pub mod ml_db;
pub mod motion_photo;
pub use motion_photo::{
diff --git a/rust/crates/photos/src/ml_db/backend.rs b/rust/crates/photos/src/ml_db/backend.rs
new file mode 100644
index 0000000000..f96741a064
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/backend.rs
@@ -0,0 +1,50 @@
+use std::sync::OnceLock;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Backend {
+ Dart,
+ Rust,
+}
+
+static BACKEND: OnceLock<Backend> = OnceLock::new();
+
+pub fn decide(prefer_rust: bool) -> Backend {
+ decide_with(&BACKEND, prefer_rust)
+}
+
+fn decide_with(latch: &OnceLock<Backend>, prefer_rust: bool) -> Backend {
+ *latch.get_or_init(|| {
+ if prefer_rust {
+ Backend::Rust
+ } else {
+ Backend::Dart
+ }
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn first_decision_wins() {
+ let latch = OnceLock::new();
+ assert_eq!(decide_with(&latch, true), Backend::Rust);
+ assert_eq!(decide_with(&latch, false), Backend::Rust);
+ assert_eq!(decide_with(&latch, true), Backend::Rust);
+ }
+
+ #[test]
+ fn dart_stays_latched() {
+ let latch = OnceLock::new();
+ assert_eq!(decide_with(&latch, false), Backend::Dart);
+ assert_eq!(decide_with(&latch, true), Backend::Dart);
+ }
+
+ #[test]
+ fn process_latch_is_stable() {
+ let first = decide(false);
+ assert_eq!(decide(true), first);
+ assert_eq!(decide(false), first);
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/caches.rs b/rust/crates/photos/src/ml_db/caches.rs
new file mode 100644
index 0000000000..67974ea2c4
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/caches.rs
@@ -0,0 +1,78 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use super::MlDb;
+use super::codec::{decode_f32, encode_f32};
+use super::constants::CLIP_ML_VERSION;
+use super::error::Result;
+
+const THREE_MONTHS_MILLIS: i64 = 90 * 24 * 60 * 60 * 1000;
+
+impl MlDb {
+ pub fn put_repeated_text_embedding_cache(&self, query: &str, embedding: &[f64]) -> Result<()> {
+ let embedding_bytes = encode_f32(embedding.iter().map(|value| *value as f32));
+ self.execute(
+ "INSERT OR REPLACE INTO text_embeddings_cache (text_query, embedding, ml_version, created_at) VALUES (?, ?, ?, ?)",
+ (query, embedding_bytes, CLIP_ML_VERSION, now_millis()),
+ )
+ }
+
+ pub fn get_repeated_text_embedding_cache(&self, query: &str) -> Result<Option<Vec<f32>>> {
+ let results: Vec<(Vec<u8>, i64, i64)> = self.read_all(
+ "SELECT embedding, ml_version, created_at FROM text_embeddings_cache WHERE text_query = ?",
+ [query],
+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+ )?;
+ if results.is_empty() {
+ return Ok(None);
+ }
+ let three_months_ago = now_millis() - THREE_MONTHS_MILLIS;
+ for (embedding, ml_version, created_at) in &results {
+ if *ml_version == CLIP_ML_VERSION && *created_at > three_months_ago {
+ return Ok(Some(decode_f32(embedding)));
+ }
+ }
+ self.execute(
+ "DELETE FROM text_embeddings_cache WHERE text_query = ?",
+ [query],
+ )?;
+ Ok(None)
+ }
+
+ pub fn put_face_id_cached_for_person_or_cluster(
+ &self,
+ person_or_cluster_id: &str,
+ face_id: &str,
+ ) -> Result<()> {
+ self.execute(
+ "INSERT OR REPLACE INTO face_cache (person_or_cluster_id, face_id) VALUES (?, ?)",
+ [person_or_cluster_id, face_id],
+ )
+ }
+
+ pub fn get_face_id_used_for_person_or_cluster(
+ &self,
+ person_or_cluster_id: &str,
+ ) -> Result<Option<String>> {
+ self.read_optional(
+ "SELECT face_id FROM face_cache WHERE person_or_cluster_id = ?",
+ [person_or_cluster_id],
+ )
+ }
+
+ pub fn remove_face_id_cached_for_person_or_cluster(
+ &self,
+ person_or_cluster_id: &str,
+ ) -> Result<()> {
+ self.execute(
+ "DELETE FROM face_cache WHERE person_or_cluster_id = ?",
+ [person_or_cluster_id],
+ )
+ }
+}
+
+fn now_millis() -> i64 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map(|elapsed| elapsed.as_millis() as i64)
+ .unwrap_or(0)
+}
diff --git a/rust/crates/photos/src/ml_db/clip.rs b/rust/crates/photos/src/ml_db/clip.rs
new file mode 100644
index 0000000000..ebe89b5616
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/clip.rs
@@ -0,0 +1,87 @@
+use std::collections::HashMap;
+
+use super::codec::{decode_f32, encode_f32};
+use super::constants::CLIP_EMBEDDING_BYTES_LENGTH;
+use super::error::Result;
+use super::schema::DELETE_CLIP_EMBEDDINGS;
+use super::types::{ClipEmbedding, ClipRow, EmbeddingVector};
+use super::{MlDb, pair};
+
+impl MlDb {
+ pub fn get_all_clip_vectors(&self) -> Result<Vec<EmbeddingVector>> {
+ let rows: Vec<(i64, Vec<u8>)> =
+ self.read_all("SELECT file_id, embedding FROM clip", (), pair)?;
+ Ok(rows
+ .into_iter()
+ .map(|(file_id, embedding)| EmbeddingVector {
+ file_id,
+ embedding: decode_f32(&embedding),
+ })
+ .filter(|vector| !vector.embedding.is_empty())
+ .collect())
+ }
+
+ pub fn clip_indexed_file_with_version(&self) -> Result<HashMap<i64, i64>> {
+ self.read_all("SELECT file_id , ml_version FROM clip", (), pair)
+ }
+
+ pub fn get_clip_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
+ self.read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ?",
+ [minimum_ml_version],
+ )
+ }
+
+ pub fn get_clip_vectorizable_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
+ self.read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ? AND LENGTH(embedding) = ?",
+ [minimum_ml_version, CLIP_EMBEDDING_BYTES_LENGTH],
+ )
+ }
+
+ pub fn insert_clip_rows(&self, embeddings: &[ClipEmbedding]) -> Result<()> {
+ if let [embedding] = embeddings {
+ return self.execute(
+ "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) VALUES (?, ?, ?)",
+ clip_row(embedding),
+ );
+ }
+ self.write_batch(
+ "INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) values(?, ?, ?)",
+ embeddings.iter().map(clip_row),
+ )
+ }
+
+ pub fn delete_clip_rows(&self, file_ids: &[i64]) -> Result<()> {
+ self.execute_chunked_in("DELETE FROM clip WHERE file_id IN ({})", file_ids)
+ }
+
+ pub fn delete_all_clip_rows(&self) -> Result<()> {
+ self.execute_statements([DELETE_CLIP_EMBEDDINGS])
+ }
+
+ pub fn count_clip_rows(&self) -> Result<i64> {
+ self.read_value("SELECT COUNT(file_id) as total FROM clip", ())
+ }
+
+ pub fn get_clip_rows_page(&self, limit: i64, offset: i64) -> Result<Vec<ClipRow>> {
+ self.read_all(
+ "SELECT file_id, embedding FROM clip ORDER BY file_id DESC LIMIT ? OFFSET ?",
+ [limit, offset],
+ |row| {
+ Ok(ClipRow {
+ file_id: row.get(0)?,
+ embedding: row.get(1)?,
+ })
+ },
+ )
+ }
+}
+
+fn clip_row(embedding: &ClipEmbedding) -> (i64, Vec<u8>, i64) {
+ (
+ embedding.file_id,
+ encode_f32(embedding.embedding.iter().map(|value| *value as f32)),
+ embedding.version,
+ )
+}
diff --git a/rust/crates/photos/src/ml_db/codec.rs b/rust/crates/photos/src/ml_db/codec.rs
new file mode 100644
index 0000000000..205b110984
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/codec.rs
@@ -0,0 +1,188 @@
+use super::error::{Error, Result};
+
+const EVECTOR_VALUES_PACKED_TAG: u64 = 0x0A;
+const EVECTOR_VALUES_FIXED64_TAG: u64 = 0x09;
+
+pub fn encode_evector(values: &[f64]) -> Vec<u8> {
+ if values.is_empty() {
+ return Vec::new();
+ }
+ let mut bytes = Vec::with_capacity(values.len() * 8 + 10);
+ bytes.push(EVECTOR_VALUES_PACKED_TAG as u8);
+ push_varint(&mut bytes, (values.len() * 8) as u64);
+ for value in values {
+ bytes.extend_from_slice(&value.to_le_bytes());
+ }
+ bytes
+}
+
+pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {
+ let mut values = Vec::new();
+ let mut offset = 0;
+ while offset < bytes.len() {
+ let tag = read_varint(bytes, &mut offset)?;
+ match (tag, tag & 7) {
+ (EVECTOR_VALUES_PACKED_TAG, _) => {
+ let length = read_length(bytes, &mut offset)?;
+ if length % 8 != 0 {
+ return Err(malformed());
+ }
+ for chunk in bytes[offset..offset + length].as_chunks::<8>().0 {
+ values.push(f64::from_le_bytes(*chunk));
+ }
+ offset += length;
+ }
+ (EVECTOR_VALUES_FIXED64_TAG, _) => {
+ let end = advance(bytes, &mut offset, 8)?;
+ values.push(f64_from_le(&bytes[end - 8..end]));
+ }
+ (_, 0) => {
+ read_varint(bytes, &mut offset)?;
+ }
+ (_, 1) => {
+ advance(bytes, &mut offset, 8)?;
+ }
+ (_, 2) => {
+ let length = read_length(bytes, &mut offset)?;
+ offset += length;
+ }
+ (_, 5) => {
+ advance(bytes, &mut offset, 4)?;
+ }
+ _ => return Err(malformed()),
+ }
+ }
+ Ok(values)
+}
+
+pub fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {
+ values
+ .into_iter()
+ .flat_map(|value| value.to_le_bytes())
+ .collect()
+}
+
+pub fn decode_f32(bytes: &[u8]) -> Vec<f32> {
+ bytes
+ .as_chunks::<4>()
+ .0
+ .iter()
+ .map(|chunk| f32::from_le_bytes(*chunk))
+ .collect()
+}
+
+fn f64_from_le(chunk: &[u8]) -> f64 {
+ let mut buffer = [0u8; 8];
+ buffer.copy_from_slice(chunk);
+ f64::from_le_bytes(buffer)
+}
+
+fn push_varint(bytes: &mut Vec<u8>, mut value: u64) {
+ while value >= 0x80 {
+ bytes.push((value as u8) | 0x80);
+ value >>= 7;
+ }
+ bytes.push(value as u8);
+}
+
+fn read_varint(bytes: &[u8], offset: &mut usize) -> Result<u64> {
+ let mut value = 0u64;
+ for shift in (0..64).step_by(7) {
+ let byte = *bytes.get(*offset).ok_or_else(malformed)?;
+ *offset += 1;
+ value |= u64::from(byte & 0x7F) << shift;
+ if byte & 0x80 == 0 {
+ return Ok(value);
+ }
+ }
+ Err(malformed())
+}
+
+fn read_length(bytes: &[u8], offset: &mut usize) -> Result<usize> {
+ let length = usize::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;
+ if bytes.len() - *offset < length {
+ return Err(malformed());
+ }
+ Ok(length)
+}
+
+fn advance(bytes: &[u8], offset: &mut usize, count: usize) -> Result<usize> {
+ if bytes.len() - *offset < count {
+ return Err(malformed());
+ }
+ *offset += count;
+ Ok(*offset)
+}
+
+fn malformed() -> Error {
+ Error::Codec("malformed EVector bytes".to_string())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn evector_matches_protobuf_packed_encoding() {
+ let bytes = encode_evector(&[1.0, 2.0]);
+ assert_eq!(
+ bytes,
+ [
+ 0x0A, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x40
+ ]
+ );
+ assert_eq!(decode_evector(&bytes).unwrap(), vec![1.0, 2.0]);
+ }
+
+ #[test]
+ fn empty_evector_is_zero_bytes() {
+ assert!(encode_evector(&[]).is_empty());
+ assert!(decode_evector(&[]).unwrap().is_empty());
+ }
+
+ #[test]
+ fn evector_round_trips_long_vectors() {
+ let values: Vec<f64> = (0..192).map(|i| i as f64 * 0.25 - 3.0).collect();
+ let bytes = encode_evector(&values);
+ assert_eq!(bytes[0], 0x0A);
+ assert_eq!(&bytes[1..3], &[0x80, 0x0C]);
+ assert_eq!(decode_evector(&bytes).unwrap(), values);
+ }
+
+ #[test]
+ fn evector_accepts_unpacked_and_unknown_fields() {
+ let mut bytes = vec![0x09];
+ bytes.extend_from_slice(&3.5f64.to_le_bytes());
+ bytes.extend_from_slice(&[0x10, 0x05]);
+ bytes.extend_from_slice(&[0x1A, 0x02, 0xAA, 0xBB]);
+ bytes.extend_from_slice(&[0x25, 1, 2, 3, 4]);
+ bytes.extend_from_slice(&[0x09]);
+ bytes.extend_from_slice(&(-1.0f64).to_le_bytes());
+ assert_eq!(decode_evector(&bytes).unwrap(), vec![3.5, -1.0]);
+ }
+
+ #[test]
+ fn evector_rejects_truncated_bytes() {
+ assert!(decode_evector(&[0x0A, 0x10, 0x00]).is_err());
+ assert!(decode_evector(&[0x0A, 0x04, 0, 0, 0, 0]).is_err());
+ assert!(decode_evector(&[0x0F]).is_err());
+ }
+
+ #[test]
+ fn f32_matches_little_endian_layout() {
+ assert_eq!(encode_f32([1.0f32]), [0x00, 0x00, 0x80, 0x3F]);
+ assert_eq!(decode_f32(&[0x00, 0x00, 0x80, 0x3F]), vec![1.0f32]);
+ assert!(encode_f32([]).is_empty());
+ assert!(decode_f32(&[]).is_empty());
+ }
+
+ #[test]
+ fn f32_round_trips_and_ignores_trailing_bytes() {
+ let values = vec![0.5f32, -2.25, 1e-7];
+ let mut bytes = encode_f32(values.iter().copied());
+ assert_eq!(decode_f32(&bytes), values);
+ bytes.push(0xFF);
+ assert_eq!(decode_f32(&bytes), values);
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/constants.rs b/rust/crates/photos/src/ml_db/constants.rs
new file mode 100644
index 0000000000..ba701db01c
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/constants.rs
@@ -0,0 +1,21 @@
+pub const FACE_ML_VERSION: i64 = 1;
+pub const CLIP_ML_VERSION: i64 = 1;
+pub const PET_ML_VERSION: i64 = 1;
+
+pub const CLIP_EMBEDDING_DIMENSIONS: usize = 512;
+pub const CLIP_EMBEDDING_BYTES_LENGTH: i64 = CLIP_EMBEDDING_DIMENSIONS as i64 * 4;
+
+pub const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;
+pub const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;
+pub const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;
+pub const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;
+pub const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;
+
+pub const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;
+
+pub fn is_bad_face_for_clustering(face_score: f64, blur_value: f64, is_sideways: bool) -> bool {
+ face_score < MINIMUM_QUALITY_FACE_SCORE
+ || blur_value < LAPLACIAN_SOFT_THRESHOLD
+ || (blur_value < LAPLACIAN_VERY_SOFT_THRESHOLD && face_score < MEDIUM_QUALITY_FACE_SCORE)
+ || is_sideways
+}
diff --git a/rust/crates/photos/src/ml_db/error.rs b/rust/crates/photos/src/ml_db/error.rs
new file mode 100644
index 0000000000..2a77e4edc9
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/error.rs
@@ -0,0 +1,15 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error(transparent)]
+ Sqlite(#[from] crate::db::Error),
+ #[error("currentVersion({current}) cannot be greater than toVersion({target})")]
+ Downgrade { current: i64, target: i64 },
+ #[error("{0}")]
+ Codec(String),
+ #[error(transparent)]
+ Io(#[from] std::io::Error),
+ #[error("{0}")]
+ InvalidArgument(String),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
diff --git a/rust/crates/photos/src/ml_db/filedata.rs b/rust/crates/photos/src/ml_db/filedata.rs
new file mode 100644
index 0000000000..e7d567d8f5
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/filedata.rs
@@ -0,0 +1,51 @@
+use std::collections::{HashMap, HashSet};
+
+use super::error::Result;
+use super::types::{FdStatus, PreviewInfo};
+use super::{MlDb, optional_parameter};
+
+impl MlDb {
+ pub fn put_fd_status(&self, fd_status_list: &[FdStatus]) -> Result<()> {
+ if fd_status_list.is_empty() {
+ return Ok(());
+ }
+ self.write_batch(
+ "INSERT OR REPLACE INTO filedata (file_id, user_id, type, size, obj_id, obj_nonce, updated_at ) values(?, ?, ?, ?, ?, ?, ?)",
+ fd_status_list.iter().map(|status| {
+ (
+ status.file_id,
+ status.user_id,
+ &status.data_type,
+ status.size,
+ &status.object_id,
+ &status.object_nonce,
+ status.updated_at,
+ )
+ }),
+ )
+ }
+
+ pub fn get_file_ids_vid_preview(&self) -> Result<HashMap<i64, PreviewInfo>> {
+ self.read_all(
+ "SELECT file_id, obj_id, size FROM filedata WHERE type='vid_preview'",
+ (),
+ |row| {
+ Ok((
+ row.get(0)?,
+ PreviewInfo {
+ object_id: row.get(1)?,
+ object_size: row.get(2)?,
+ },
+ ))
+ },
+ )
+ }
+
+ pub fn get_file_ids_with_fd_data(&self, data_type: Option<&str>) -> Result<HashSet<i64>> {
+ let sql = match data_type {
+ None => "SELECT file_id FROM filedata",
+ Some(_) => "SELECT file_id FROM filedata WHERE type = ?",
+ };
+ self.read_column(sql, optional_parameter(&data_type).as_slice())
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/migrate.rs b/rust/crates/photos/src/ml_db/migrate.rs
new file mode 100644
index 0000000000..6c049e74e1
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/migrate.rs
@@ -0,0 +1,37 @@
+use crate::db::{Connection, TransactionBehavior};
+
+use super::error::{Error, Result};
+use super::schema::MIGRATION_SCRIPTS;
+
+pub const TARGET_VERSION: i64 = MIGRATION_SCRIPTS.len() as i64;
+
+pub fn migrate(connection: &mut Connection) -> Result<()> {
+ let probed: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?;
+ check_not_downgrade(probed)?;
+ if probed == TARGET_VERSION {
+ return Ok(());
+ }
+
+ let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let current: i64 = transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;
+ check_not_downgrade(current)?;
+ if current == TARGET_VERSION {
+ return Ok(());
+ }
+ for script in &MIGRATION_SCRIPTS[current as usize..] {
+ transaction.execute_batch(script)?;
+ }
+ transaction.pragma_update(None, "user_version", TARGET_VERSION)?;
+ transaction.commit()?;
+ Ok(())
+}
+
+fn check_not_downgrade(current: i64) -> Result<()> {
+ if current > TARGET_VERSION {
+ return Err(Error::Downgrade {
+ current,
+ target: TARGET_VERSION,
+ });
+ }
+ Ok(())
+}
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
new file mode 100644
index 0000000000..611bc39103
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -0,0 +1,161 @@
+pub mod backend;
+mod caches;
+mod clip;
+pub mod codec;
+pub mod constants;
+mod error;
+mod filedata;
+mod migrate;
+mod pool;
+pub mod schema;
+pub mod types;
+
+use std::path::Path;
+
+use crate::db::{
+ Connection, OptionalExtension, Params, Row, ToSql, TransactionBehavior, bind_placeholders,
+ params_from_iter, types::FromSql,
+};
+
+pub use backend::{Backend, decide};
+pub use error::{Error, Result};
+pub use migrate::TARGET_VERSION;
+pub use types::*;
+
+use constants::MAX_SQL_BIND_PARAMS_PER_QUERY;
+use pool::Pool;
+
+pub struct MlDb {
+ pool: Pool,
+}
+
+impl MlDb {
+ pub fn open(path: impl AsRef<Path>) -> Result<Self> {
+ let path = path.as_ref();
+ let mut writer = Connection::open(path)?;
+ migrate::migrate(&mut writer)?;
+ let readers = [Connection::open(path)?, Connection::open(path)?];
+ Ok(Self {
+ pool: Pool::new(writer, readers),
+ })
+ }
+
+ pub fn clear_non_pet_tables(&self) -> Result<()> {
+ self.execute_statements([
+ schema::DELETE_FACES,
+ schema::DELETE_FACE_CLUSTERS,
+ schema::DELETE_CLUSTER_PERSON,
+ schema::DELETE_CLUSTER_SUMMARY,
+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ schema::DELETE_NOT_PERSON_FEEDBACK,
+ schema::DELETE_CLIP_EMBEDDINGS,
+ schema::DELETE_FILE_DATA,
+ ])
+ }
+
+ pub fn clear_pet_tables(&self) -> Result<()> {
+ self.execute_statements([
+ schema::DELETE_PET_FACES,
+ schema::DELETE_PET_BODIES,
+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
+ ])
+ }
+
+ fn read_all<C: FromIterator<T>, T, P: Params>(
+ &self,
+ sql: &str,
+ parameters: P,
+ map: impl FnMut(&Row<'_>) -> crate::db::Result<T>,
+ ) -> Result<C> {
+ self.pool.read(|connection| {
+ let mut statement = connection.prepare(sql)?;
+ let rows = statement.query_map(parameters, map)?;
+ Ok(rows.collect::<crate::db::Result<C>>()?)
+ })
+ }
+
+ fn read_column<C: FromIterator<T>, T: FromSql, P: Params>(
+ &self,
+ sql: &str,
+ parameters: P,
+ ) -> Result<C> {
+ self.read_all(sql, parameters, |row| row.get(0))
+ }
+
+ fn read_value<T: FromSql>(&self, sql: &str, parameters: impl Params) -> Result<T> {
+ self.pool
+ .read(|connection| Ok(connection.query_row(sql, parameters, |row| row.get(0))?))
+ }
+
+ fn read_optional<T: FromSql>(&self, sql: &str, parameters: impl Params) -> Result<Option<T>> {
+ self.pool.read(|connection| {
+ Ok(connection
+ .query_row(sql, parameters, |row| row.get(0))
+ .optional()?)
+ })
+ }
+
+ fn execute(&self, sql: &str, parameters: impl Params) -> Result<()> {
+ self.pool.write(|connection| {
+ connection.execute(sql, parameters)?;
+ Ok(())
+ })
+ }
+
+ fn execute_chunked_in<I: ToSql>(&self, sql: &str, ids: &[I]) -> Result<()> {
+ self.pool.write(|connection| {
+ for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
+ let sql = expand_in_clause(sql, chunk.len());
+ connection.execute(&sql, params_from_iter(chunk))?;
+ }
+ Ok(())
+ })
+ }
+
+ fn execute_statements<'a>(&self, statements: impl IntoIterator<Item = &'a str>) -> Result<()> {
+ self.pool.write(|connection| {
+ for statement in statements {
+ connection.execute_batch(statement)?;
+ }
+ Ok(())
+ })
+ }
+
+ fn write_batch<P: Params>(
+ &self,
+ sql: &str,
+ parameter_sets: impl IntoIterator<Item = P>,
+ ) -> Result<()> {
+ self.pool
+ .write(|connection| batch_execute(connection, sql, parameter_sets))
+ }
+}
+
+fn batch_execute<P: Params>(
+ connection: &mut Connection,
+ sql: &str,
+ parameter_sets: impl IntoIterator<Item = P>,
+) -> Result<()> {
+ let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ {
+ let mut statement = transaction.prepare(sql)?;
+ for parameters in parameter_sets {
+ statement.execute(parameters)?;
+ }
+ }
+ transaction.commit()?;
+ Ok(())
+}
+
+fn expand_in_clause(sql: &str, count: usize) -> String {
+ sql.replacen("{}", &bind_placeholders(count), 1)
+}
+
+fn pair<A: FromSql, B: FromSql>(row: &Row<'_>) -> crate::db::Result<(A, B)> {
+ Ok((row.get(0)?, row.get(1)?))
+}
+
+fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql> {
+ value.iter().map(|value| value as &dyn ToSql).collect()
+}
diff --git a/rust/crates/photos/src/ml_db/pool.rs b/rust/crates/photos/src/ml_db/pool.rs
new file mode 100644
index 0000000000..57ffa2e7e3
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/pool.rs
@@ -0,0 +1,37 @@
+use std::sync::{Mutex, MutexGuard, PoisonError};
+
+use crate::db::Connection;
+
+use super::error::Result;
+
+pub struct Pool {
+ writer: Mutex<Connection>,
+ readers: [Mutex<Connection>; 2],
+}
+
+impl Pool {
+ pub fn new(writer: Connection, readers: [Connection; 2]) -> Self {
+ let [first, second] = readers;
+ Self {
+ writer: Mutex::new(writer),
+ readers: [Mutex::new(first), Mutex::new(second)],
+ }
+ }
+
+ pub fn read<T>(&self, query: impl FnOnce(&Connection) -> Result<T>) -> Result<T> {
+ for reader in &self.readers {
+ if let Ok(connection) = reader.try_lock() {
+ return query(&connection);
+ }
+ }
+ query(&lock(&self.readers[0]))
+ }
+
+ pub fn write<T>(&self, statement: impl FnOnce(&mut Connection) -> Result<T>) -> Result<T> {
+ statement(&mut lock(&self.writer))
+ }
+}
+
+fn lock(connection: &Mutex<Connection>) -> MutexGuard<'_, Connection> {
+ connection.lock().unwrap_or_else(PoisonError::into_inner)
+}
diff --git a/rust/crates/photos/src/ml_db/schema.rs b/rust/crates/photos/src/ml_db/schema.rs
new file mode 100644
index 0000000000..9b80d93469
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/schema.rs
@@ -0,0 +1,183 @@
+pub const FACES_TABLE: &str = "faces";
+pub const FACE_CLUSTERS_TABLE: &str = "face_clusters";
+pub const CLUSTER_PERSON_TABLE: &str = "cluster_person";
+pub const CLUSTER_SUMMARY_TABLE: &str = "cluster_summary";
+pub const NOT_PERSON_FEEDBACK_TABLE: &str = "not_person_feedback";
+pub const CLIP_TABLE: &str = "clip";
+pub const FILEDATA_TABLE: &str = "filedata";
+pub const FACE_CACHE_TABLE: &str = "face_cache";
+pub const TEXT_EMBEDDINGS_CACHE_TABLE: &str = "text_embeddings_cache";
+pub const CLUSTER_CENTROID_VECTOR_ID_MAP_TABLE: &str = "cluster_centroid_vector_id_map";
+pub const PET_FACES_TABLE: &str = "pet_faces";
+pub const PET_BODIES_TABLE: &str = "pet_bodies";
+pub const PET_FACE_VECTOR_ID_MAP_TABLE: &str = "pet_face_vector_id_map";
+pub const PET_BODY_VECTOR_ID_MAP_TABLE: &str = "pet_body_vector_id_map";
+
+pub const ALL_TABLES: [&str; 14] = [
+ FACES_TABLE,
+ FACE_CLUSTERS_TABLE,
+ CLUSTER_PERSON_TABLE,
+ CLUSTER_SUMMARY_TABLE,
+ NOT_PERSON_FEEDBACK_TABLE,
+ CLIP_TABLE,
+ FILEDATA_TABLE,
+ FACE_CACHE_TABLE,
+ TEXT_EMBEDDINGS_CACHE_TABLE,
+ CLUSTER_CENTROID_VECTOR_ID_MAP_TABLE,
+ PET_FACES_TABLE,
+ PET_BODIES_TABLE,
+ PET_FACE_VECTOR_ID_MAP_TABLE,
+ PET_BODY_VECTOR_ID_MAP_TABLE,
+];
+
+pub const CREATE_FACES_TABLE: &str = "CREATE TABLE IF NOT EXISTS faces (
+ file_id INTEGER NOT NULL,
+ face_id TEXT NOT NULL UNIQUE,
+ detection TEXT NOT NULL,
+ embedding BLOB NOT NULL,
+ score REAL NOT NULL,
+ blur REAL NOT NULL DEFAULT 10000.0,
+ is_sideways INTEGER NOT NULL DEFAULT 0,
+ height INTEGER NOT NULL DEFAULT 0,
+ width INTEGER NOT NULL DEFAULT 0,
+ ml_version INTEGER NOT NULL DEFAULT -1,
+ PRIMARY KEY(file_id, face_id)
+);";
+
+pub const CREATE_FACE_CLUSTERS_TABLE: &str = "CREATE TABLE IF NOT EXISTS face_clusters (
+ face_id TEXT NOT NULL,
+ cluster_id TEXT NOT NULL,
+ PRIMARY KEY(face_id)
+);";
+
+pub const FC_CLUSTER_ID_INDEX: &str =
+ "CREATE INDEX IF NOT EXISTS idx_fcClusterID ON face_clusters(cluster_id);";
+
+pub const CREATE_CLUSTER_PERSON_TABLE: &str = "CREATE TABLE IF NOT EXISTS cluster_person (
+ person_id TEXT NOT NULL,
+ cluster_id TEXT NOT NULL,
+ PRIMARY KEY(person_id, cluster_id)
+);";
+
+pub const CREATE_CLUSTER_SUMMARY_TABLE: &str = "CREATE TABLE IF NOT EXISTS cluster_summary (
+ cluster_id TEXT NOT NULL,
+ avg BLOB NOT NULL,
+ count INTEGER NOT NULL,
+ PRIMARY KEY(cluster_id)
+);";
+
+pub const CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE: &str =
+ "CREATE TABLE IF NOT EXISTS cluster_centroid_vector_id_map (
+ cluster_vector_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+ cluster_id TEXT NOT NULL UNIQUE
+);";
+
+pub const CREATE_NOT_PERSON_FEEDBACK_TABLE: &str =
+ "CREATE TABLE IF NOT EXISTS not_person_feedback (
+ person_id TEXT NOT NULL,
+ cluster_id TEXT NOT NULL,
+ PRIMARY KEY(person_id, cluster_id)
+);";
+
+pub const CREATE_CLIP_EMBEDDINGS_TABLE: &str = "CREATE TABLE IF NOT EXISTS clip (
+ file_id INTEGER NOT NULL,
+ embedding BLOB NOT NULL,
+ ml_version INTEGER NOT NULL,
+ PRIMARY KEY (file_id)
+);";
+
+pub const CREATE_FILE_DATA_TABLE: &str = "CREATE TABLE IF NOT EXISTS filedata (
+ file_id INTEGER NOT NULL,
+ user_id INTEGER NOT NULL,
+ type TEXT NOT NULL,
+ size INTEGER NOT NULL,
+ obj_id TEXT,
+ obj_nonce TEXT,
+ updated_at INTEGER NOT NULL,
+ PRIMARY KEY (file_id, type)
+);";
+
+pub const CREATE_FACE_CACHE_TABLE: &str = "CREATE TABLE IF NOT EXISTS face_cache (
+ person_or_cluster_id TEXT NOT NULL UNIQUE,
+ face_id TEXT NOT NULL UNIQUE,
+ PRIMARY KEY (person_or_cluster_id)
+);";
+
+pub const CREATE_TEXT_EMBEDDINGS_CACHE_TABLE: &str =
+ "CREATE TABLE IF NOT EXISTS text_embeddings_cache (
+ text_query TEXT NOT NULL,
+ embedding BLOB NOT NULL,
+ ml_version INTEGER NOT NULL,
+ created_at INTEGER NOT NULL,
+ PRIMARY KEY (text_query)
+);";
+
+pub const CREATE_PET_FACES_TABLE: &str = "CREATE TABLE IF NOT EXISTS pet_faces (
+ file_id INTEGER NOT NULL,
+ pet_face_id TEXT NOT NULL UNIQUE,
+ detection TEXT NOT NULL,
+ face_vector_id INTEGER UNIQUE,
+ species INTEGER NOT NULL,
+ score REAL NOT NULL,
+ height INTEGER NOT NULL DEFAULT 0,
+ width INTEGER NOT NULL DEFAULT 0,
+ ml_version INTEGER NOT NULL DEFAULT -1,
+ PRIMARY KEY(file_id, pet_face_id)
+);";
+
+pub const CREATE_PET_BODIES_TABLE: &str = "CREATE TABLE IF NOT EXISTS pet_bodies (
+ file_id INTEGER NOT NULL,
+ pet_body_id TEXT NOT NULL UNIQUE,
+ detection TEXT NOT NULL,
+ body_vector_id INTEGER UNIQUE,
+ species INTEGER NOT NULL,
+ score REAL NOT NULL,
+ height INTEGER NOT NULL DEFAULT 0,
+ width INTEGER NOT NULL DEFAULT 0,
+ ml_version INTEGER NOT NULL DEFAULT -1,
+ PRIMARY KEY(file_id, pet_body_id)
+);";
+
+pub const CREATE_PET_FACE_VECTOR_ID_MAPPING_TABLE: &str =
+ "CREATE TABLE IF NOT EXISTS pet_face_vector_id_map (
+ pet_face_vector_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+ pet_face_id TEXT NOT NULL UNIQUE
+);";
+
+pub const CREATE_PET_BODY_VECTOR_ID_MAPPING_TABLE: &str =
+ "CREATE TABLE IF NOT EXISTS pet_body_vector_id_map (
+ pet_body_vector_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+ pet_body_id TEXT NOT NULL UNIQUE
+);";
+
+pub const MIGRATION_SCRIPTS: [&str; 15] = [
+ CREATE_FACES_TABLE,
+ CREATE_FACE_CLUSTERS_TABLE,
+ CREATE_CLUSTER_PERSON_TABLE,
+ CREATE_CLUSTER_SUMMARY_TABLE,
+ CREATE_NOT_PERSON_FEEDBACK_TABLE,
+ FC_CLUSTER_ID_INDEX,
+ CREATE_CLIP_EMBEDDINGS_TABLE,
+ CREATE_FILE_DATA_TABLE,
+ CREATE_FACE_CACHE_TABLE,
+ CREATE_TEXT_EMBEDDINGS_CACHE_TABLE,
+ CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE,
+ CREATE_PET_FACES_TABLE,
+ CREATE_PET_BODIES_TABLE,
+ CREATE_PET_FACE_VECTOR_ID_MAPPING_TABLE,
+ CREATE_PET_BODY_VECTOR_ID_MAPPING_TABLE,
+];
+
+pub const DELETE_FACES: &str = "DELETE FROM faces";
+pub const DELETE_FACE_CLUSTERS: &str = "DELETE FROM face_clusters";
+pub const DELETE_CLUSTER_PERSON: &str = "DELETE FROM cluster_person";
+pub const DELETE_CLUSTER_SUMMARY: &str = "DELETE FROM cluster_summary";
+pub const DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING: &str =
+ "DELETE FROM cluster_centroid_vector_id_map";
+pub const DELETE_NOT_PERSON_FEEDBACK: &str = "DELETE FROM not_person_feedback";
+pub const DELETE_CLIP_EMBEDDINGS: &str = "DELETE FROM clip";
+pub const DELETE_FILE_DATA: &str = "DELETE FROM filedata";
+pub const DELETE_PET_FACES: &str = "DELETE FROM pet_faces";
+pub const DELETE_PET_BODIES: &str = "DELETE FROM pet_bodies";
+pub const DELETE_PET_FACE_VECTOR_ID_MAPPING: &str = "DELETE FROM pet_face_vector_id_map";
+pub const DELETE_PET_BODY_VECTOR_ID_MAPPING: &str = "DELETE FROM pet_body_vector_id_map";
diff --git a/rust/crates/photos/src/ml_db/types.rs b/rust/crates/photos/src/ml_db/types.rs
new file mode 100644
index 0000000000..01671e432b
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/types.rs
@@ -0,0 +1,126 @@
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceRow {
+ pub file_id: i64,
+ pub face_id: String,
+ pub detection_json: String,
+ pub embedding: Vec<f64>,
+ pub score: f64,
+ pub blur: f64,
+ pub is_sideways: bool,
+ pub image_height: i64,
+ pub image_width: i64,
+ pub ml_version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceWithoutEmbedding {
+ pub face_id: String,
+ pub file_id: i64,
+ pub score: f64,
+ pub detection_json: String,
+ pub blur: f64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceDbInfoForClustering {
+ pub face_id: String,
+ pub cluster_id: Option<String>,
+ pub embedding_bytes: Vec<u8>,
+ pub face_score: f64,
+ pub blur_value: f64,
+ pub is_sideways: bool,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetFaceRow {
+ pub file_id: i64,
+ pub pet_face_id: String,
+ pub detection_json: String,
+ pub face_vector_id: Option<i64>,
+ pub species: i64,
+ pub face_score: f64,
+ pub image_height: i64,
+ pub image_width: i64,
+ pub ml_version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetBodyRow {
+ pub file_id: i64,
+ pub pet_body_id: String,
+ pub detection_json: String,
+ pub body_vector_id: Option<i64>,
+ pub species: i64,
+ pub score: f64,
+ pub image_height: i64,
+ pub image_width: i64,
+ pub ml_version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetFaceVectorRow {
+ pub pet_face_id: String,
+ pub face_vector_id: Option<i64>,
+ pub species: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetBodyVectorRow {
+ pub pet_body_id: String,
+ pub body_vector_id: Option<i64>,
+ pub species: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetRowsForFiles {
+ pub faces: Vec<PetFaceVectorRow>,
+ pub bodies: Vec<PetBodyVectorRow>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ClipEmbedding {
+ pub file_id: i64,
+ pub embedding: Vec<f64>,
+ pub version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct EmbeddingVector {
+ pub file_id: i64,
+ pub embedding: Vec<f32>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ClipRow {
+ pub file_id: i64,
+ pub embedding: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClusterSummary {
+ pub avg: Vec<u8>,
+ pub count: i64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClusterCentroidRow {
+ pub cluster_id: String,
+ pub avg: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct FdStatus {
+ pub file_id: i64,
+ pub user_id: i64,
+ pub data_type: String,
+ pub size: i64,
+ pub object_id: Option<String>,
+ pub object_nonce: Option<String>,
+ pub updated_at: i64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct PreviewInfo {
+ pub object_id: String,
+ pub object_size: i64,
+}
diff --git a/rust/crates/photos/tests/ml_db_test.rs b/rust/crates/photos/tests/ml_db_test.rs
new file mode 100644
index 0000000000..8c8f95569d
--- /dev/null
+++ b/rust/crates/photos/tests/ml_db_test.rs
@@ -0,0 +1,384 @@
+use std::collections::{HashMap, HashSet};
+use std::fmt::Debug;
+use std::path::Path;
+
+use ente_photos::db::Connection;
+use ente_photos::ml_db::codec::encode_f32;
+use ente_photos::ml_db::constants::CLIP_EMBEDDING_DIMENSIONS;
+use ente_photos::ml_db::schema::ALL_TABLES;
+use ente_photos::ml_db::{
+ ClipEmbedding, ClipRow, Error, FdStatus, MlDb, PreviewInfo, TARGET_VERSION,
+};
+use tempfile::TempDir;
+
+type Query<T> = fn(&MlDb) -> Result<T, Error>;
+
+macro_rules! cases {
+ ($($name:literal: $expected:expr => |$db:ident| $query:expr),* $(,)?) => {
+ [$(($name, $expected, |$db| $query)),*]
+ };
+}
+
+fn open() -> (TempDir, MlDb) {
+ let directory = tempfile::tempdir().unwrap();
+ let db = MlDb::open(directory.path().join("ente.ml.db")).unwrap();
+ (directory, db)
+}
+
+fn user_version(path: &Path) -> i64 {
+ Connection::open(path)
+ .unwrap()
+ .pragma_query_value(None, "user_version", |row| row.get(0))
+ .unwrap()
+}
+
+fn index_count(path: &Path) -> i64 {
+ Connection::open(path)
+ .unwrap()
+ .query_row(
+ "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_fcClusterID'",
+ (),
+ |row| row.get(0),
+ )
+ .unwrap()
+}
+
+fn clip(file_id: i64, embedding: Vec<f64>) -> ClipEmbedding {
+ ClipEmbedding {
+ file_id,
+ embedding,
+ version: 1,
+ }
+}
+
+fn full_clip(file_id: i64) -> ClipEmbedding {
+ clip(file_id, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])
+}
+
+fn status(file_id: i64, data_type: &str, object_id: Option<&str>) -> FdStatus {
+ FdStatus {
+ file_id,
+ user_id: 42,
+ data_type: data_type.to_string(),
+ size: file_id * 10,
+ object_id: object_id.map(str::to_string),
+ object_nonce: None,
+ updated_at: 1000,
+ }
+}
+
+fn preview(object_id: &str, object_size: i64) -> PreviewInfo {
+ PreviewInfo {
+ object_id: object_id.to_string(),
+ object_size,
+ }
+}
+
+fn ids<const N: usize>(values: [i64; N]) -> HashSet<i64> {
+ HashSet::from(values)
+}
+
+fn check_seeded<T: PartialEq + Debug>(cases: &[(&str, T, Query<T>)]) {
+ let (_directory, db) = seeded();
+ for (name, expected, query) in cases {
+ assert_eq!(&query(&db).unwrap(), expected, "{name}");
+ }
+}
+
+fn seeded() -> (TempDir, MlDb) {
+ let (directory, db) = open();
+ db.insert_clip_rows(&[
+ full_clip(1),
+ ClipEmbedding {
+ version: 2,
+ ..full_clip(2)
+ },
+ clip(3, vec![1.0, 2.0]),
+ clip(4, vec![]),
+ ])
+ .unwrap();
+ db.put_fd_status(&[
+ status(1, "vid_preview", Some("obj1")),
+ status(2, "mldata", None),
+ status(3, "vid_preview", Some("obj3")),
+ ])
+ .unwrap();
+ db.put_repeated_text_embedding_cache("dog", &[0.5, -1.0])
+ .unwrap();
+ db.put_face_id_cached_for_person_or_cluster("p1", "1_0")
+ .unwrap();
+ (directory, db)
+}
+
+#[test]
+fn ml_db_is_send_and_sync() {
+ fn assert_send_sync<T: Send + Sync>() {}
+ assert_send_sync::<MlDb>();
+}
+
+#[test]
+fn open_migrates_to_target_version_and_creates_all_tables() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("ente.ml.db");
+ let _db = MlDb::open(&path).unwrap();
+ assert_eq!(TARGET_VERSION, 15);
+ assert_eq!(user_version(&path), 15);
+ let connection = Connection::open(&path).unwrap();
+ for table in ALL_TABLES {
+ let count: i64 = connection
+ .query_row(
+ "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?",
+ [table],
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(count, 1, "missing table {table}");
+ }
+ assert_eq!(index_count(&path), 1);
+}
+
+#[test]
+fn reopen_is_idempotent_and_keeps_data() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("ente.ml.db");
+ {
+ let db = MlDb::open(&path).unwrap();
+ db.insert_clip_rows(&[full_clip(1)]).unwrap();
+ }
+ let db = MlDb::open(&path).unwrap();
+ assert_eq!(user_version(&path), 15);
+ assert_eq!(db.count_clip_rows().unwrap(), 1);
+}
+
+#[test]
+fn open_refuses_downgrade() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("ente.ml.db");
+ drop(MlDb::open(&path).unwrap());
+ Connection::open(&path)
+ .unwrap()
+ .pragma_update(None, "user_version", 16)
+ .unwrap();
+ match MlDb::open(&path) {
+ Err(Error::Downgrade { current, target }) => {
+ assert_eq!(current, 16);
+ assert_eq!(target, 15);
+ }
+ other => panic!("expected downgrade error, got {:?}", other.err()),
+ }
+}
+
+#[test]
+fn partial_migration_runs_remaining_scripts() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("ente.ml.db");
+ {
+ let connection = Connection::open(&path).unwrap();
+ connection
+ .execute_batch(ente_photos::ml_db::schema::CREATE_FACES_TABLE)
+ .unwrap();
+ connection.pragma_update(None, "user_version", 1).unwrap();
+ }
+ let db = MlDb::open(&path).unwrap();
+ assert_eq!(user_version(&path), 15);
+ assert_eq!(db.count_clip_rows().unwrap(), 0);
+}
+
+#[test]
+fn seeded_counts() {
+ check_seeded(&cases![
+ "clip files": 4 => |db| db.get_clip_indexed_file_count(1),
+ "clip files v2": 1 => |db| db.get_clip_indexed_file_count(2),
+ "clip vectorizable": 2 => |db| db.get_clip_vectorizable_file_count(1),
+ "clip vectorizable v2": 1 => |db| db.get_clip_vectorizable_file_count(2),
+ "clip rows": 4 => |db| db.count_clip_rows(),
+ ]);
+}
+
+#[test]
+fn seeded_file_id_sets() {
+ check_seeded(&cases![
+ "fd data": ids([1, 2, 3]) => |db| db.get_file_ids_with_fd_data(None),
+ "fd mldata": ids([2]) => |db| db.get_file_ids_with_fd_data(Some("mldata")),
+ ]);
+}
+
+#[test]
+fn seeded_optional_values() {
+ check_seeded(&cases![
+ "cached face p1": Some("1_0".to_string()) =>
+ |db| db.get_face_id_used_for_person_or_cluster("p1"),
+ "cached face p2": None => |db| db.get_face_id_used_for_person_or_cluster("p2"),
+ ]);
+ check_seeded(&cases![
+ "cached dog": Some(vec![0.5, -1.0]) => |db| db.get_repeated_text_embedding_cache("dog"),
+ "uncached cat": None => |db| db.get_repeated_text_embedding_cache("cat"),
+ ]);
+}
+
+#[test]
+fn seeded_maps() {
+ check_seeded(&cases![
+ "clip versions": HashMap::from([(1, 1), (2, 2), (3, 1), (4, 1)]) =>
+ |db| db.clip_indexed_file_with_version(),
+ ]);
+ check_seeded(&cases![
+ "video previews": HashMap::from([(1, preview("obj1", 10)), (3, preview("obj3", 30))]) =>
+ |db| db.get_file_ids_vid_preview(),
+ ]);
+}
+
+#[test]
+fn seeded_rows() {
+ let (_directory, db) = seeded();
+ let mut vectors = db.get_all_clip_vectors().unwrap();
+ vectors.sort_by_key(|vector| vector.file_id);
+ let dimensions: Vec<(i64, usize)> = vectors
+ .iter()
+ .map(|v| (v.file_id, v.embedding.len()))
+ .collect();
+ assert_eq!(
+ dimensions,
+ vec![
+ (1, CLIP_EMBEDDING_DIMENSIONS),
+ (2, CLIP_EMBEDDING_DIMENSIONS),
+ (3, 2)
+ ]
+ );
+ assert_eq!(vectors[2].embedding, vec![1.0f32, 2.0]);
+ let page = db.get_clip_rows_page(2, 1).unwrap();
+ let clip_row = |file_id, embedding: Vec<f32>| ClipRow {
+ file_id,
+ embedding: encode_f32(embedding),
+ };
+ assert_eq!(
+ page,
+ vec![
+ clip_row(3, vec![1.0, 2.0]),
+ clip_row(2, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])
+ ]
+ );
+}
+
+#[test]
+fn clip_rows_replace_and_delete() {
+ let (_directory, db) = seeded();
+ db.insert_clip_rows(&[]).unwrap();
+ db.insert_clip_rows(&[clip(1, vec![1.0])]).unwrap();
+ db.insert_clip_rows(&[clip(3, vec![3.0]), full_clip(5)])
+ .unwrap();
+ assert_eq!(
+ db.clip_indexed_file_with_version().unwrap(),
+ HashMap::from([(1, 1), (2, 2), (3, 1), (4, 1), (5, 1)])
+ );
+ let mut vectors = db.get_all_clip_vectors().unwrap();
+ vectors.sort_by_key(|vector| vector.file_id);
+ assert_eq!(vectors[0].embedding, vec![1.0f32]);
+ assert_eq!(vectors[2].embedding, vec![3.0f32]);
+ assert_eq!(db.get_clip_vectorizable_file_count(1).unwrap(), 2);
+
+ db.delete_clip_rows(&[]).unwrap();
+ db.delete_clip_rows(&[1, 3]).unwrap();
+ assert_eq!(db.count_clip_rows().unwrap(), 3);
+ db.delete_all_clip_rows().unwrap();
+ assert_eq!(db.count_clip_rows().unwrap(), 0);
+}
+
+#[test]
+fn filedata_upsert() {
+ let (_directory, db) = seeded();
+ db.put_fd_status(&[]).unwrap();
+ db.put_fd_status(&[status(1, "vid_preview", Some("obj1b"))])
+ .unwrap();
+ assert_eq!(
+ db.get_file_ids_vid_preview().unwrap(),
+ HashMap::from([(1, preview("obj1b", 10)), (3, preview("obj3", 30))])
+ );
+ assert_eq!(db.get_file_ids_with_fd_data(None).unwrap(), ids([1, 2, 3]));
+}
+
+#[test]
+fn text_embedding_cache_replaces_and_expires() {
+ let (directory, db) = seeded();
+ db.put_repeated_text_embedding_cache("dog", &[2.0]).unwrap();
+ assert_eq!(
+ db.get_repeated_text_embedding_cache("dog").unwrap(),
+ Some(vec![2.0f32])
+ );
+
+ let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();
+ connection
+ .execute(
+ "UPDATE text_embeddings_cache SET created_at = 1 WHERE text_query = ?",
+ ["dog"],
+ )
+ .unwrap();
+ assert_eq!(db.get_repeated_text_embedding_cache("dog").unwrap(), None);
+ let remaining: i64 = connection
+ .query_row("SELECT COUNT(*) FROM text_embeddings_cache", (), |row| {
+ row.get(0)
+ })
+ .unwrap();
+ assert_eq!(remaining, 0);
+}
+
+#[test]
+fn face_cache_replaces_and_removes() {
+ let (_directory, db) = seeded();
+ db.put_face_id_cached_for_person_or_cluster("p1", "2_0")
+ .unwrap();
+ assert_eq!(
+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
+ Some("2_0".to_string())
+ );
+ db.remove_face_id_cached_for_person_or_cluster("p1")
+ .unwrap();
+ assert_eq!(
+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
+ None
+ );
+}
+
+#[test]
+fn clear_non_pet_tables_leaves_pets_and_caches() {
+ let (_directory, db) = seeded();
+ db.clear_non_pet_tables().unwrap();
+ assert_eq!(db.count_clip_rows().unwrap(), 0);
+ assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());
+ assert_eq!(
+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
+ Some("1_0".to_string())
+ );
+ assert_eq!(
+ db.get_repeated_text_embedding_cache("dog").unwrap(),
+ Some(vec![0.5f32, -1.0])
+ );
+}
+
+#[test]
+fn clear_pet_tables_leaves_non_pet_tables() {
+ let (directory, db) = seeded();
+ let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();
+ connection
+ .execute_batch(
+ "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');
+ INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",
+ )
+ .unwrap();
+ db.clear_pet_tables().unwrap();
+ let row_count = |table: &str| -> i64 {
+ connection
+ .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {
+ row.get(0)
+ })
+ .unwrap()
+ };
+ assert_eq!(row_count("pet_face_vector_id_map"), 0);
+ assert_eq!(row_count("pet_body_vector_id_map"), 0);
+ assert_eq!(db.count_clip_rows().unwrap(), 4);
+ assert_eq!(db.get_file_ids_with_fd_data(None).unwrap(), ids([1, 2, 3]));
+ assert_eq!(
+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
+ Some("1_0".to_string())
+ );
+}