Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Port ML DB queries to Rust

laurenspriem1dc6d3a692ab
diff --git a/rust/crates/photos/src/db.rs b/rust/crates/photos/src/db.rs
index 71cb216736..dbb99444eb 100644
--- a/rust/crates/photos/src/db.rs
+++ b/rust/crates/photos/src/db.rs
@@ -1,21 +1,23 @@
+use std::collections::HashMap;
+use std::hash::Hash;
use std::ops::{Deref, DerefMut};
use std::path::Path;
use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
-use rusqlite::{OpenFlags, TransactionBehavior, params_from_iter, types::FromSql};
+use rusqlite::{OpenFlags, TransactionBehavior, types::FromSql};
pub use rusqlite::types;
pub use rusqlite::{
Error as SqliteError, OptionalExtension, Params, Result as SqliteResult, Row, ToSql,
- Transaction,
+ Transaction, params_from_iter,
};
const WRITER_PRAGMAS: &str = "PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
PRAGMA journal_size_limit = 6291456;";
const BUSY_TIMEOUT: Duration = Duration::from_secs(30);
-const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;
+pub(crate) const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;
#[derive(Debug, thiserror::Error)]
pub enum Error {
@@ -127,6 +129,15 @@ impl Database {
self.read_all(sql, parameters, |row| row.get(0))
}
+ pub fn read_grouped<K: FromSql + Eq + Hash, V: FromSql, C: FromIterator<V>, P: Params>(
+ &self,
+ sql: &str,
+ parameters: P,
+ ) -> Result<HashMap<K, C>> {
+ let rows: Vec<(K, V)> = self.read_all(sql, parameters, pair)?;
+ Ok(group_into(rows))
+ }
+
pub fn read_value<T: FromSql>(&self, sql: &str, parameters: impl Params) -> Result<T> {
self.pool.read(|connection| {
Ok(connection
@@ -148,6 +159,22 @@ impl Database {
})
}
+ pub fn read_chunked_in<C: FromIterator<T>, T, I: ToSql>(
+ &self,
+ sql: &str,
+ ids: &[I],
+ chunk_size: usize,
+ mut map: impl FnMut(&Row<'_>) -> SqliteResult<T>,
+ ) -> Result<C> {
+ let mut rows = Vec::new();
+ for chunk in ids.chunks(chunk_size) {
+ let sql = expand_in_clause(sql, chunk.len());
+ let chunk_rows: Vec<T> = self.read_all(&sql, params_from_iter(chunk), &mut map)?;
+ rows.extend(chunk_rows);
+ }
+ Ok(rows.into_iter().collect())
+ }
+
pub fn read_transaction<T>(
&self,
read: impl FnOnce(&Transaction<'_>) -> Result<T>,
@@ -185,7 +212,7 @@ impl Database {
}
self.write_transaction(|transaction| {
for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
- let sql = sql.replacen("{}", &bind_placeholders(chunk.len()), 1);
+ let sql = expand_in_clause(sql, chunk.len());
transaction.execute(&sql, params_from_iter(chunk))?;
}
Ok(())
@@ -217,6 +244,29 @@ impl Database {
Ok(())
})
}
+
+ pub fn write_in_batches<P: Params>(
+ &self,
+ sql: &str,
+ batch_size: usize,
+ parameter_sets: impl IntoIterator<Item = P>,
+ ) -> Result<()> {
+ self.pool.write(|connection| {
+ let mut parameter_sets = parameter_sets.into_iter().peekable();
+ while parameter_sets.peek().is_some() {
+ let transaction =
+ connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ {
+ let mut statement = transaction.prepare_cached(sql)?;
+ for parameters in parameter_sets.by_ref().take(batch_size) {
+ statement.execute(parameters)?;
+ }
+ }
+ transaction.commit()?;
+ }
+ Ok(())
+ })
+ }
}
struct Pool {
@@ -311,6 +361,10 @@ pub fn bind_placeholders(count: usize) -> String {
vec!["?"; count].join(", ")
}
+fn expand_in_clause(sql: &str, count: usize) -> String {
+ sql.replacen("{}", &bind_placeholders(count), 1)
+}
+
pub(crate) fn pair<A: types::FromSql, B: types::FromSql>(row: &Row<'_>) -> SqliteResult<(A, B)> {
Ok((row.get(0)?, row.get(1)?))
}
@@ -319,14 +373,31 @@ pub(crate) fn optional_parameter(value: &Option<impl ToSql>) -> Vec<&dyn ToSql>
value.iter().map(|value| value as &dyn ToSql).collect()
}
+pub(crate) fn group_into<K: Eq + Hash, V, C: FromIterator<V>>(
+ rows: impl IntoIterator<Item = (K, V)>,
+) -> HashMap<K, C> {
+ let mut groups: HashMap<K, Vec<V>> = HashMap::new();
+ for (key, value) in rows {
+ groups.entry(key).or_default().push(value);
+ }
+ groups
+ .into_iter()
+ .map(|(key, values)| (key, values.into_iter().collect()))
+ .collect()
+}
+
#[cfg(test)]
mod tests {
+ use std::collections::{HashMap, HashSet};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::mpsc::{self, RecvTimeoutError};
use std::thread;
use std::time::Duration;
- use super::{Connection, Database, Error, OpenOptions, bind_placeholders, lock, types};
+ use super::{
+ Connection, Database, Error, MAX_SQL_BIND_PARAMS_PER_QUERY, OpenOptions, bind_placeholders,
+ lock, types,
+ };
const CREATE_ITEMS: &str = "CREATE TABLE items (id INTEGER PRIMARY KEY)";
const ADD_LABEL: &str = "ALTER TABLE items ADD COLUMN label TEXT NOT NULL DEFAULT 'item'";
@@ -643,6 +714,126 @@ mod tests {
);
}
+ #[test]
+ fn batched_writes_commit_each_completed_batch() {
+ let (_directory, db) = open();
+ db.write_in_batches(
+ "INSERT INTO missing_table VALUES (?)",
+ 2,
+ Vec::<[i64; 1]>::new(),
+ )
+ .unwrap();
+ assert!(
+ db.write_in_batches(
+ "INSERT INTO items (id) VALUES (?)",
+ 2,
+ [[1], [2], [3], [4], [1]]
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ [1, 2, 3, 4]
+ );
+ assert!(
+ db.write_in_batches(
+ "INSERT INTO items (id) VALUES (?)",
+ 2,
+ [[5], [6], [7], [5], [8]]
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ [1, 2, 3, 4, 5, 6]
+ );
+ db.write_in_batches("INSERT INTO items (id) VALUES (?)", 2, [[7], [8], [9]])
+ .unwrap();
+ assert_eq!(
+ db.read_column::<Vec<i64>, _, _>("SELECT id FROM items ORDER BY id", ())
+ .unwrap(),
+ [1, 2, 3, 4, 5, 6, 7, 8, 9]
+ );
+ }
+
+ #[test]
+ fn chunked_reads_query_each_chunk_in_order() {
+ let (_directory, db) = open();
+ db.write_batch("INSERT INTO items (id) VALUES (?)", (1..=7).map(|id| [id]))
+ .unwrap();
+ let ids: Vec<i64> = (1..=7).collect();
+ let by_chunk: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
+ &ids,
+ 3,
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(by_chunk, [3, 2, 1, 6, 5, 4, 7]);
+ let single_chunk: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({}) ORDER BY id DESC",
+ &ids,
+ 10,
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(single_chunk, [7, 6, 5, 4, 3, 2, 1]);
+ let none: Vec<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM missing_table WHERE id IN ({})",
+ &Vec::<i64>::new(),
+ 3,
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert!(none.is_empty());
+ let many: Vec<i64> = (0..=MAX_SQL_BIND_PARAMS_PER_QUERY as i64).collect();
+ let found: HashSet<i64> = db
+ .read_chunked_in(
+ "SELECT id FROM items WHERE id IN ({})",
+ &many,
+ MAX_SQL_BIND_PARAMS_PER_QUERY,
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(found, HashSet::from([1, 2, 3, 4, 5, 6, 7]));
+ }
+
+ #[test]
+ fn grouped_reads_collect_values_per_key() {
+ let (_directory, db) = open();
+ db.write_batch("INSERT INTO items (id) VALUES (?)", (1..=5).map(|id| [id]))
+ .unwrap();
+ let by_parity: HashMap<i64, Vec<i64>> = db
+ .read_grouped("SELECT id % 2, id FROM items ORDER BY id", ())
+ .unwrap();
+ assert_eq!(
+ by_parity,
+ HashMap::from([(1, vec![1, 3, 5]), (0, vec![2, 4])])
+ );
+ let by_label: HashMap<String, HashSet<i64>> = db
+ .read_grouped(
+ "SELECT CASE WHEN id < 3 THEN 'low' ELSE 'high' END, id FROM items",
+ (),
+ )
+ .unwrap();
+ assert_eq!(
+ by_label,
+ HashMap::from([
+ ("low".to_string(), HashSet::from([1, 2])),
+ ("high".to_string(), HashSet::from([3, 4, 5]))
+ ])
+ );
+ let empty: HashMap<i64, Vec<i64>> = db
+ .read_grouped("SELECT id, id FROM items WHERE id > 5", ())
+ .unwrap();
+ assert!(empty.is_empty());
+ }
+
#[test]
fn open_migrates_a_caller_schema_and_refuses_downgrade() {
let directory = tempfile::tempdir().unwrap();
diff --git a/rust/crates/photos/src/ml_db/clusters.rs b/rust/crates/photos/src/ml_db/clusters.rs
new file mode 100644
index 0000000000..ffc7a0de1e
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/clusters.rs
@@ -0,0 +1,708 @@
+use std::collections::{HashMap, HashSet};
+
+use crate::db::{
+ MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, bind_placeholders, group_into,
+ optional_parameter, pair, params_from_iter,
+};
+
+use super::queries::{ClusterCentroidRow, ClusterSummary, is_bad_face_for_clustering};
+use super::schema::{
+ CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE, CREATE_CLUSTER_PERSON_TABLE,
+ CREATE_CLUSTER_SUMMARY_TABLE, CREATE_FACE_CLUSTERS_TABLE, CREATE_FACES_TABLE,
+ CREATE_NOT_PERSON_FEEDBACK_TABLE, DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ DELETE_CLUSTER_PERSON, DELETE_CLUSTER_SUMMARY, DELETE_FACE_CLUSTERS, DELETE_FACES,
+ DELETE_NOT_PERSON_FEEDBACK, FC_CLUSTER_ID_INDEX,
+};
+use super::{MlDb, Result, file_id_to_cluster_ids, unique_in_order};
+
+const UPSERT_FACE_CLUSTER: &str = "INSERT INTO face_clusters (face_id, cluster_id) VALUES (?, ?) ON CONFLICT(face_id) DO UPDATE SET cluster_id = excluded.cluster_id";
+const UPSERT_CLUSTER_SUMMARY: &str = "INSERT INTO cluster_summary (cluster_id, avg, count) VALUES (?, ?, ?) ON CONFLICT(cluster_id) DO UPDATE SET avg = excluded.avg, count = excluded.count";
+
+impl MlDb {
+ pub fn update_face_id_to_cluster_id(
+ &self,
+ face_id_to_cluster_id: &HashMap<String, String>,
+ ) -> Result<()> {
+ self.db
+ .write_in_batches(UPSERT_FACE_CLUSTER, 500, face_id_to_cluster_id.iter())
+ .map_err(Into::into)
+ }
+
+ pub fn cluster_id_to_face_count(&self) -> Result<HashMap<String, i64>> {
+ self.db
+ .read_all(
+ "SELECT cluster_id, COUNT(*) as count FROM face_clusters where cluster_id IS NOT NULL GROUP BY cluster_id",
+ (),
+ pair,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_bad_face_singleton_cluster_ids(&self) -> Result<HashSet<String>> {
+ let rows: Vec<(String, f64, f64, i64)> = self.db.read_all(
+ "SELECT fc.cluster_id, f.score, f.blur, f.is_sideways FROM face_clusters fc INNER JOIN faces f ON fc.face_id = f.face_id GROUP BY fc.cluster_id HAVING COUNT(*) = 1",
+ (),
+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
+ )?;
+ Ok(rows
+ .into_iter()
+ .filter(|(_, score, blur, is_sideways)| {
+ is_bad_face_for_clustering(*score, *blur, *is_sideways == 1)
+ })
+ .map(|(cluster_id, _, _, _)| cluster_id)
+ .collect())
+ }
+
+ pub fn get_cluster_to_face_ids(
+ &self,
+ cluster_ids: &HashSet<String>,
+ ) -> Result<HashMap<String, Vec<String>>> {
+ if cluster_ids.is_empty() {
+ return Ok(HashMap::new());
+ }
+ let cluster_id_list: Vec<&String> = cluster_ids.iter().collect();
+ let rows: Vec<(String, String)> = self.db.read_chunked_in(
+ "SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",
+ &cluster_id_list,
+ MAX_SQL_BIND_PARAMS_PER_QUERY,
+ pair,
+ )?;
+ Ok(group_into(rows))
+ }
+
+ pub fn get_cluster_id_for_face_id(&self, face_id: &str) -> Result<Option<String>> {
+ self.db
+ .read_optional(
+ "SELECT cluster_id FROM face_clusters WHERE face_id = ?",
+ [face_id],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_all_cluster_id_to_face_ids(&self) -> Result<HashMap<String, Vec<String>>> {
+ self.db
+ .read_grouped("SELECT cluster_id, face_id FROM face_clusters", ())
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_ids_for_cluster(&self, cluster_id: &str) -> Result<Vec<String>> {
+ self.db
+ .read_column(
+ "SELECT face_id FROM face_clusters WHERE face_clusters.cluster_id = ?",
+ [cluster_id],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_ids_for_cluster_ordered_by_score(
+ &self,
+ cluster_id: &str,
+ limit: i64,
+ ) -> Result<Vec<String>> {
+ self.db
+ .read_column(
+ "SELECT faces.face_id FROM faces JOIN face_clusters ON faces.face_id = face_clusters.face_id WHERE face_clusters.cluster_id = ? ORDER BY faces.score DESC LIMIT ?",
+ (cluster_id, limit),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_blur_values_for_cluster(&self, cluster_id: &str) -> Result<Vec<f64>> {
+ let blur_values: Vec<f64> = self.db.read_column(
+ "SELECT faces.blur FROM faces JOIN face_clusters ON faces.face_id = face_clusters.face_id WHERE face_clusters.cluster_id = ?",
+ [cluster_id],
+ )?;
+ let mut seen = HashSet::new();
+ Ok(blur_values
+ .into_iter()
+ .filter(|blur| seen.insert(blur.to_bits()))
+ .collect())
+ }
+
+ pub fn get_face_ids_to_cluster_ids(
+ &self,
+ face_ids: &[String],
+ ) -> Result<HashMap<String, Option<String>>> {
+ self.db
+ .read_chunked_in(
+ "SELECT face_id, cluster_id FROM face_clusters where face_id IN ({})",
+ face_ids,
+ MAX_SQL_BIND_PARAMS_PER_QUERY,
+ pair,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_file_id_to_cluster_ids(&self) -> Result<HashMap<i64, HashSet<String>>> {
+ self.db
+ .read_all("SELECT cluster_id, face_id FROM face_clusters", (), pair)
+ .map_err(Into::into)
+ .and_then(file_id_to_cluster_ids)
+ }
+
+ pub fn force_update_cluster_ids(
+ &self,
+ face_id_to_cluster_id: &HashMap<String, String>,
+ ) -> Result<()> {
+ self.db
+ .write_batch(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())
+ .map_err(Into::into)
+ }
+
+ pub fn remove_face_id_to_cluster_id(
+ &self,
+ face_id_to_cluster_id: &HashMap<String, String>,
+ ) -> Result<()> {
+ self.db
+ .write_batch(
+ "DELETE FROM face_clusters WHERE face_id = ? AND cluster_id = ?",
+ face_id_to_cluster_id.iter(),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_file_id_to_cluster_id_set_for_cluster(
+ &self,
+ cluster_ids: &HashSet<String>,
+ ) -> Result<HashMap<i64, HashSet<String>>> {
+ let sql = format!(
+ "SELECT cluster_id, face_id FROM face_clusters WHERE cluster_id IN ({})",
+ bind_placeholders(cluster_ids.len())
+ );
+ self.db
+ .read_all(&sql, params_from_iter(cluster_ids), pair)
+ .map_err(Into::into)
+ .and_then(file_id_to_cluster_ids)
+ }
+
+ pub fn get_file_ids_of_cluster_id(&self, cluster_id: &str) -> Result<Vec<i64>> {
+ self.db
+ .read_column(
+ "SELECT DISTINCT faces.file_id FROM face_clusters JOIN faces ON face_clusters.face_id = faces.face_id WHERE face_clusters.cluster_id = ?",
+ [cluster_id],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_cluster_centroid_vector_id_map(
+ &self,
+ cluster_ids: &[String],
+ create_if_missing: bool,
+ ) -> Result<HashMap<String, i64>> {
+ let unique_cluster_ids = unique_in_order(cluster_ids);
+ if unique_cluster_ids.is_empty() {
+ return Ok(HashMap::new());
+ }
+ if create_if_missing {
+ self.db.write_batch(
+ "INSERT OR IGNORE INTO cluster_centroid_vector_id_map (cluster_id) VALUES (?)",
+ unique_cluster_ids.iter().map(|cluster_id| [cluster_id]),
+ )?;
+ }
+ self.db
+ .read_chunked_in(
+ "SELECT cluster_id, cluster_vector_id FROM cluster_centroid_vector_id_map WHERE cluster_id IN ({})",
+ &unique_cluster_ids,
+ 800,
+ pair,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn delete_cluster_centroid_vector_id_mapping(&self, cluster_id: &str) -> Result<()> {
+ self.db.execute(
+ "DELETE FROM cluster_centroid_vector_id_map WHERE cluster_id = ?",
+ [cluster_id],
+ )?;
+ Ok(())
+ }
+
+ pub fn clear_cluster_centroid_vector_id_mappings(&self) -> Result<()> {
+ self.db
+ .execute_statements([DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING])
+ .map_err(Into::into)
+ }
+
+ pub fn upsert_cluster_summary_rows(
+ &self,
+ summary: &HashMap<String, ClusterSummary>,
+ ) -> Result<()> {
+ self.db
+ .write_in_batches(
+ UPSERT_CLUSTER_SUMMARY,
+ 400,
+ summary
+ .iter()
+ .map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn delete_cluster_summary_row(&self, cluster_id: &str) -> Result<()> {
+ self.db.execute(
+ "DELETE FROM cluster_summary WHERE cluster_id = ?",
+ [cluster_id],
+ )?;
+ Ok(())
+ }
+
+ pub fn get_all_cluster_summary(
+ &self,
+ min_cluster_size: Option<i64>,
+ ) -> Result<HashMap<String, ClusterSummary>> {
+ let sql = format!(
+ "SELECT * FROM cluster_summary{}",
+ if min_cluster_size.is_some() {
+ " WHERE count >= ?"
+ } else {
+ ""
+ }
+ );
+ self.db
+ .read_all(
+ &sql,
+ optional_parameter(&min_cluster_size).as_slice(),
+ read_cluster_summary,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_cluster_to_cluster_summary(
+ &self,
+ cluster_ids: &[String],
+ ) -> Result<HashMap<String, ClusterSummary>> {
+ let sql = format!(
+ "SELECT * FROM cluster_summary WHERE cluster_id IN ({})",
+ bind_placeholders(cluster_ids.len())
+ );
+ self.db
+ .read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary)
+ .map_err(Into::into)
+ }
+
+ pub fn count_cluster_summaries(&self) -> Result<i64> {
+ self.db
+ .read_value("SELECT COUNT(cluster_id) as total FROM cluster_summary", ())
+ .map_err(Into::into)
+ }
+
+ pub fn get_cluster_summary_page(
+ &self,
+ before_cluster_id: Option<&str>,
+ limit: i64,
+ ) -> Result<Vec<ClusterCentroidRow>> {
+ match before_cluster_id {
+ None => self.db.read_all(
+ "SELECT cluster_id, avg FROM cluster_summary ORDER BY cluster_id DESC LIMIT ?",
+ [limit],
+ read_cluster_centroid,
+ ),
+ Some(before_cluster_id) => self.db.read_all(
+ "SELECT cluster_id, avg FROM cluster_summary WHERE cluster_id < ? ORDER BY cluster_id DESC LIMIT ?",
+ (before_cluster_id, limit),
+ read_cluster_centroid,
+ ),
+ }
+ .map_err(Into::into)
+ }
+
+ pub fn reset_cluster_tables(&self, faces: bool) -> Result<()> {
+ let face_statements: &[&str] = if faces {
+ &[
+ DELETE_FACES,
+ CREATE_FACES_TABLE,
+ DELETE_FACE_CLUSTERS,
+ CREATE_FACE_CLUSTERS_TABLE,
+ FC_CLUSTER_ID_INDEX,
+ ]
+ } else {
+ &[]
+ };
+ self.db
+ .execute_statements(face_statements.iter().copied().chain([
+ DELETE_CLUSTER_PERSON,
+ DELETE_NOT_PERSON_FEEDBACK,
+ DELETE_CLUSTER_SUMMARY,
+ DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ DELETE_FACE_CLUSTERS,
+ CREATE_CLUSTER_PERSON_TABLE,
+ CREATE_NOT_PERSON_FEEDBACK_TABLE,
+ CREATE_CLUSTER_SUMMARY_TABLE,
+ CREATE_CLUSTER_CENTROID_VECTOR_ID_MAPPING_TABLE,
+ CREATE_FACE_CLUSTERS_TABLE,
+ FC_CLUSTER_ID_INDEX,
+ ]))
+ .map_err(Into::into)
+ }
+
+ pub fn get_clusters_for_memory_lane(
+ &self,
+ assigned: &HashSet<String>,
+ ) -> Result<HashSet<String>> {
+ let batch_size: i64 = 256;
+ let mut clusters = HashSet::new();
+ let mut offset: i64 = 0;
+ while clusters.len() < 20 {
+ let batch: Vec<String> = self.db.read_column(
+ "SELECT cluster_id, COUNT(*) AS count FROM face_clusters WHERE cluster_id IS NOT NULL GROUP BY cluster_id ORDER BY count DESC, cluster_id LIMIT ? OFFSET ?",
+ [batch_size, offset],
+ )?;
+ let batch_len = batch.len() as i64;
+ for cluster in batch {
+ if !assigned.contains(&cluster) {
+ clusters.insert(cluster);
+ if clusters.len() == 20 {
+ break;
+ }
+ }
+ }
+ if batch_len < batch_size {
+ break;
+ }
+ offset += batch_size;
+ }
+ Ok(clusters)
+ }
+}
+
+fn read_cluster_summary(row: &Row<'_>) -> SqliteResult<(String, ClusterSummary)> {
+ Ok((
+ row.get("cluster_id")?,
+ ClusterSummary {
+ avg: row.get("avg")?,
+ count: row.get("count")?,
+ },
+ ))
+}
+
+fn read_cluster_centroid(row: &Row<'_>) -> SqliteResult<ClusterCentroidRow> {
+ Ok(ClusterCentroidRow {
+ cluster_id: row.get(0)?,
+ avg: row.get(1)?,
+ })
+}
+
+#[cfg(test)]
+pub(super) mod tests {
+ use std::collections::HashMap;
+
+ use super::{ClusterCentroidRow, ClusterSummary, MlDb};
+ use crate::ml_db::tests::{
+ cases, check, grouped, grouped_by_file, index_count, map, open, pairs, set, sorted, strings,
+ };
+ use crate::ml_db::vector_encoding::encode_evector;
+ use crate::ml_db::{faces, persons};
+ use tempfile::TempDir;
+
+ fn summary(count: i64) -> ClusterSummary {
+ ClusterSummary {
+ avg: encode_evector(&[count as f64]),
+ count,
+ }
+ }
+
+ fn summaries<const N: usize>(entries: [(&str, i64); N]) -> HashMap<String, ClusterSummary> {
+ pairs(entries.map(|(key, count)| (key, summary(count))))
+ }
+
+ fn sorted_lists(mut lists: HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
+ lists.values_mut().for_each(|values| values.sort());
+ lists
+ }
+
+ pub(in crate::ml_db) fn seed(db: &MlDb) {
+ db.update_face_id_to_cluster_id(&map([
+ ("1_0", "c1"),
+ ("1_1", "c2"),
+ ("2_0", "c1"),
+ ("3_0", "c3"),
+ ("7_0", "c7"),
+ ("8_0", "c8"),
+ ("9_0", "c9"),
+ ]))
+ .unwrap();
+ db.upsert_cluster_summary_rows(&summaries([("c1", 2), ("c2", 1), ("c3", 1)]))
+ .unwrap();
+ db.get_cluster_centroid_vector_id_map(&strings(["c1", "c2"]), true)
+ .unwrap();
+ }
+
+ fn seeded() -> (TempDir, MlDb) {
+ let (directory, db) = open();
+ faces::tests::seed(&db);
+ seed(&db);
+ persons::tests::seed(&db);
+ (directory, db)
+ }
+
+ #[test]
+ fn seeded_counts() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "cluster summaries": 3 => |db| db.count_cluster_summaries(),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_id_sets() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "bad singletons": set(["c7", "c8", "c9"]) => |db| db.get_bad_face_singleton_cluster_ids(),
+ "memory lane": set(["c2", "c3", "c7", "c8", "c9"]) =>
+ |db| db.get_clusters_for_memory_lane(&set(["c1"])),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_ordered_lists() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "c1 by score": strings(["1_0", "2_0"]) =>
+ |db| db.get_face_ids_for_cluster_ordered_by_score("c1", 10),
+ "c1 by score limited": strings(["1_0"]) =>
+ |db| db.get_face_ids_for_cluster_ordered_by_score("c1", 1),
+ "c1 faces": strings(["1_0", "2_0"]) => |db| db.get_face_ids_for_cluster("c1").map(sorted),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "c1 files": vec![1, 2] => |db| db.get_file_ids_of_cluster_id("c1").map(sorted),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "c1 blur deduplicated": vec![300.0] => |db| db.get_blur_values_for_cluster("c1"),
+ "c9 blur": vec![20.0] => |db| db.get_blur_values_for_cluster("c9"),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_optional_values() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "cluster of 1_1": Some("c2".to_string()) => |db| db.get_cluster_id_for_face_id("1_1"),
+ "cluster of unclustered": None => |db| db.get_cluster_id_for_face_id("4_0"),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_maps() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "cluster sizes": pairs([
+ ("c1", 2), ("c2", 1), ("c3", 1), ("c7", 1), ("c8", 1), ("c9", 1),
+ ]) => |db| db.cluster_id_to_face_count(),
+ "centroid vector ids": pairs([("c1", 1), ("c2", 2)]) =>
+ |db| db.get_cluster_centroid_vector_id_map(&strings(["c1", "c2", "c3"]), false),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "file clusters": grouped_by_file([
+ (1, &["c1", "c2"]), (2, &["c1"]), (3, &["c3"]),
+ (7, &["c7"]), (8, &["c8"]), (9, &["c9"]),
+ ]) => |db| db.get_file_id_to_cluster_ids(),
+ "file clusters for c2 and c9": grouped_by_file([(1, &["c2"]), (9, &["c9"])]) =>
+ |db| db.get_file_id_to_cluster_id_set_for_cluster(&set(["c2", "c9"])),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "all cluster faces": grouped([
+ ("c1", &["1_0", "2_0"]), ("c2", &["1_1"]), ("c3", &["3_0"]),
+ ("c7", &["7_0"]), ("c8", &["8_0"]), ("c9", &["9_0"]),
+ ]) => |db| db.get_all_cluster_id_to_face_ids().map(sorted_lists),
+ "some cluster faces": grouped([("c1", &["1_0", "2_0"]), ("c2", &["1_1"])]) =>
+ |db| db.get_cluster_to_face_ids(&set(["c1", "c2", "missing"])).map(sorted_lists),
+ "no cluster faces": grouped([]) => |db| db.get_cluster_to_face_ids(&set([])),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "all summaries": summaries([("c1", 2), ("c2", 1), ("c3", 1)]) =>
+ |db| db.get_all_cluster_summary(None),
+ "large summaries": summaries([("c1", 2)]) => |db| db.get_all_cluster_summary(Some(2)),
+ "some summaries": summaries([("c1", 2), ("c3", 1)]) =>
+ |db| db.get_cluster_to_cluster_summary(&strings(["c1", "c3", "nope"])),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_face_clusters() {
+ let (_directory, db) = seeded();
+ assert_eq!(
+ db.get_face_ids_to_cluster_ids(&strings(["1_0", "1_1", "4_0"]))
+ .unwrap(),
+ pairs([
+ ("1_0", Some("c1".to_string())),
+ ("1_1", Some("c2".to_string()))
+ ])
+ );
+ assert!(db.get_face_ids_to_cluster_ids(&[]).unwrap().is_empty());
+ }
+
+ #[test]
+ fn cluster_assignment_upserts_and_removals() {
+ let (_directory, db) = seeded();
+ db.update_face_id_to_cluster_id(&map([("1_1", "c5"), ("4_0", "c5")]))
+ .unwrap();
+ assert_eq!(
+ db.get_cluster_id_for_face_id("1_1").unwrap(),
+ Some("c5".to_string())
+ );
+ db.force_update_cluster_ids(&map([("1_0", "c9"), ("5_0", "c9")]))
+ .unwrap();
+ assert_eq!(
+ db.get_cluster_id_for_face_id("1_0").unwrap(),
+ Some("c9".to_string())
+ );
+ assert_eq!(
+ db.get_cluster_id_for_face_id("5_0").unwrap(),
+ Some("c9".to_string())
+ );
+ db.remove_face_id_to_cluster_id(&map([("2_0", "c1"), ("5_0", "c1")]))
+ .unwrap();
+ assert_eq!(db.get_cluster_id_for_face_id("2_0").unwrap(), None);
+ assert_eq!(
+ db.get_cluster_id_for_face_id("5_0").unwrap(),
+ Some("c9".to_string())
+ );
+ let sizes = db.cluster_id_to_face_count().unwrap();
+ assert_eq!((sizes.get("c1"), sizes["c5"], sizes["c9"]), (None, 2, 3));
+ }
+
+ #[test]
+ fn reset_cluster_tables_keeps_or_drops_faces() {
+ let (directory, db) = seeded();
+ db.reset_cluster_tables(false).unwrap();
+ assert_eq!(db.get_total_face_count().unwrap(), 9);
+ assert!(db.cluster_id_to_face_count().unwrap().is_empty());
+ assert!(db.get_person_to_cluster_ids().unwrap().is_empty());
+ assert!(db.get_person_to_rejected_suggestions().unwrap().is_empty());
+ assert_eq!(db.count_cluster_summaries().unwrap(), 0);
+ assert!(
+ db.get_cluster_centroid_vector_id_map(&strings(["c1"]), false)
+ .unwrap()
+ .is_empty()
+ );
+
+ db.reset_cluster_tables(true).unwrap();
+ assert_eq!(db.get_total_face_count().unwrap(), 0);
+ assert_eq!(index_count(&directory.path().join("ente.ml.db")), 1);
+ }
+
+ #[test]
+ fn cluster_summary_upsert_and_keyset_paging() {
+ let (_directory, db) = open();
+ let mut rows = HashMap::new();
+ for index in 0..1000 {
+ rows.insert(format!("c{index:04}"), summary(index));
+ }
+ db.upsert_cluster_summary_rows(&rows).unwrap();
+ db.upsert_cluster_summary_rows(&summaries([("c0001", 77)]))
+ .unwrap();
+
+ assert_eq!(db.count_cluster_summaries().unwrap(), 1000);
+ let all = db.get_all_cluster_summary(None).unwrap();
+ assert_eq!(all.len(), 1000);
+ assert_eq!(all["c0001"], summary(77));
+ assert_eq!(db.get_all_cluster_summary(Some(998)).unwrap().len(), 2);
+ assert_eq!(
+ db.get_cluster_to_cluster_summary(&strings(["c0002", "c0003", "nope"]))
+ .unwrap(),
+ summaries([("c0002", 2), ("c0003", 3)])
+ );
+
+ let first_page = db.get_cluster_summary_page(None, 2).unwrap();
+ let ids = |page: &[ClusterCentroidRow]| -> Vec<String> {
+ page.iter().map(|row| row.cluster_id.clone()).collect()
+ };
+ assert_eq!(ids(&first_page), strings(["c0999", "c0998"]));
+ assert_eq!(first_page[0].avg, encode_evector(&[999.0]));
+ let next_page = db.get_cluster_summary_page(Some("c0998"), 2).unwrap();
+ assert_eq!(ids(&next_page), strings(["c0997", "c0996"]));
+ assert!(
+ db.get_cluster_summary_page(Some("c0000"), 2)
+ .unwrap()
+ .is_empty()
+ );
+
+ db.delete_cluster_summary_row("c0999").unwrap();
+ assert_eq!(db.count_cluster_summaries().unwrap(), 999);
+ assert!(
+ !db.get_all_cluster_summary(None)
+ .unwrap()
+ .contains_key("c0999")
+ );
+ }
+
+ #[test]
+ fn cluster_centroid_vector_ids() {
+ let (_directory, db) = seeded();
+ assert!(
+ db.get_cluster_centroid_vector_id_map(&[], true)
+ .unwrap()
+ .is_empty()
+ );
+ let created = db
+ .get_cluster_centroid_vector_id_map(&strings(["c2", "c3", "c2"]), true)
+ .unwrap();
+ assert_eq!(created.len(), 2);
+ assert_eq!(created["c2"], 2);
+ assert!(created["c3"] > 2);
+ db.delete_cluster_centroid_vector_id_mapping("c1").unwrap();
+ assert_eq!(
+ db.get_cluster_centroid_vector_id_map(&strings(["c1", "c2"]), false)
+ .unwrap(),
+ pairs([("c2", 2)])
+ );
+ db.clear_cluster_centroid_vector_id_mappings().unwrap();
+ assert!(
+ db.get_cluster_centroid_vector_id_map(&strings(["c2", "c3"]), false)
+ .unwrap()
+ .is_empty()
+ );
+ }
+
+ #[test]
+ fn clusters_for_memory_lane_skips_assigned_and_caps_at_twenty() {
+ let (_directory, db) = open();
+ let mut mapping = HashMap::new();
+ for cluster in 0..30 {
+ for face in 0..=cluster {
+ mapping.insert(format!("{face}_{cluster}"), format!("c{cluster:02}"));
+ }
+ }
+ db.update_face_id_to_cluster_id(&mapping).unwrap();
+ let clusters = db
+ .get_clusters_for_memory_lane(&set(["c29", "c27"]))
+ .unwrap();
+ assert_eq!(clusters.len(), 20);
+ assert!(!clusters.contains("c29"));
+ assert!(!clusters.contains("c27"));
+ assert!(clusters.contains("c28"));
+ assert!(clusters.contains("c08"));
+ assert!(!clusters.contains("c07"));
+ assert_eq!(db.get_clusters_for_memory_lane(&set([])).unwrap().len(), 20);
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/faces.rs b/rust/crates/photos/src/ml_db/faces.rs
new file mode 100644
index 0000000000..ab4ebf55c8
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/faces.rs
@@ -0,0 +1,805 @@
+use std::collections::{HashMap, HashSet};
+
+use crate::db::{
+ MAX_SQL_BIND_PARAMS_PER_QUERY, Row, SqliteResult, ToSql, bind_placeholders, group_into,
+ optional_parameter, pair, params_from_iter,
+};
+
+use super::queries::{
+ CLIP_ML_VERSION, FACE_ML_VERSION, FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding,
+ LAPLACIAN_HARD_THRESHOLD, MINIMUM_QUALITY_FACE_SCORE, PET_ML_VERSION,
+};
+use super::vector_encoding::{decode_evector, encode_evector};
+use super::{
+ Error, MlDb, Result, file_id_from_face_id, limit_clause, non_empty, try_file_id_from_face_id,
+ unique_in_order,
+};
+
+const FACE_EMBEDDING_MAP_MAX_ROWS: usize = 20000;
+
+const UPSERT_FACE: &str = "INSERT INTO faces (file_id, face_id, detection, embedding, score, blur, is_sideways, height, width, ml_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(file_id, face_id) DO UPDATE SET face_id = excluded.face_id, detection = excluded.detection, embedding = excluded.embedding, score = excluded.score, blur = excluded.blur, is_sideways = excluded.is_sideways, height = excluded.height, width = excluded.width, ml_version = excluded.ml_version";
+
+impl MlDb {
+ pub fn bulk_insert_faces(&self, faces: &[FaceRow]) -> Result<()> {
+ self.db
+ .write_in_batches(
+ UPSERT_FACE,
+ 500,
+ faces.iter().map(|face| {
+ (
+ face.file_id,
+ &face.face_id,
+ &face.detection_json,
+ encode_evector(&face.embedding),
+ face.score,
+ face.blur,
+ face.is_sideways,
+ face.image_height,
+ face.image_width,
+ face.ml_version,
+ )
+ }),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn face_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {
+ self.db
+ .read_all(
+ "SELECT file_id, ml_version FROM faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ pair,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
+ self.db
+ .read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_embeddings_for_cluster(
+ &self,
+ cluster_id: &str,
+ limit: Option<i64>,
+ ) -> Result<Vec<Vec<u8>>> {
+ let sql = format!(
+ "SELECT embedding FROM faces WHERE face_id in (SELECT face_id from face_clusters where cluster_id = ?){}",
+ limit_clause(limit)
+ );
+ let mut parameters: Vec<&dyn ToSql> = vec![&cluster_id];
+ parameters.extend(optional_parameter(&limit));
+ self.db
+ .read_column(&sql, parameters.as_slice())
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_embeddings_for_clusters(
+ &self,
+ cluster_ids: &[String],
+ limit: Option<i64>,
+ ) -> Result<HashMap<String, Vec<Vec<u8>>>> {
+ if cluster_ids.is_empty() {
+ return Ok(HashMap::new());
+ }
+ let unique_cluster_ids = unique_in_order(cluster_ids);
+ let max_cluster_ids_per_query = if limit.is_some() {
+ MAX_SQL_BIND_PARAMS_PER_QUERY - 1
+ } else {
+ MAX_SQL_BIND_PARAMS_PER_QUERY
+ };
+ let mut remaining_limit = limit;
+ let mut result: HashMap<String, Vec<Vec<u8>>> = HashMap::new();
+ for cluster_chunk in unique_cluster_ids.chunks(max_cluster_ids_per_query) {
+ if remaining_limit.is_some_and(|remaining| remaining <= 0) {
+ break;
+ }
+ let sql = format!(
+ "SELECT fc.cluster_id, fe.embedding FROM face_clusters fc INNER JOIN faces fe ON fc.face_id = fe.face_id WHERE fc.cluster_id IN ({}){}",
+ bind_placeholders(cluster_chunk.len()),
+ limit_clause(remaining_limit)
+ );
+ let mut parameters: Vec<&dyn ToSql> =
+ cluster_chunk.iter().map(|id| id as &dyn ToSql).collect();
+ parameters.extend(optional_parameter(&remaining_limit));
+ let rows: Vec<(String, Vec<u8>)> =
+ self.db.read_all(&sql, parameters.as_slice(), pair)?;
+ if let Some(remaining) = remaining_limit.as_mut() {
+ *remaining -= rows.len() as i64;
+ }
+ for (cluster_id, embedding) in rows {
+ result.entry(cluster_id).or_default().push(embedding);
+ }
+ }
+ Ok(result)
+ }
+
+ pub fn get_cover_face_for_person(
+ &self,
+ recent_file_id: i64,
+ person_id: Option<&str>,
+ avatar_face_id: Option<&str>,
+ cluster_id: Option<&str>,
+ ) -> Result<Option<FaceRow>> {
+ if let Some(person_id) = person_id {
+ let mut file_ids = vec![recent_file_id];
+ let avatar_file_id = avatar_face_id.and_then(try_file_id_from_face_id);
+ if let Some(avatar_file_id) = avatar_file_id {
+ file_ids.push(avatar_file_id);
+ }
+ let cluster_ids: Vec<String> = self.db.read_column(
+ "SELECT cluster_id FROM cluster_person WHERE person_id = ?",
+ [person_id],
+ )?;
+ let sql = format!(
+ "SELECT * FROM faces WHERE face_id IN (SELECT face_id FROM face_clusters WHERE cluster_id IN ({})) AND file_id IN ({}) ORDER BY score DESC",
+ bind_placeholders(cluster_ids.len()),
+ bind_placeholders(file_ids.len())
+ );
+ let mut parameters: Vec<&dyn ToSql> =
+ cluster_ids.iter().map(|id| id as &dyn ToSql).collect();
+ parameters.extend(file_ids.iter().map(|id| id as &dyn ToSql));
+ let faces: Vec<StoredFace> =
+ self.db
+ .read_all(&sql, parameters.as_slice(), read_stored_face)?;
+ if !faces.is_empty() {
+ if let Some(avatar_file_id) = avatar_file_id
+ && let Some(face) = faces.iter().find(|face| face.row.file_id == avatar_file_id)
+ {
+ return face.clone().decode().map(Some);
+ }
+ return faces[0].clone().decode().map(Some);
+ }
+ }
+ if let Some(cluster_id) = cluster_id {
+ let face_ids: Vec<String> = self.db.read_column(
+ "SELECT face_id FROM face_clusters WHERE cluster_id = ?",
+ [cluster_id],
+ )?;
+ if let Some(faces) = self.get_faces_for_given_file_id(recent_file_id)? {
+ for face in faces {
+ if face_ids.contains(&face.face_id) {
+ return Ok(Some(face));
+ }
+ }
+ }
+ }
+ if person_id.is_none() && cluster_id.is_none() {
+ return Err(Error::InvalidArgument(
+ "personID and clusterID cannot be null".to_string(),
+ ));
+ }
+ Ok(None)
+ }
+
+ pub fn get_faces_for_given_file_id(&self, file_upload_id: i64) -> Result<Option<Vec<FaceRow>>> {
+ let faces: Vec<StoredFace> = self.db.read_all(
+ "SELECT * FROM faces WHERE file_id = ?",
+ [file_upload_id],
+ read_stored_face,
+ )?;
+ non_empty(faces)
+ .map(|faces| faces.into_iter().map(StoredFace::decode).collect())
+ .transpose()
+ }
+
+ pub fn get_file_ids_to_faces_without_embedding(
+ &self,
+ ) -> Result<HashMap<i64, Vec<FaceWithoutEmbedding>>> {
+ let faces: Vec<FaceWithoutEmbedding> = self.db.read_all(
+ "SELECT face_id, file_id, score, detection, blur FROM faces",
+ (),
+ |row| {
+ Ok(FaceWithoutEmbedding {
+ face_id: row.get(0)?,
+ file_id: row.get(1)?,
+ score: row.get(2)?,
+ detection_json: row.get(3)?,
+ blur: row.get(4)?,
+ })
+ },
+ )?;
+ Ok(group_into(
+ faces.into_iter().map(|face| (face.file_id, face)),
+ ))
+ }
+
+ pub fn get_face_info_for_clustering(
+ &self,
+ max_faces: i64,
+ offset: i64,
+ batch_size: i64,
+ ) -> Result<Vec<FaceDbInfoForClustering>> {
+ let mut offset = offset;
+ let mut result = Vec::new();
+ loop {
+ let rows: Vec<(String, Vec<u8>, f64, f64, i64)> = self.db.read_all(
+ "SELECT face_id, embedding, score, blur, is_sideways FROM faces WHERE score > ? AND blur > ? ORDER BY face_id DESC LIMIT ? OFFSET ?",
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD, batch_size, offset),
+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)),
+ )?;
+ if rows.is_empty() {
+ break;
+ }
+ let face_ids: Vec<String> = rows.iter().map(|row| row.0.clone()).collect();
+ let face_id_to_cluster_id = self.get_face_ids_to_cluster_ids(&face_ids)?;
+ for (face_id, embedding_bytes, face_score, blur_value, is_sideways) in rows {
+ let cluster_id = face_id_to_cluster_id.get(&face_id).cloned().flatten();
+ result.push(FaceDbInfoForClustering {
+ face_id,
+ cluster_id,
+ embedding_bytes,
+ face_score,
+ blur_value,
+ is_sideways: is_sideways == 1,
+ });
+ }
+ if result.len() as i64 >= max_faces {
+ break;
+ }
+ offset += batch_size;
+ }
+ Ok(result)
+ }
+
+ pub fn get_face_embedding_map_for_faces(
+ &self,
+ face_ids: &[String],
+ ) -> Result<HashMap<String, Vec<u8>>> {
+ let mut rows: Vec<(String, Vec<u8>)> = self.db.read_chunked_in(
+ "SELECT face_id, embedding FROM faces WHERE face_id IN ({}) ORDER BY face_id DESC",
+ face_ids,
+ MAX_SQL_BIND_PARAMS_PER_QUERY,
+ pair,
+ )?;
+ if rows.len() > FACE_EMBEDDING_MAP_MAX_ROWS {
+ rows.sort_by(|left, right| right.0.cmp(&left.0));
+ rows.dedup_by(|left, right| left.0 == right.0);
+ rows.truncate(FACE_EMBEDDING_MAP_MAX_ROWS);
+ }
+ Ok(rows.into_iter().collect())
+ }
+
+ pub fn get_total_face_count(&self) -> Result<i64> {
+ self.db
+ .read_value(
+ "SELECT COUNT(*) as count FROM faces WHERE score > ? AND blur > ?",
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_errored_face_count(&self) -> Result<i64> {
+ self.db
+ .read_value("SELECT COUNT(*) as count FROM faces WHERE score < 0", ())
+ .map_err(Into::into)
+ }
+
+ pub fn get_errored_file_ids(&self) -> Result<HashSet<i64>> {
+ self.db
+ .read_column("SELECT DISTINCT file_id FROM faces WHERE score < 0", ())
+ .map_err(Into::into)
+ }
+
+ pub fn prune_resolved_face_error_results(&self, file_ids: &[i64]) -> Result<()> {
+ if file_ids.is_empty() {
+ return Ok(());
+ }
+ self.db
+ .execute_chunked_in(
+ "DELETE FROM faces WHERE file_id IN ({}) AND score < 0 AND EXISTS (SELECT 1 FROM faces AS successful WHERE successful.file_id = faces.file_id AND successful.score >= 0 AND successful.ml_version >= faces.ml_version)",
+ file_ids,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_file_ids_with_error_results(&self, file_ids: &[i64]) -> Result<HashSet<i64>> {
+ if file_ids.is_empty() {
+ return Ok(HashSet::new());
+ }
+ let mut result = HashSet::new();
+ for chunk in file_ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY / 3) {
+ let placeholders = bind_placeholders(chunk.len());
+ let sql = format!(
+ "SELECT failed.file_id FROM faces AS failed WHERE failed.file_id IN ({placeholders}) AND failed.score < 0 AND NOT EXISTS (SELECT 1 FROM faces AS successful WHERE successful.file_id = failed.file_id AND successful.score >= 0 AND successful.ml_version >= failed.ml_version) UNION SELECT file_id FROM clip WHERE file_id IN ({placeholders}) AND LENGTH(embedding) = 0 UNION SELECT file_id FROM pet_faces WHERE file_id IN ({placeholders}) AND score < 0"
+ );
+ let matches: Vec<i64> = self.db.read_column(
+ &sql,
+ params_from_iter(chunk.iter().chain(chunk).chain(chunk)),
+ )?;
+ result.extend(matches);
+ }
+ Ok(result)
+ }
+
+ pub fn delete_face_index_for_files(&self, file_ids: &[i64]) -> Result<()> {
+ self.db
+ .execute_chunked_in("DELETE FROM faces WHERE file_id IN ({})", file_ids)
+ .map_err(Into::into)
+ }
+
+ pub fn delete_unclustered_face_index_for_files(&self, file_ids: &[i64]) -> Result<()> {
+ if file_ids.is_empty() {
+ return Ok(());
+ }
+ self.db
+ .execute_chunked_in(
+ "DELETE FROM faces WHERE file_id IN ({}) AND NOT EXISTS (SELECT 1 FROM face_clusters WHERE face_clusters.face_id = faces.face_id)",
+ file_ids,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_clustered_or_faceless_file_count(&self) -> Result<i64> {
+ let clustered_face_ids: Vec<String> = self
+ .db
+ .read_column("SELECT face_id FROM face_clusters", ())?;
+ let clustered_file_ids = clustered_face_ids
+ .iter()
+ .map(|face_id| file_id_from_face_id(face_id))
+ .collect::<Result<HashSet<i64>>>()?;
+
+ let bad_file_ids: HashSet<i64> = self.db.read_column(
+ "SELECT DISTINCT file_id FROM faces WHERE score <= ? OR blur <= ?",
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ )?;
+ let good_file_ids: HashSet<i64> = self.db.read_column(
+ "SELECT DISTINCT file_id FROM faces WHERE score > ? AND blur > ?",
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ )?;
+ let truly_faceless_files = bad_file_ids.difference(&good_file_ids).count();
+ Ok((clustered_file_ids.len() + truly_faceless_files) as i64)
+ }
+
+ pub fn get_unclustered_face_count(&self) -> Result<i64> {
+ self.db
+ .read_value(
+ "SELECT COUNT(*) as count FROM faces f LEFT JOIN face_clusters fc ON f.face_id = fc.face_id WHERE f.score > ? AND f.blur > ? AND fc.face_id IS NULL",
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_all_file_ids_of_face_ids_not_in_any_cluster(&self) -> Result<HashSet<i64>> {
+ self.db
+ .read_column(
+ "SELECT DISTINCT file_id FROM faces LEFT JOIN face_clusters ON faces.face_id = face_clusters.face_id WHERE face_clusters.face_id IS NULL",
+ (),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_all_files_associated_with_all_clusters(
+ &self,
+ except_clusters: Option<&[String]>,
+ ) -> Result<HashSet<i64>> {
+ let except_clusters = except_clusters.unwrap_or_default();
+ let sql = format!(
+ "SELECT DISTINCT faces.file_id FROM faces JOIN face_clusters on face_clusters.face_id = faces.face_id WHERE face_clusters.cluster_id NOT IN ({})",
+ bind_placeholders(except_clusters.len())
+ );
+ self.db
+ .read_column(&sql, params_from_iter(except_clusters))
+ .map_err(Into::into)
+ }
+
+ pub fn get_fully_indexed_file_ids(&self, include_pets: bool) -> Result<HashSet<i64>> {
+ let mut sql = String::from(
+ "SELECT file_id FROM faces WHERE ml_version >= ? INTERSECT SELECT file_id FROM clip WHERE ml_version >= ?",
+ );
+ let mut parameters = vec![FACE_ML_VERSION, CLIP_ML_VERSION];
+ if include_pets {
+ sql.push_str(" INTERSECT SELECT file_id FROM pet_faces WHERE ml_version >= ?");
+ parameters.push(PET_ML_VERSION);
+ }
+ self.db
+ .read_column(&sql, params_from_iter(parameters))
+ .map_err(Into::into)
+ }
+}
+
+#[derive(Clone)]
+struct StoredFace {
+ row: FaceRow,
+ embedding_bytes: Vec<u8>,
+}
+
+impl StoredFace {
+ fn decode(mut self) -> Result<FaceRow> {
+ self.row.embedding = decode_evector(&self.embedding_bytes)?;
+ Ok(self.row)
+ }
+}
+
+fn read_stored_face(row: &Row<'_>) -> SqliteResult<StoredFace> {
+ Ok(StoredFace {
+ row: FaceRow {
+ file_id: row.get("file_id")?,
+ face_id: row.get("face_id")?,
+ detection_json: row.get("detection")?,
+ embedding: Vec::new(),
+ score: row.get("score")?,
+ blur: row.get("blur")?,
+ is_sideways: row.get::<_, i64>("is_sideways")? == 1,
+ image_height: row.get("height")?,
+ image_width: row.get("width")?,
+ ml_version: row.get("ml_version")?,
+ },
+ embedding_bytes: row.get("embedding")?,
+ })
+}
+
+#[cfg(test)]
+pub(super) mod tests {
+ use std::collections::HashMap;
+ use std::ops::RangeInclusive;
+
+ use super::{FaceRow, MlDb};
+ use crate::ml_db::tests::{cases, check, ids, open, pairs, sorted, strings};
+ use crate::ml_db::vector_encoding::encode_evector;
+ use crate::ml_db::{Error, PetFaceRow, clusters, persons, pets, queries};
+ use tempfile::TempDir;
+
+ fn face(file_id: i64, index: i64, score: f64, blur: f64, is_sideways: bool) -> FaceRow {
+ FaceRow {
+ file_id,
+ face_id: format!("{file_id}_{index}"),
+ detection_json: format!("{{\"index\":{index}}}"),
+ embedding: vec![file_id as f64, index as f64, 0.5],
+ score,
+ blur,
+ is_sideways,
+ image_height: 100,
+ image_width: 200,
+ ml_version: 1,
+ }
+ }
+
+ fn good_face(file_id: i64, index: i64) -> FaceRow {
+ face(file_id, index, 0.9, 300.0, false)
+ }
+
+ fn versions(file_ids: RangeInclusive<i64>, version: i64) -> HashMap<i64, i64> {
+ file_ids.map(|file_id| (file_id, version)).collect()
+ }
+
+ pub(in crate::ml_db) fn seed(db: &MlDb) {
+ db.bulk_insert_faces(&[
+ good_face(1, 0),
+ face(1, 1, 0.95, 300.0, false),
+ face(2, 0, 0.85, 300.0, false),
+ face(3, 0, 0.95, 300.0, false),
+ face(4, 0, 0.5, 300.0, false),
+ face(5, 0, -1.0, 300.0, false),
+ face(6, 0, 0.9, 5.0, false),
+ good_face(6, 1),
+ face(7, 0, 0.9, 300.0, true),
+ face(8, 0, 0.82, 150.0, false),
+ face(9, 0, 0.9, 20.0, false),
+ FaceRow {
+ ml_version: 0,
+ ..good_face(10, 0)
+ },
+ ])
+ .unwrap();
+ }
+
+ fn seeded() -> (TempDir, MlDb) {
+ let (directory, db) = open();
+ queries::tests::seed(&db);
+ seed(&db);
+ clusters::tests::seed(&db);
+ persons::tests::seed(&db);
+ pets::tests::seed(&db);
+ (directory, db)
+ }
+
+ #[test]
+ fn seeded_counts() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "total faces": 9 => |db| db.get_total_face_count(),
+ "errored faces": 1 => |db| db.get_errored_face_count(),
+ "face files": 9 => |db| db.get_face_indexed_file_count(1),
+ "face files v0": 10 => |db| db.get_face_indexed_file_count(0),
+ "unclustered faces": 2 => |db| db.get_unclustered_face_count(),
+ "clustered or faceless files": 8 => |db| db.get_clustered_or_faceless_file_count(),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_file_id_sets() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "errored": ids([5]) => |db| db.get_errored_file_ids(),
+ "fully indexed": ids([1, 2, 3, 4]) => |db| db.get_fully_indexed_file_ids(false),
+ "fully indexed with pets": ids([1, 2]) => |db| db.get_fully_indexed_file_ids(true),
+ "unclustered": ids([4, 5, 6, 10]) =>
+ |db| db.get_all_file_ids_of_face_ids_not_in_any_cluster(),
+ "clustered": ids([1, 2, 3, 7, 8, 9]) =>
+ |db| db.get_all_files_associated_with_all_clusters(None),
+ "clustered except c1": ids([1, 3, 7, 8, 9]) =>
+ |db| db.get_all_files_associated_with_all_clusters(Some(&strings(["c1"]))),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_versions() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "face versions": versions(1..=9, 1) => |db| db.face_indexed_file_ids(1),
+ "face versions v0": versions(1..=9, 1).into_iter().chain([(10, 0)]).collect() =>
+ |db| db.face_indexed_file_ids(0),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_rows() {
+ let (_directory, db) = seeded();
+ let mut faces = db.get_faces_for_given_file_id(1).unwrap().unwrap();
+ faces.sort_by(|left, right| left.face_id.cmp(&right.face_id));
+ assert_eq!(faces, vec![good_face(1, 0), face(1, 1, 0.95, 300.0, false)]);
+ assert_eq!(db.get_faces_for_given_file_id(99).unwrap(), None);
+
+ let without_embedding = db.get_file_ids_to_faces_without_embedding().unwrap();
+ assert_eq!(without_embedding.len(), 10);
+ assert_eq!(without_embedding[&6].len(), 2);
+ let blurry = without_embedding[&6]
+ .iter()
+ .find(|face| face.face_id == "6_0")
+ .unwrap();
+ assert_eq!((blurry.file_id, blurry.score, blurry.blur), (6, 0.9, 5.0));
+ assert_eq!(blurry.detection_json, "{\"index\":0}");
+ }
+
+ #[test]
+ fn face_embeddings_by_cluster_and_face() {
+ let (_directory, db) = seeded();
+ let c1 = db.get_face_embeddings_for_cluster("c1", None).unwrap();
+ assert_eq!(
+ sorted(c1),
+ sorted(vec![
+ encode_evector(&[1.0, 0.0, 0.5]),
+ encode_evector(&[2.0, 0.0, 0.5])
+ ])
+ );
+ assert_eq!(
+ db.get_face_embeddings_for_cluster("c1", Some(1))
+ .unwrap()
+ .len(),
+ 1
+ );
+
+ let all = db
+ .get_face_embeddings_for_clusters(&strings(["c1", "c2", "c1"]), None)
+ .unwrap();
+ assert_eq!(all["c1"].len(), 2);
+ assert_eq!(all["c2"], vec![encode_evector(&[1.0, 1.0, 0.5])]);
+ let limited = db
+ .get_face_embeddings_for_clusters(&strings(["c1", "c2"]), Some(2))
+ .unwrap();
+ assert_eq!(limited.values().map(Vec::len).sum::<usize>(), 2);
+ assert!(
+ db.get_face_embeddings_for_clusters(&[], Some(2))
+ .unwrap()
+ .is_empty()
+ );
+
+ assert_eq!(
+ db.get_face_embedding_map_for_faces(&strings(["1_0", "2_0", "nope"]))
+ .unwrap(),
+ pairs([
+ ("1_0", encode_evector(&[1.0, 0.0, 0.5])),
+ ("2_0", encode_evector(&[2.0, 0.0, 0.5]))
+ ])
+ );
+ assert!(db.get_face_embedding_map_for_faces(&[]).unwrap().is_empty());
+ }
+
+ #[test]
+ fn face_info_for_clustering_filters_and_pages() {
+ let (_directory, db) = seeded();
+ let info = db.get_face_info_for_clustering(20000, 0, 3).unwrap();
+ let ids: Vec<&str> = info.iter().map(|face| face.face_id.as_str()).collect();
+ assert_eq!(
+ ids,
+ [
+ "9_0", "8_0", "7_0", "6_1", "3_0", "2_0", "1_1", "1_0", "10_0"
+ ]
+ );
+ assert_eq!(info[0].cluster_id, Some("c9".to_string()));
+ assert_eq!(info[0].embedding_bytes, encode_evector(&[9.0, 0.0, 0.5]));
+ assert_eq!(
+ (info[0].face_score, info[0].blur_value, info[0].is_sideways),
+ (0.9, 20.0, false)
+ );
+ assert!(info[2].is_sideways);
+ assert_eq!(info[3].cluster_id, None);
+
+ assert_eq!(db.get_face_info_for_clustering(3, 0, 3).unwrap().len(), 3);
+ let last_page = db.get_face_info_for_clustering(20000, 6, 3).unwrap();
+ let last_ids: Vec<&str> = last_page.iter().map(|face| face.face_id.as_str()).collect();
+ assert_eq!(last_ids, ["1_1", "1_0", "10_0"]);
+ assert!(
+ db.get_face_info_for_clustering(20000, 9, 3)
+ .unwrap()
+ .is_empty()
+ );
+ }
+
+ #[test]
+ fn face_embedding_map_keeps_twenty_thousand_highest_face_ids() {
+ let (_directory, db) = open();
+ let faces: Vec<FaceRow> = (0..20005).map(|index| good_face(index, 0)).collect();
+ db.bulk_insert_faces(&faces).unwrap();
+ assert_eq!(db.get_total_face_count().unwrap(), 20005);
+ let mut face_ids: Vec<String> = faces.iter().map(|face| face.face_id.clone()).collect();
+
+ let embeddings = db.get_face_embedding_map_for_faces(&face_ids).unwrap();
+ assert_eq!(embeddings.len(), 20000);
+ face_ids.sort();
+ for face_id in &face_ids[..5] {
+ assert!(!embeddings.contains_key(face_id));
+ }
+ for face_id in &face_ids[5..] {
+ assert!(embeddings.contains_key(face_id));
+ }
+
+ let mut with_duplicates = face_ids[5..].to_vec();
+ with_duplicates.extend(face_ids[5..10].iter().cloned());
+ let embeddings = db
+ .get_face_embedding_map_for_faces(&with_duplicates)
+ .unwrap();
+ assert_eq!(embeddings.len(), 20000);
+ for face_id in &face_ids[5..] {
+ assert!(embeddings.contains_key(face_id));
+ }
+ }
+
+ #[test]
+ fn cover_face_for_person_prefers_avatar_then_score() {
+ let (_directory, db) = seeded();
+ let cover = |recent_file_id, person_id, avatar_face_id, cluster_id| {
+ db.get_cover_face_for_person(recent_file_id, person_id, avatar_face_id, cluster_id)
+ .unwrap()
+ .map(|face| face.face_id)
+ };
+ let by_score = db
+ .get_cover_face_for_person(2, Some("p1"), None, None)
+ .unwrap()
+ .unwrap();
+ assert_eq!(by_score, face(2, 0, 0.85, 300.0, false));
+ assert_eq!(
+ cover(2, Some("p1"), Some("1_1"), None),
+ Some("1_1".to_string())
+ );
+ assert_eq!(
+ cover(2, Some("p1"), Some("9_0"), None),
+ Some("2_0".to_string())
+ );
+ assert_eq!(
+ cover(3, Some("p1"), None, Some("c3")),
+ Some("3_0".to_string())
+ );
+ assert_eq!(cover(1, None, None, Some("c1")), Some("1_0".to_string()));
+ assert_eq!(cover(1, None, None, Some("c3")), None);
+ assert_eq!(cover(9, Some("p9"), None, None), None);
+ assert!(matches!(
+ db.get_cover_face_for_person(1, None, None, None),
+ Err(Error::InvalidArgument(_))
+ ));
+ }
+
+ #[test]
+ fn face_upsert_and_deletion() {
+ let (_directory, db) = seeded();
+ let updated = FaceRow {
+ score: 0.99,
+ embedding: vec![9.0, 8.0],
+ ..good_face(1, 0)
+ };
+ db.bulk_insert_faces(std::slice::from_ref(&updated))
+ .unwrap();
+ let faces = db.get_faces_for_given_file_id(1).unwrap().unwrap();
+ assert_eq!(faces.len(), 2);
+ assert_eq!(
+ faces.iter().find(|face| face.face_id == "1_0"),
+ Some(&updated)
+ );
+ assert_eq!(db.get_total_face_count().unwrap(), 9);
+
+ db.delete_unclustered_face_index_for_files(&[]).unwrap();
+ db.delete_unclustered_face_index_for_files(&[1, 6]).unwrap();
+ assert_eq!(db.get_faces_for_given_file_id(1).unwrap().unwrap().len(), 2);
+ assert_eq!(db.get_faces_for_given_file_id(6).unwrap(), None);
+
+ db.delete_face_index_for_files(&[]).unwrap();
+ db.delete_face_index_for_files(&[1, 2]).unwrap();
+ assert_eq!(db.get_face_indexed_file_count(1).unwrap(), 6);
+ assert_eq!(db.get_faces_for_given_file_id(1).unwrap(), None);
+ }
+
+ #[test]
+ fn error_results_lookup_and_prune() {
+ let (_directory, db) = seeded();
+ db.bulk_insert_faces(&[
+ face(11, 0, -1.0, 300.0, false),
+ good_face(11, 1),
+ FaceRow {
+ ml_version: 2,
+ ..face(12, 0, -1.0, 300.0, false)
+ },
+ good_face(12, 1),
+ ])
+ .unwrap();
+ db.bulk_insert_pet_faces(&[PetFaceRow {
+ face_score: -1.0,
+ ..pets::tests::pet_face(13, 0, 0)
+ }])
+ .unwrap();
+ let file_ids: Vec<i64> = (1..=13).chain([99]).collect();
+ let many: Vec<i64> = (0..=10000).collect();
+ assert_eq!(db.get_file_ids_with_error_results(&[]).unwrap(), ids([]));
+ assert_eq!(
+ db.get_file_ids_with_error_results(&file_ids).unwrap(),
+ ids([4, 5, 12, 13])
+ );
+ assert_eq!(
+ db.get_file_ids_with_error_results(&many).unwrap(),
+ ids([4, 5, 12, 13])
+ );
+
+ db.prune_resolved_face_error_results(&[]).unwrap();
+ db.prune_resolved_face_error_results(&[5]).unwrap();
+ assert_eq!(db.get_errored_file_ids().unwrap(), ids([5, 11, 12]));
+ db.prune_resolved_face_error_results(&[5, 11, 12]).unwrap();
+ assert_eq!(db.get_errored_file_ids().unwrap(), ids([5, 12]));
+ assert_eq!(
+ db.get_faces_for_given_file_id(11).unwrap().unwrap().len(),
+ 1
+ );
+ assert_eq!(
+ db.get_file_ids_with_error_results(&file_ids).unwrap(),
+ ids([4, 5, 12, 13])
+ );
+ }
+
+ #[test]
+ fn large_id_lists_are_chunked() {
+ let (_directory, db) = open();
+ let faces: Vec<FaceRow> = (0..30).map(|index| good_face(index, 0)).collect();
+ db.bulk_insert_faces(&faces).unwrap();
+ let mapping: HashMap<String, String> = (0..30)
+ .map(|index| (format!("{index}_0"), format!("c{}", index % 3)))
+ .collect();
+ db.update_face_id_to_cluster_id(&mapping).unwrap();
+
+ let face_ids: Vec<String> = (0..12000).map(|index| format!("{index}_0")).collect();
+ assert_eq!(db.get_face_ids_to_cluster_ids(&face_ids).unwrap().len(), 30);
+ assert_eq!(
+ db.get_face_embedding_map_for_faces(&face_ids)
+ .unwrap()
+ .len(),
+ 30
+ );
+ let file_ids: Vec<i64> = (0..12000).collect();
+ db.delete_unclustered_face_index_for_files(&file_ids)
+ .unwrap();
+ assert_eq!(db.get_total_face_count().unwrap(), 30);
+ db.delete_face_index_for_files(&file_ids).unwrap();
+ assert_eq!(db.get_total_face_count().unwrap(), 0);
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index 7c1645c0a9..113643a113 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -1,14 +1,20 @@
mod caches;
+mod clusters;
+mod faces;
mod filedata;
+mod persons;
+mod pets;
mod queries;
mod schema;
mod vector_encoding;
+use std::collections::{HashMap, HashSet};
use std::path::Path;
use crate::db::{Database, OpenOptions};
pub use filedata::{FdStatus, PreviewInfo};
+pub use persons::PersonToClusterIdToFaceIds;
pub use queries::{
CLIP_EMBEDDING_BYTES_LENGTH, CLIP_EMBEDDING_DIMENSIONS, CLIP_ML_VERSION, ClipEmbedding,
ClipRow, ClusterCentroidRow, ClusterSummary, EmbeddingVector, FACE_ML_VERSION,
@@ -51,8 +57,47 @@ impl MlDb {
}
}
+fn file_id_to_cluster_ids(
+ cluster_and_face_ids: Vec<(String, String)>,
+) -> Result<HashMap<i64, HashSet<String>>> {
+ let mut result: HashMap<i64, HashSet<String>> = HashMap::new();
+ for (cluster_id, face_id) in cluster_and_face_ids {
+ let file_id = file_id_from_face_id(&face_id)?;
+ result.entry(file_id).or_default().insert(cluster_id);
+ }
+ Ok(result)
+}
+
+fn file_id_from_face_id(face_id: &str) -> Result<i64> {
+ try_file_id_from_face_id(face_id)
+ .ok_or_else(|| Error::Codec(format!("Error parsing faceId: {face_id}")))
+}
+
+fn try_file_id_from_face_id(face_id: &str) -> Option<i64> {
+ face_id
+ .split_once('_')
+ .and_then(|(file_id, _)| file_id.parse().ok())
+}
+
+fn unique_in_order(ids: &[String]) -> Vec<&str> {
+ let mut seen = HashSet::new();
+ ids.iter()
+ .map(String::as_str)
+ .filter(|id| seen.insert(*id))
+ .collect()
+}
+
+fn limit_clause(limit: Option<i64>) -> &'static str {
+ if limit.is_some() { " LIMIT ?" } else { "" }
+}
+
+fn non_empty<T>(rows: Vec<T>) -> Option<Vec<T>> {
+ (!rows.is_empty()).then_some(rows)
+}
+
#[cfg(test)]
mod tests {
+ use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use std::path::Path;
@@ -84,7 +129,7 @@ mod tests {
.unwrap()
}
- fn index_count(path: &Path) -> i64 {
+ pub(super) fn index_count(path: &Path) -> i64 {
Connection::open(path)
.unwrap()
.query_row(
@@ -101,6 +146,53 @@ mod tests {
}
}
+ fn strings_of<C: FromIterator<String>>(values: &[&str]) -> C {
+ values.iter().map(|value| value.to_string()).collect()
+ }
+
+ pub(super) fn strings<const N: usize>(values: [&str; N]) -> Vec<String> {
+ strings_of(&values)
+ }
+
+ pub(super) fn set<const N: usize>(values: [&str; N]) -> HashSet<String> {
+ strings_of(&values)
+ }
+
+ pub(super) fn ids<const N: usize>(values: [i64; N]) -> HashSet<i64> {
+ HashSet::from(values)
+ }
+
+ pub(super) fn pairs<V, const N: usize>(entries: [(&str, V); N]) -> HashMap<String, V> {
+ entries
+ .into_iter()
+ .map(|(key, value)| (key.to_string(), value))
+ .collect()
+ }
+
+ pub(super) fn map<const N: usize>(entries: [(&str, &str); N]) -> HashMap<String, String> {
+ pairs(entries.map(|(key, value)| (key, value.to_string())))
+ }
+
+ pub(super) fn grouped<C: FromIterator<String>, const N: usize>(
+ entries: [(&str, &[&str]); N],
+ ) -> HashMap<String, C> {
+ pairs(entries.map(|(key, values)| (key, strings_of(values))))
+ }
+
+ pub(super) fn grouped_by_file<const N: usize>(
+ entries: [(i64, &[&str]); N],
+ ) -> HashMap<i64, HashSet<String>> {
+ entries
+ .into_iter()
+ .map(|(file_id, values)| (file_id, strings_of(values)))
+ .collect()
+ }
+
+ pub(super) fn sorted<T: Ord>(mut values: Vec<T>) -> Vec<T> {
+ values.sort();
+ values
+ }
+
#[test]
fn ml_db_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
diff --git a/rust/crates/photos/src/ml_db/persons.rs b/rust/crates/photos/src/ml_db/persons.rs
new file mode 100644
index 0000000000..fd95fa0ee3
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/persons.rs
@@ -0,0 +1,419 @@
+use std::collections::{HashMap, HashSet};
+
+use crate::db::{MAX_SQL_BIND_PARAMS_PER_QUERY, pair};
+
+use super::schema::{
+ CREATE_CLUSTER_PERSON_TABLE, CREATE_NOT_PERSON_FEEDBACK_TABLE, DELETE_CLUSTER_PERSON,
+ DELETE_NOT_PERSON_FEEDBACK,
+};
+use super::{MlDb, Result, file_id_to_cluster_ids};
+
+pub type PersonToClusterIdToFaceIds = HashMap<String, HashMap<String, HashSet<String>>>;
+
+const INSERT_CLUSTER_PERSON: &str = "INSERT INTO cluster_person (person_id, cluster_id) VALUES (?, ?) ON CONFLICT(person_id, cluster_id) DO NOTHING";
+const INSERT_NOT_PERSON_FEEDBACK: &str =
+ "INSERT INTO not_person_feedback (person_id, cluster_id) VALUES (?, ?) ON CONFLICT DO NOTHING";
+
+impl MlDb {
+ pub fn get_clusters_with_three_or_more_not_person_feedback(&self) -> Result<HashSet<String>> {
+ self.db
+ .read_column(
+ "SELECT cluster_id FROM not_person_feedback GROUP BY cluster_id HAVING COUNT(*) >= 3",
+ (),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_person_ignored_clusters(&self, person_id: &str) -> Result<HashSet<String>> {
+ let mut ignored_cluster_ids: HashSet<String> = self.db.read_column(
+ "SELECT cluster_id FROM cluster_person WHERE person_id != ? AND person_id IS NOT NULL",
+ [person_id],
+ )?;
+ let reject_cluster_ids: Vec<String> = self.db.read_column(
+ "SELECT cluster_id FROM not_person_feedback WHERE person_id = ?",
+ [person_id],
+ )?;
+ ignored_cluster_ids.extend(reject_cluster_ids);
+ Ok(ignored_cluster_ids)
+ }
+
+ pub fn get_person_to_rejected_suggestions(&self) -> Result<HashMap<String, HashSet<String>>> {
+ self.db
+ .read_grouped("SELECT person_id, cluster_id FROM not_person_feedback", ())
+ .map_err(Into::into)
+ }
+
+ pub fn get_person_cluster_ids(&self, person_id: &str) -> Result<HashSet<String>> {
+ self.db
+ .read_column(
+ "SELECT cluster_id FROM cluster_person WHERE person_id = ?",
+ [person_id],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_persons_cluster_ids(&self, person_ids: &[String]) -> Result<HashSet<String>> {
+ self.db
+ .read_chunked_in(
+ "SELECT cluster_id FROM cluster_person WHERE person_id IN ({})",
+ person_ids,
+ MAX_SQL_BIND_PARAMS_PER_QUERY,
+ |row| row.get(0),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_person_to_cluster_id_to_face_ids(&self) -> Result<PersonToClusterIdToFaceIds> {
+ let rows: Vec<(String, String, String)> = self.db.read_all(
+ "SELECT person_id, face_clusters.cluster_id, face_id FROM cluster_person INNER JOIN face_clusters ON cluster_person.cluster_id = face_clusters.cluster_id",
+ (),
+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+ )?;
+ let mut result: PersonToClusterIdToFaceIds = HashMap::new();
+ for (person_id, cluster_id, face_id) in rows {
+ result
+ .entry(person_id)
+ .or_default()
+ .entry(cluster_id)
+ .or_default()
+ .insert(face_id);
+ }
+ Ok(result)
+ }
+
+ pub fn get_person_to_cluster_ids(&self) -> Result<HashMap<String, HashSet<String>>> {
+ self.db
+ .read_grouped("SELECT person_id, cluster_id FROM cluster_person", ())
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_id_to_person_id_for_faces(
+ &self,
+ face_ids: &[String],
+ ) -> Result<HashMap<String, String>> {
+ self.db
+ .read_chunked_in(
+ "SELECT face_id, person_id FROM cluster_person INNER JOIN face_clusters ON cluster_person.cluster_id = face_clusters.cluster_id WHERE face_id IN ({})",
+ face_ids,
+ MAX_SQL_BIND_PARAMS_PER_QUERY,
+ pair,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_cluster_id_to_face_ids_for_person(
+ &self,
+ person_id: &str,
+ ) -> Result<HashMap<String, HashSet<String>>> {
+ self.db
+ .read_grouped(
+ "SELECT face_clusters.cluster_id, face_id FROM cluster_person INNER JOIN face_clusters ON cluster_person.cluster_id = face_clusters.cluster_id WHERE person_id = ?",
+ [person_id],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_ids_for_person(&self, person_id: &str) -> Result<HashSet<String>> {
+ self.db
+ .read_column(
+ "SELECT face_id FROM face_clusters LEFT JOIN cluster_person ON face_clusters.cluster_id = cluster_person.cluster_id WHERE cluster_person.person_id = ?",
+ [person_id],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_face_ids_for_person_ordered_by_score(
+ &self,
+ person_id: &str,
+ limit: i64,
+ ) -> Result<Vec<String>> {
+ self.db
+ .read_column(
+ "SELECT faces.face_id FROM faces JOIN face_clusters ON faces.face_id = face_clusters.face_id JOIN cluster_person ON face_clusters.cluster_id = cluster_person.cluster_id WHERE cluster_person.person_id = ? ORDER BY faces.score DESC LIMIT ?",
+ (person_id, limit),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn remove_person(&self, person_id: &str) -> Result<()> {
+ self.db
+ .write_transaction(|transaction| {
+ transaction.execute(
+ "DELETE FROM cluster_person WHERE person_id = ?",
+ [person_id],
+ )?;
+ transaction.execute(
+ "DELETE FROM not_person_feedback WHERE person_id = ?",
+ [person_id],
+ )?;
+ Ok(())
+ })
+ .map_err(Into::into)
+ }
+
+ pub fn assign_cluster_to_person(&self, person_id: &str, cluster_id: &str) -> Result<()> {
+ self.db
+ .execute(INSERT_CLUSTER_PERSON, [person_id, cluster_id])?;
+ Ok(())
+ }
+
+ pub fn bulk_assign_cluster_to_person_id(
+ &self,
+ cluster_to_person_id: &HashMap<String, String>,
+ ) -> Result<()> {
+ self.db
+ .write_batch(
+ INSERT_CLUSTER_PERSON,
+ cluster_to_person_id
+ .iter()
+ .map(|(cluster_id, person_id)| (person_id, cluster_id)),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn capture_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {
+ self.db
+ .execute(INSERT_NOT_PERSON_FEEDBACK, [person_id, cluster_id])?;
+ Ok(())
+ }
+
+ pub fn bulk_capture_not_person_feedback(
+ &self,
+ cluster_to_person_id: &HashMap<String, String>,
+ ) -> Result<()> {
+ self.db
+ .write_batch(
+ INSERT_NOT_PERSON_FEEDBACK,
+ cluster_to_person_id
+ .iter()
+ .map(|(cluster_id, person_id)| (person_id, cluster_id)),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn remove_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {
+ self.db.execute(
+ "DELETE FROM not_person_feedback WHERE person_id = ? AND cluster_id = ?",
+ [person_id, cluster_id],
+ )?;
+ Ok(())
+ }
+
+ pub fn remove_cluster_to_person(&self, person_id: &str, cluster_id: &str) -> Result<()> {
+ self.db.execute(
+ "DELETE FROM cluster_person WHERE person_id = ? AND cluster_id = ?",
+ [person_id, cluster_id],
+ )?;
+ Ok(())
+ }
+
+ pub fn get_file_id_to_cluster_id_set(
+ &self,
+ person_id: &str,
+ ) -> Result<HashMap<i64, HashSet<String>>> {
+ self.db
+ .read_all(
+ "SELECT face_clusters.cluster_id, face_id FROM face_clusters INNER JOIN cluster_person ON face_clusters.cluster_id = cluster_person.cluster_id WHERE cluster_person.person_id = ?",
+ [person_id],
+ pair,
+ )
+ .map_err(Into::into)
+ .and_then(file_id_to_cluster_ids)
+ }
+
+ pub fn get_cluster_id_to_person_id(&self) -> Result<HashMap<String, String>> {
+ self.db
+ .read_all(
+ "SELECT person_id, cluster_id FROM cluster_person",
+ (),
+ |row| Ok((row.get(1)?, row.get(0)?)),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn drop_faces_feedback_tables(&self) -> Result<()> {
+ self.db
+ .execute_statements([
+ DELETE_CLUSTER_PERSON,
+ DELETE_NOT_PERSON_FEEDBACK,
+ CREATE_CLUSTER_PERSON_TABLE,
+ CREATE_NOT_PERSON_FEEDBACK_TABLE,
+ ])
+ .map_err(Into::into)
+ }
+
+ pub fn get_file_ids_of_person_id(&self, person_id: &str) -> Result<Vec<i64>> {
+ self.db
+ .read_column(
+ "SELECT DISTINCT faces.file_id FROM cluster_person JOIN face_clusters ON cluster_person.cluster_id = face_clusters.cluster_id JOIN faces ON face_clusters.face_id = faces.face_id WHERE cluster_person.person_id = ?",
+ [person_id],
+ )
+ .map_err(Into::into)
+ }
+}
+
+#[cfg(test)]
+pub(super) mod tests {
+ use std::collections::HashMap;
+
+ use super::MlDb;
+ use crate::ml_db::tests::{
+ cases, check, grouped, grouped_by_file, map, open, set, sorted, strings,
+ };
+ use crate::ml_db::{clusters, faces};
+ use tempfile::TempDir;
+
+ pub(in crate::ml_db) fn seed(db: &MlDb) {
+ db.assign_cluster_to_person("p1", "c1").unwrap();
+ db.assign_cluster_to_person("p1", "c2").unwrap();
+ db.assign_cluster_to_person("p2", "c3").unwrap();
+ db.capture_not_person_feedback("p1", "c3").unwrap();
+ db.capture_not_person_feedback("p1", "c3").unwrap();
+ db.capture_not_person_feedback("p2", "c3").unwrap();
+ db.capture_not_person_feedback("p2", "c1").unwrap();
+ db.bulk_capture_not_person_feedback(&map([("c1", "p2"), ("c2", "p3")]))
+ .unwrap();
+ }
+
+ fn seeded() -> (TempDir, MlDb) {
+ let (directory, db) = open();
+ faces::tests::seed(&db);
+ clusters::tests::seed(&db);
+ seed(&db);
+ (directory, db)
+ }
+
+ #[test]
+ fn seeded_id_sets() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "p1 clusters": set(["c1", "c2"]) => |db| db.get_person_cluster_ids("p1"),
+ "unknown person clusters": set([]) => |db| db.get_person_cluster_ids("p9"),
+ "persons clusters": set(["c1", "c2", "c3"]) =>
+ |db| db.get_persons_cluster_ids(&strings(["p1", "p2"])),
+ "no persons clusters": set([]) => |db| db.get_persons_cluster_ids(&[]),
+ "p1 ignored": set(["c3"]) => |db| db.get_person_ignored_clusters("p1"),
+ "p2 ignored": set(["c1", "c2", "c3"]) => |db| db.get_person_ignored_clusters("p2"),
+ "p3 ignored": set(["c1", "c2", "c3"]) => |db| db.get_person_ignored_clusters("p3"),
+ "rejected thrice": set([]) => |db| db.get_clusters_with_three_or_more_not_person_feedback(),
+ "p1 faces": set(["1_0", "1_1", "2_0"]) => |db| db.get_face_ids_for_person("p1"),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_ordered_lists() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "p1 by score": strings(["1_1", "1_0", "2_0"]) =>
+ |db| db.get_face_ids_for_person_ordered_by_score("p1", 10),
+ "p1 by score limited": strings(["1_1", "1_0"]) =>
+ |db| db.get_face_ids_for_person_ordered_by_score("p1", 2),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "p1 files": vec![1, 2] => |db| db.get_file_ids_of_person_id("p1").map(sorted),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_maps() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "cluster persons": map([("c1", "p1"), ("c2", "p1"), ("c3", "p2")]) =>
+ |db| db.get_cluster_id_to_person_id(),
+ "face persons": map([("1_0", "p1"), ("3_0", "p2")]) =>
+ |db| db.get_face_id_to_person_id_for_faces(&strings(["1_0", "3_0", "4_0", "7_0"])),
+ "no face persons": map([]) => |db| db.get_face_id_to_person_id_for_faces(&[]),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "person clusters": grouped([("p1", &["c1", "c2"]), ("p2", &["c3"])]) =>
+ |db| db.get_person_to_cluster_ids(),
+ "p1 cluster faces": grouped([("c1", &["1_0", "2_0"]), ("c2", &["1_1"])]) =>
+ |db| db.get_cluster_id_to_face_ids_for_person("p1"),
+ "rejected suggestions": grouped([
+ ("p1", &["c3"]), ("p2", &["c3", "c1"]), ("p3", &["c2"]),
+ ]) => |db| db.get_person_to_rejected_suggestions(),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "file clusters for p1": grouped_by_file([(1, &["c1", "c2"]), (2, &["c1"])]) =>
+ |db| db.get_file_id_to_cluster_id_set("p1"),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_nested_faces() {
+ let (_directory, db) = seeded();
+ let nested = db.get_person_to_cluster_id_to_face_ids().unwrap();
+ assert_eq!(nested["p1"]["c1"], set(["1_0", "2_0"]));
+ assert_eq!(nested["p1"]["c2"], set(["1_1"]));
+ assert_eq!(nested["p2"]["c3"], set(["3_0"]));
+ }
+
+ #[test]
+ fn person_and_feedback_mutations() {
+ let (_directory, db) = seeded();
+ db.capture_not_person_feedback("p3", "c3").unwrap();
+ assert_eq!(
+ db.get_clusters_with_three_or_more_not_person_feedback()
+ .unwrap(),
+ set(["c3"])
+ );
+ db.remove_not_person_feedback("p2", "c1").unwrap();
+ assert_eq!(
+ db.get_person_to_rejected_suggestions().unwrap()["p2"],
+ set(["c3"])
+ );
+ assert_eq!(
+ db.get_person_ignored_clusters("p2").unwrap(),
+ set(["c1", "c2", "c3"])
+ );
+
+ db.remove_person("p1").unwrap();
+ assert!(db.get_person_cluster_ids("p1").unwrap().is_empty());
+ assert!(
+ !db.get_person_to_rejected_suggestions()
+ .unwrap()
+ .contains_key("p1")
+ );
+ assert_eq!(db.get_person_cluster_ids("p2").unwrap(), set(["c3"]));
+ db.remove_cluster_to_person("p2", "c3").unwrap();
+ assert!(db.get_person_cluster_ids("p2").unwrap().is_empty());
+
+ db.assign_cluster_to_person("p5", "c3").unwrap();
+ db.assign_cluster_to_person("p5", "c3").unwrap();
+ db.bulk_assign_cluster_to_person_id(&map([("c1", "p5"), ("c2", "p5")]))
+ .unwrap();
+ db.bulk_assign_cluster_to_person_id(&map([("c1", "p5")]))
+ .unwrap();
+ db.bulk_capture_not_person_feedback(&HashMap::new())
+ .unwrap();
+ assert_eq!(
+ db.get_person_cluster_ids("p5").unwrap(),
+ set(["c1", "c2", "c3"])
+ );
+ assert_eq!(db.get_cluster_id_to_person_id().unwrap().len(), 3);
+
+ db.drop_faces_feedback_tables().unwrap();
+ assert!(db.get_person_to_cluster_ids().unwrap().is_empty());
+ assert!(db.get_person_to_rejected_suggestions().unwrap().is_empty());
+ assert_eq!(db.get_total_face_count().unwrap(), 9);
+ assert_eq!(db.cluster_id_to_face_count().unwrap().len(), 6);
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/pets.rs b/rust/crates/photos/src/ml_db/pets.rs
new file mode 100644
index 0000000000..a861f6d489
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/pets.rs
@@ -0,0 +1,499 @@
+use std::collections::HashMap;
+
+use crate::db::{Row, SqliteResult, bind_placeholders, pair, params_from_iter};
+
+use super::queries::{PetBodyRow, PetBodyVectorRow, PetFaceRow, PetFaceVectorRow, PetRowsForFiles};
+use super::{MlDb, Result, non_empty, unique_in_order};
+
+const UPSERT_PET_FACE: &str = "INSERT INTO pet_faces (file_id, pet_face_id, detection, face_vector_id, species, score, height, width, ml_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(file_id, pet_face_id) DO UPDATE SET detection = excluded.detection, face_vector_id = excluded.face_vector_id, species = excluded.species, score = excluded.score, height = excluded.height, width = excluded.width, ml_version = excluded.ml_version";
+const UPSERT_PET_BODY: &str = "INSERT INTO pet_bodies (file_id, pet_body_id, detection, body_vector_id, species, score, height, width, ml_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(file_id, pet_body_id) DO UPDATE SET detection = excluded.detection, body_vector_id = excluded.body_vector_id, species = excluded.species, score = excluded.score, height = excluded.height, width = excluded.width, ml_version = excluded.ml_version";
+
+impl MlDb {
+ pub fn bulk_insert_pet_faces(&self, pet_faces: &[PetFaceRow]) -> Result<()> {
+ self.db
+ .write_in_batches(
+ UPSERT_PET_FACE,
+ 500,
+ pet_faces.iter().map(|pet_face| {
+ (
+ pet_face.file_id,
+ &pet_face.pet_face_id,
+ &pet_face.detection_json,
+ pet_face.face_vector_id,
+ pet_face.species,
+ pet_face.face_score,
+ pet_face.image_height,
+ pet_face.image_width,
+ pet_face.ml_version,
+ )
+ }),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn bulk_insert_pet_bodies(&self, pet_bodies: &[PetBodyRow]) -> Result<()> {
+ self.db
+ .write_in_batches(
+ UPSERT_PET_BODY,
+ 500,
+ pet_bodies.iter().map(|pet_body| {
+ (
+ pet_body.file_id,
+ &pet_body.pet_body_id,
+ &pet_body.detection_json,
+ pet_body.body_vector_id,
+ pet_body.species,
+ pet_body.score,
+ pet_body.image_height,
+ pet_body.image_width,
+ pet_body.ml_version,
+ )
+ }),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn update_pet_face_vector_ids(
+ &self,
+ pet_face_id_to_vector_id: &HashMap<String, i64>,
+ ) -> Result<()> {
+ if pet_face_id_to_vector_id.is_empty() {
+ return Ok(());
+ }
+ self.db
+ .write_in_batches(
+ "UPDATE pet_faces SET face_vector_id = ? WHERE pet_face_id = ?",
+ 500,
+ pet_face_id_to_vector_id
+ .iter()
+ .map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn update_pet_body_vector_ids(
+ &self,
+ pet_body_id_to_vector_id: &HashMap<String, i64>,
+ ) -> Result<()> {
+ if pet_body_id_to_vector_id.is_empty() {
+ return Ok(());
+ }
+ self.db
+ .write_in_batches(
+ "UPDATE pet_bodies SET body_vector_id = ? WHERE pet_body_id = ?",
+ 500,
+ pet_body_id_to_vector_id
+ .iter()
+ .map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_pet_faces_for_file_id(
+ &self,
+ file_upload_id: i64,
+ ) -> Result<Option<Vec<PetFaceRow>>> {
+ let pet_faces = self.db.read_all(
+ "SELECT * FROM pet_faces WHERE file_id = ? AND species != -1",
+ [file_upload_id],
+ read_pet_face,
+ )?;
+ Ok(non_empty(pet_faces))
+ }
+
+ pub fn get_pet_bodies_for_file_id(
+ &self,
+ file_upload_id: i64,
+ ) -> Result<Option<Vec<PetBodyRow>>> {
+ let pet_bodies = self.db.read_all(
+ "SELECT * FROM pet_bodies WHERE file_id = ? AND species != -1",
+ [file_upload_id],
+ read_pet_body,
+ )?;
+ Ok(non_empty(pet_bodies))
+ }
+
+ pub fn pet_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {
+ self.db
+ .read_all(
+ "SELECT DISTINCT file_id, ml_version FROM pet_faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ pair,
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_pet_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
+ self.db
+ .read_value(
+ "SELECT COUNT(DISTINCT file_id) as count FROM pet_faces WHERE ml_version >= ?",
+ [minimum_ml_version],
+ )
+ .map_err(Into::into)
+ }
+
+ pub fn get_pet_rows_for_files(&self, file_ids: &[i64]) -> Result<PetRowsForFiles> {
+ let placeholders = bind_placeholders(file_ids.len());
+ let faces_sql = format!(
+ "SELECT pet_face_id, face_vector_id, species FROM pet_faces WHERE file_id IN ({placeholders})"
+ );
+ let bodies_sql = format!(
+ "SELECT pet_body_id, body_vector_id, species FROM pet_bodies WHERE file_id IN ({placeholders})"
+ );
+ let faces = self
+ .db
+ .read_all(&faces_sql, params_from_iter(file_ids), |row| {
+ Ok(PetFaceVectorRow {
+ pet_face_id: row.get(0)?,
+ face_vector_id: row.get(1)?,
+ species: row.get(2)?,
+ })
+ })?;
+ let bodies = self
+ .db
+ .read_all(&bodies_sql, params_from_iter(file_ids), |row| {
+ Ok(PetBodyVectorRow {
+ pet_body_id: row.get(0)?,
+ body_vector_id: row.get(1)?,
+ species: row.get(2)?,
+ })
+ })?;
+ Ok(PetRowsForFiles { faces, bodies })
+ }
+
+ pub fn delete_pet_rows_for_files(
+ &self,
+ file_ids: &[i64],
+ pet_face_ids: &[String],
+ pet_body_ids: &[String],
+ ) -> Result<()> {
+ let placeholders = bind_placeholders(file_ids.len());
+ self.db
+ .write_transaction(|transaction| {
+ if !pet_face_ids.is_empty() {
+ transaction.execute(
+ &format!(
+ "DELETE FROM pet_face_vector_id_map WHERE pet_face_id IN ({})",
+ bind_placeholders(pet_face_ids.len())
+ ),
+ params_from_iter(pet_face_ids.iter()),
+ )?;
+ }
+ if !pet_body_ids.is_empty() {
+ transaction.execute(
+ &format!(
+ "DELETE FROM pet_body_vector_id_map WHERE pet_body_id IN ({})",
+ bind_placeholders(pet_body_ids.len())
+ ),
+ params_from_iter(pet_body_ids.iter()),
+ )?;
+ }
+ transaction.execute(
+ &format!("DELETE FROM pet_faces WHERE file_id IN ({placeholders})"),
+ params_from_iter(file_ids.iter()),
+ )?;
+ transaction.execute(
+ &format!("DELETE FROM pet_bodies WHERE file_id IN ({placeholders})"),
+ params_from_iter(file_ids.iter()),
+ )?;
+ Ok(())
+ })
+ .map_err(Into::into)
+ }
+
+ pub fn get_pet_face_vector_id_map(
+ &self,
+ pet_face_ids: &[String],
+ create_if_missing: bool,
+ ) -> Result<HashMap<String, i64>> {
+ self.vector_id_map(
+ pet_face_ids,
+ create_if_missing,
+ "INSERT OR IGNORE INTO pet_face_vector_id_map (pet_face_id) VALUES (?)",
+ "SELECT pet_face_id, pet_face_vector_id FROM pet_face_vector_id_map WHERE pet_face_id IN",
+ )
+ }
+
+ pub fn get_pet_body_vector_id_map(
+ &self,
+ pet_body_ids: &[String],
+ create_if_missing: bool,
+ ) -> Result<HashMap<String, i64>> {
+ self.vector_id_map(
+ pet_body_ids,
+ create_if_missing,
+ "INSERT OR IGNORE INTO pet_body_vector_id_map (pet_body_id) VALUES (?)",
+ "SELECT pet_body_id, pet_body_vector_id FROM pet_body_vector_id_map WHERE pet_body_id IN",
+ )
+ }
+
+ fn vector_id_map(
+ &self,
+ ids: &[String],
+ create_if_missing: bool,
+ insert_sql: &str,
+ select_prefix: &str,
+ ) -> Result<HashMap<String, i64>> {
+ let unique_ids = unique_in_order(ids);
+ if unique_ids.is_empty() {
+ return Ok(HashMap::new());
+ }
+ if create_if_missing {
+ self.db
+ .write_batch(insert_sql, unique_ids.iter().map(|id| [id]))?;
+ }
+ let select_sql = select_prefix.to_owned() + " ({})";
+ self.db
+ .read_chunked_in(&select_sql, &unique_ids, 800, pair)
+ .map_err(Into::into)
+ }
+}
+
+fn read_pet_face(row: &Row<'_>) -> SqliteResult<PetFaceRow> {
+ Ok(PetFaceRow {
+ file_id: row.get("file_id")?,
+ pet_face_id: row.get("pet_face_id")?,
+ detection_json: row.get("detection")?,
+ face_vector_id: row.get("face_vector_id")?,
+ species: row.get("species")?,
+ face_score: row.get("score")?,
+ image_height: row.get("height")?,
+ image_width: row.get("width")?,
+ ml_version: row.get("ml_version")?,
+ })
+}
+
+fn read_pet_body(row: &Row<'_>) -> SqliteResult<PetBodyRow> {
+ Ok(PetBodyRow {
+ file_id: row.get("file_id")?,
+ pet_body_id: row.get("pet_body_id")?,
+ detection_json: row.get("detection")?,
+ body_vector_id: row.get("body_vector_id")?,
+ species: row.get("species")?,
+ score: row.get("score")?,
+ image_height: row.get("height")?,
+ image_width: row.get("width")?,
+ ml_version: row.get("ml_version")?,
+ })
+}
+
+#[cfg(test)]
+pub(super) mod tests {
+ use std::collections::HashMap;
+
+ use super::{MlDb, PetBodyRow, PetFaceRow};
+ use crate::ml_db::tests::{cases, check, open, pairs, sorted, strings};
+ use tempfile::TempDir;
+
+ pub(in crate::ml_db) fn pet_face(file_id: i64, index: i64, species: i64) -> PetFaceRow {
+ PetFaceRow {
+ file_id,
+ pet_face_id: format!("{file_id}_pet_{index}"),
+ detection_json: "{}".to_string(),
+ face_vector_id: None,
+ species,
+ face_score: 0.7,
+ image_height: 10,
+ image_width: 20,
+ ml_version: 1,
+ }
+ }
+
+ fn pet_body(file_id: i64, index: i64, species: i64) -> PetBodyRow {
+ PetBodyRow {
+ file_id,
+ pet_body_id: format!("{file_id}_body_{index}"),
+ detection_json: "{}".to_string(),
+ body_vector_id: None,
+ species,
+ score: 0.6,
+ image_height: 10,
+ image_width: 20,
+ ml_version: 1,
+ }
+ }
+
+ pub(in crate::ml_db) fn seed(db: &MlDb) {
+ db.bulk_insert_pet_faces(&[
+ pet_face(1, 0, 0),
+ pet_face(1, 1, -1),
+ PetFaceRow {
+ ml_version: 2,
+ ..pet_face(2, 0, 1)
+ },
+ ])
+ .unwrap();
+ db.bulk_insert_pet_bodies(&[pet_body(1, 0, 0), pet_body(3, 0, -1)])
+ .unwrap();
+ let face_vector_ids = db
+ .get_pet_face_vector_id_map(&strings(["1_pet_0", "1_pet_1"]), true)
+ .unwrap();
+ db.update_pet_face_vector_ids(&face_vector_ids).unwrap();
+ let body_vector_ids = db
+ .get_pet_body_vector_id_map(&strings(["1_body_0"]), true)
+ .unwrap();
+ db.update_pet_body_vector_ids(&body_vector_ids).unwrap();
+ }
+
+ fn seeded() -> (TempDir, MlDb) {
+ let (directory, db) = open();
+ seed(&db);
+ (directory, db)
+ }
+
+ #[test]
+ fn seeded_counts() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "pet files": 2 => |db| db.get_pet_indexed_file_count(1),
+ "pet files v2": 1 => |db| db.get_pet_indexed_file_count(2),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_maps() {
+ let (_directory, db) = seeded();
+ check(
+ &db,
+ &cases![
+ "pet face vector ids": pairs([("1_pet_0", 1), ("1_pet_1", 2)]) =>
+ |db| db.get_pet_face_vector_id_map(&strings(["1_pet_0", "1_pet_1", "2_pet_0"]), false),
+ "pet body vector ids": pairs([("1_body_0", 1)]) =>
+ |db| db.get_pet_body_vector_id_map(&strings(["1_body_0"]), false),
+ ],
+ );
+ check(
+ &db,
+ &cases![
+ "pet versions": HashMap::from([(1, 1), (2, 2)]) => |db| db.pet_indexed_file_ids(1),
+ "pet versions v2": HashMap::from([(2, 2)]) => |db| db.pet_indexed_file_ids(2),
+ ],
+ );
+ }
+
+ #[test]
+ fn seeded_rows() {
+ let (_directory, db) = seeded();
+ let linked_pet_face = PetFaceRow {
+ face_vector_id: Some(1),
+ ..pet_face(1, 0, 0)
+ };
+ assert_eq!(
+ db.get_pet_faces_for_file_id(1).unwrap(),
+ Some(vec![linked_pet_face])
+ );
+ let new_pet_face = PetFaceRow {
+ ml_version: 2,
+ ..pet_face(2, 0, 1)
+ };
+ assert_eq!(
+ db.get_pet_faces_for_file_id(2).unwrap(),
+ Some(vec![new_pet_face])
+ );
+ assert_eq!(db.get_pet_faces_for_file_id(9).unwrap(), None);
+ let linked_pet_body = PetBodyRow {
+ body_vector_id: Some(1),
+ ..pet_body(1, 0, 0)
+ };
+ assert_eq!(
+ db.get_pet_bodies_for_file_id(1).unwrap(),
+ Some(vec![linked_pet_body])
+ );
+ assert_eq!(db.get_pet_bodies_for_file_id(3).unwrap(), None);
+
+ let rows = db.get_pet_rows_for_files(&[1, 3]).unwrap();
+ let faces: Vec<(&str, Option<i64>, i64)> = rows
+ .faces
+ .iter()
+ .map(|row| (row.pet_face_id.as_str(), row.face_vector_id, row.species))
+ .collect();
+ assert_eq!(
+ sorted(faces),
+ [("1_pet_0", Some(1), 0), ("1_pet_1", Some(2), -1)]
+ );
+ let bodies: Vec<(&str, Option<i64>, i64)> = rows
+ .bodies
+ .iter()
+ .map(|row| (row.pet_body_id.as_str(), row.body_vector_id, row.species))
+ .collect();
+ assert_eq!(
+ sorted(bodies),
+ [("1_body_0", Some(1), 0), ("3_body_0", None, -1)]
+ );
+ }
+
+ #[test]
+ fn pet_upsert_vector_ids_and_deletion() {
+ let (_directory, db) = seeded();
+ let updated = PetFaceRow {
+ face_score: 0.99,
+ ..pet_face(2, 0, 1)
+ };
+ db.bulk_insert_pet_faces(std::slice::from_ref(&updated))
+ .unwrap();
+ assert_eq!(
+ db.get_pet_faces_for_file_id(2).unwrap(),
+ Some(vec![updated])
+ );
+ db.bulk_insert_pet_bodies(&[pet_body(2, 0, 1)]).unwrap();
+ assert_eq!(
+ db.get_pet_bodies_for_file_id(2).unwrap(),
+ Some(vec![pet_body(2, 0, 1)])
+ );
+
+ let face_vector_ids = db
+ .get_pet_face_vector_id_map(&strings(["2_pet_0", "1_pet_0", "2_pet_0"]), true)
+ .unwrap();
+ assert_eq!(face_vector_ids, pairs([("1_pet_0", 1), ("2_pet_0", 3)]));
+ assert_eq!(
+ db.get_pet_body_vector_id_map(&strings(["2_body_0"]), true)
+ .unwrap(),
+ pairs([("2_body_0", 2)])
+ );
+ db.update_pet_face_vector_ids(&HashMap::new()).unwrap();
+ db.update_pet_body_vector_ids(&HashMap::new()).unwrap();
+ db.update_pet_face_vector_ids(&face_vector_ids).unwrap();
+ assert_eq!(
+ db.get_pet_faces_for_file_id(2).unwrap().unwrap()[0].face_vector_id,
+ Some(3)
+ );
+
+ db.delete_pet_rows_for_files(
+ &[1],
+ &strings(["1_pet_0", "1_pet_1"]),
+ &strings(["1_body_0"]),
+ )
+ .unwrap();
+ assert_eq!(db.get_pet_faces_for_file_id(1).unwrap(), None);
+ assert_eq!(db.get_pet_bodies_for_file_id(1).unwrap(), None);
+ assert!(
+ db.get_pet_face_vector_id_map(&strings(["1_pet_0", "1_pet_1"]), false)
+ .unwrap()
+ .is_empty()
+ );
+ assert!(
+ db.get_pet_body_vector_id_map(&strings(["1_body_0"]), false)
+ .unwrap()
+ .is_empty()
+ );
+ db.delete_pet_rows_for_files(&[2], &[], &[]).unwrap();
+ assert_eq!(db.get_pet_faces_for_file_id(2).unwrap(), None);
+ assert_eq!(db.get_pet_bodies_for_file_id(2).unwrap(), None);
+ assert_eq!(
+ db.get_pet_face_vector_id_map(&strings(["2_pet_0"]), false)
+ .unwrap(),
+ pairs([("2_pet_0", 3)])
+ );
+
+ db.clear_pet_tables().unwrap();
+ assert!(
+ db.get_pet_face_vector_id_map(&strings(["2_pet_0"]), false)
+ .unwrap()
+ .is_empty()
+ );
+ assert_eq!(db.get_pet_indexed_file_count(0).unwrap(), 0);
+ }
+}
diff --git a/rust/crates/photos/src/ml_db/queries.rs b/rust/crates/photos/src/ml_db/queries.rs
index 499d97a95c..0a527e1063 100644
--- a/rust/crates/photos/src/ml_db/queries.rs
+++ b/rust/crates/photos/src/ml_db/queries.rs
@@ -263,9 +263,9 @@ pub(super) mod tests {
use super::{CLIP_EMBEDDING_DIMENSIONS, ClipEmbedding, ClipRow, MlDb};
use crate::db::Connection;
- use crate::ml_db::tests::{cases, check, open};
+ use crate::ml_db::tests::{cases, check, open, strings};
use crate::ml_db::vector_encoding::encode_f32;
- use crate::ml_db::{caches, filedata};
+ use crate::ml_db::{caches, clusters, faces, filedata, persons, pets};
use tempfile::TempDir;
fn clip(file_id: i64, embedding: Vec<f64>) -> ClipEmbedding {
@@ -280,7 +280,7 @@ pub(super) mod tests {
clip(file_id, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])
}
- fn seed(db: &MlDb) {
+ pub(in crate::ml_db) fn seed(db: &MlDb) {
db.insert_clip_rows(&[
full_clip(1),
ClipEmbedding {
@@ -303,6 +303,10 @@ pub(super) mod tests {
let (directory, db) = seeded();
filedata::tests::seed(&db);
caches::tests::seed(&db);
+ faces::tests::seed(&db);
+ clusters::tests::seed(&db);
+ persons::tests::seed(&db);
+ pets::tests::seed(&db);
(directory, db)
}
@@ -393,8 +397,19 @@ pub(super) mod tests {
fn clear_non_pet_tables_leaves_pets_and_caches() {
let (_directory, db) = seeded_all();
db.clear_non_pet_tables().unwrap();
+ assert_eq!(db.get_total_face_count().unwrap(), 0);
+ assert!(db.cluster_id_to_face_count().unwrap().is_empty());
+ assert!(db.get_person_to_cluster_ids().unwrap().is_empty());
+ assert!(db.get_person_to_rejected_suggestions().unwrap().is_empty());
+ assert_eq!(db.count_cluster_summaries().unwrap(), 0);
+ assert!(
+ db.get_cluster_centroid_vector_id_map(&strings(["c1"]), false)
+ .unwrap()
+ .is_empty()
+ );
assert_eq!(db.count_clip_rows().unwrap(), 0);
assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());
+ assert_eq!(db.get_pet_indexed_file_count(1).unwrap(), 2);
assert_eq!(
db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
Some("1_0".to_string())
@@ -409,13 +424,6 @@ pub(super) mod tests {
fn clear_pet_tables_leaves_non_pet_tables() {
let (directory, db) = seeded_all();
let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();
- connection
- .execute_batch(
- "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');
- INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",
- )
- .unwrap();
- db.clear_pet_tables().unwrap();
let row_count = |table: &str| -> i64 {
connection
.query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {
@@ -423,8 +431,19 @@ pub(super) mod tests {
})
.unwrap()
};
+ assert_eq!(row_count("pet_face_vector_id_map"), 2);
+ assert_eq!(row_count("pet_body_vector_id_map"), 1);
+ db.clear_pet_tables().unwrap();
assert_eq!(row_count("pet_face_vector_id_map"), 0);
assert_eq!(row_count("pet_body_vector_id_map"), 0);
+ assert_eq!(db.get_pet_indexed_file_count(0).unwrap(), 0);
+ assert_eq!(
+ db.get_pet_rows_for_files(&[1, 2, 3]).unwrap().faces.len(),
+ 0
+ );
+ assert_eq!(db.get_total_face_count().unwrap(), 9);
+ assert_eq!(db.cluster_id_to_face_count().unwrap().len(), 6);
+ assert_eq!(db.get_person_cluster_ids("p1").unwrap().len(), 2);
assert_eq!(db.count_clip_rows().unwrap(), 4);
assert_eq!(
db.get_file_ids_with_fd_data(None).unwrap(),