Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][photos] Simplify ML database helpers

laurenspriem3231428131d4
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index d74de67e2f..a365e579f5 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -13,17 +13,12 @@ pub use queries::clip::{
ClipRow, EmbeddingVector,
};
pub use queries::clusters::{ClusterCentroidRow, ClusterSummary};
-pub use queries::faces::{
- FACE_ML_VERSION, FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding,
- LAPLACIAN_HARD_THRESHOLD, LAPLACIAN_SOFT_THRESHOLD, LAPLACIAN_VERY_SOFT_THRESHOLD,
- MEDIUM_QUALITY_FACE_SCORE, MINIMUM_QUALITY_FACE_SCORE, is_bad_face_for_clustering,
-};
+pub use queries::faces::{FACE_ML_VERSION, FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding};
pub use queries::filedata::{FdStatus, PreviewInfo};
pub use queries::persons::PersonToClusterIdToFaceIds;
pub use queries::pets::{
PET_ML_VERSION, PetBodyRow, PetBodyVectorRow, PetFaceRow, PetFaceVectorRow, PetRowsForFiles,
};
-pub use vector_encoding::{decode_evector, decode_f32, encode_evector, encode_f32};
#[derive(Debug, thiserror::Error)]
pub enum Error {
@@ -33,8 +28,6 @@ pub enum Error {
Downgrade { current: i64, target: i64 },
#[error("{0}")]
Codec(String),
- #[error(transparent)]
- Io(#[from] std::io::Error),
#[error("{0}")]
InvalidArgument(String),
}
@@ -76,7 +69,6 @@ impl MlDb {
schema::DELETE_CLIP_EMBEDDINGS,
schema::DELETE_FILE_DATA,
])
- .map_err(Into::into)
}
pub fn clear_pet_tables(&self) -> Result<()> {
@@ -86,7 +78,6 @@ impl MlDb {
schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
])
- .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/caches.rs b/rust/crates/photos/src/ml_db/queries/caches.rs
index 946276d174..af298e6208 100644
--- a/rust/crates/photos/src/ml_db/queries/caches.rs
+++ b/rust/crates/photos/src/ml_db/queries/caches.rs
@@ -20,7 +20,7 @@ impl MlDb {
}
pub fn get_repeated_text_embedding_cache(&self, query: &str) -> Result<Option<Vec<f32>>> {
- let results: Vec<(Vec<u8>, i64, i64)> = self.read_all(
+ let cached: Option<(Vec<u8>, i64, i64)> = self.read_optional(
r#"
SELECT embedding, ml_version, created_at
FROM text_embeddings_cache
@@ -29,14 +29,12 @@ impl MlDb {
[query],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)?;
- if results.is_empty() {
+ let Some((embedding, ml_version, created_at)) = cached else {
return Ok(None);
- }
+ };
let three_months_ago = now_millis() - THREE_MONTHS_MILLIS;
- for (embedding, ml_version, created_at) in &results {
- if *ml_version == CLIP_ML_VERSION && *created_at > three_months_ago {
- return Ok(Some(decode_f32(embedding)));
- }
+ if ml_version == CLIP_ML_VERSION && created_at > three_months_ago {
+ return Ok(Some(decode_f32(&embedding)));
}
self.execute(
"DELETE FROM text_embeddings_cache WHERE text_query = ?",
@@ -64,8 +62,8 @@ impl MlDb {
self.read_optional(
"SELECT face_id FROM face_cache WHERE person_or_cluster_id = ?",
[person_or_cluster_id],
+ |row| Ok(row.get(0)?),
)
- .map_err(Into::into)
}
pub fn remove_face_id_cached_for_person_or_cluster(
diff --git a/rust/crates/photos/src/ml_db/queries/clip.rs b/rust/crates/photos/src/ml_db/queries/clip.rs
index aa6c28dd47..0001693f0b 100644
--- a/rust/crates/photos/src/ml_db/queries/clip.rs
+++ b/rust/crates/photos/src/ml_db/queries/clip.rs
@@ -45,7 +45,6 @@ impl MlDb {
pub fn clip_indexed_file_with_version(&self) -> Result<HashMap<i64, i64>> {
self.read_all("SELECT file_id , ml_version FROM clip", (), pair)
- .map_err(Into::into)
}
pub fn get_clip_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
@@ -53,7 +52,6 @@ impl MlDb {
"SELECT COUNT(DISTINCT file_id) as count FROM clip WHERE ml_version >= ?",
[minimum_ml_version],
)
- .map_err(Into::into)
}
pub fn get_clip_vectorizable_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
@@ -66,7 +64,6 @@ impl MlDb {
"#,
[minimum_ml_version, CLIP_EMBEDDING_BYTES_LENGTH],
)
- .map_err(Into::into)
}
pub fn insert_clip_rows(&self, embeddings: &[ClipEmbedding]) -> Result<()> {
@@ -81,22 +78,18 @@ impl MlDb {
"INSERT OR REPLACE INTO clip (file_id, embedding, ml_version) values(?, ?, ?)",
embeddings.iter().map(clip_row),
)
- .map_err(Into::into)
}
pub fn delete_clip_rows(&self, file_ids: &[i64]) -> Result<()> {
self.execute_chunked_in("DELETE FROM clip WHERE file_id IN ({})", file_ids)
- .map_err(Into::into)
}
pub fn delete_all_clip_rows(&self) -> Result<()> {
self.execute_statements([schema::DELETE_CLIP_EMBEDDINGS])
- .map_err(Into::into)
}
pub fn count_clip_rows(&self) -> Result<i64> {
self.read_value("SELECT COUNT(file_id) as total FROM clip", ())
- .map_err(Into::into)
}
pub fn get_clip_rows_page(&self, limit: i64, offset: i64) -> Result<Vec<ClipRow>> {
@@ -110,7 +103,6 @@ impl MlDb {
})
},
)
- .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index 73e29636d1..d8c58f2392 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -53,7 +53,6 @@ impl MlDb {
const { NonZeroUsize::new(500).unwrap() },
face_id_to_cluster_id.iter(),
)
- .map_err(Into::into)
}
pub fn cluster_id_to_face_count(&self) -> Result<HashMap<String, i64>> {
@@ -67,7 +66,6 @@ impl MlDb {
(),
pair,
)
- .map_err(Into::into)
}
pub fn get_bad_face_singleton_cluster_ids(&self) -> Result<HashSet<String>> {
@@ -113,13 +111,12 @@ impl MlDb {
self.read_optional(
"SELECT cluster_id FROM face_clusters WHERE face_id = ?",
[face_id],
+ |row| Ok(row.get(0)?),
)
- .map_err(Into::into)
}
pub fn get_all_cluster_id_to_face_ids(&self) -> Result<HashMap<String, Vec<String>>> {
self.read_grouped("SELECT cluster_id, face_id FROM face_clusters", ())
- .map_err(Into::into)
}
pub fn get_face_ids_for_cluster(&self, cluster_id: &str) -> Result<Vec<String>> {
@@ -127,7 +124,6 @@ impl MlDb {
"SELECT face_id FROM face_clusters WHERE face_clusters.cluster_id = ?",
[cluster_id],
)
- .map_err(Into::into)
}
pub fn get_face_ids_for_cluster_ordered_by_score(
@@ -147,7 +143,6 @@ impl MlDb {
"#,
(cluster_id, limit),
)
- .map_err(Into::into)
}
pub fn get_blur_values_for_cluster(&self, cluster_id: &str) -> Result<Vec<f64>> {
@@ -178,12 +173,10 @@ impl MlDb {
const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)
- .map_err(Into::into)
}
pub fn get_file_id_to_cluster_ids(&self) -> Result<HashMap<i64, HashSet<String>>> {
self.read_all("SELECT cluster_id, face_id FROM face_clusters", (), pair)
- .map_err(Into::into)
.and_then(file_id_to_cluster_ids)
}
@@ -192,7 +185,6 @@ impl MlDb {
face_id_to_cluster_id: &HashMap<String, String>,
) -> Result<()> {
self.write_batch_atomic(UPSERT_FACE_CLUSTER, face_id_to_cluster_id.iter())
- .map_err(Into::into)
}
pub fn remove_face_id_to_cluster_id(
@@ -203,7 +195,6 @@ impl MlDb {
"DELETE FROM face_clusters WHERE face_id = ? AND cluster_id = ?",
face_id_to_cluster_id.iter(),
)
- .map_err(Into::into)
}
pub fn get_file_id_to_cluster_id_set_for_cluster(
@@ -215,7 +206,6 @@ impl MlDb {
bind_placeholders(cluster_ids.len())
);
self.read_all(&sql, params_from_iter(cluster_ids), pair)
- .map_err(Into::into)
.and_then(file_id_to_cluster_ids)
}
@@ -230,7 +220,6 @@ impl MlDb {
"#,
[cluster_id],
)
- .map_err(Into::into)
}
pub fn get_cluster_centroid_vector_id_map(
@@ -258,7 +247,6 @@ impl MlDb {
const { NonZeroUsize::new(800).unwrap() },
pair,
)
- .map_err(Into::into)
}
pub fn delete_cluster_centroid_vector_id_mapping(&self, cluster_id: &str) -> Result<()> {
@@ -271,7 +259,6 @@ impl MlDb {
pub fn clear_cluster_centroid_vector_id_mappings(&self) -> Result<()> {
self.execute_statements([DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING])
- .map_err(Into::into)
}
pub fn upsert_cluster_summary_rows(
@@ -285,7 +272,6 @@ impl MlDb {
.iter()
.map(|(cluster_id, summary)| (cluster_id, &summary.avg, summary.count)),
)
- .map_err(Into::into)
}
pub fn delete_cluster_summary_row(&self, cluster_id: &str) -> Result<()> {
@@ -313,7 +299,6 @@ impl MlDb {
optional_parameter(&min_cluster_size).as_slice(),
read_cluster_summary,
)
- .map_err(Into::into)
}
pub fn get_cluster_to_cluster_summary(
@@ -325,12 +310,10 @@ impl MlDb {
bind_placeholders(cluster_ids.len())
);
self.read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary)
- .map_err(Into::into)
}
pub fn count_cluster_summaries(&self) -> Result<i64> {
self.read_value("SELECT COUNT(cluster_id) as total FROM cluster_summary", ())
- .map_err(Into::into)
}
pub fn get_cluster_summary_page(
@@ -356,7 +339,6 @@ impl MlDb {
read_cluster_centroid,
),
}
- .map_err(Into::into)
}
pub fn reset_cluster_tables(&self, faces: bool) -> Result<()> {
@@ -384,7 +366,6 @@ impl MlDb {
CREATE_FACE_CLUSTERS_TABLE,
FC_CLUSTER_ID_INDEX,
]))
- .map_err(Into::into)
}
pub fn get_clusters_for_memory_lane(
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index cdf50c527c..e7a09ac9f8 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -14,11 +14,11 @@ use crate::ml_db::{Error, MlDb, Result};
pub const FACE_ML_VERSION: i64 = 1;
-pub const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;
-pub const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;
-pub const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;
-pub const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;
-pub const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;
+const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;
+const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;
+const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;
+const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;
+const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;
#[derive(Clone, Debug, PartialEq)]
pub struct FaceRow {
@@ -93,7 +93,6 @@ impl MlDb {
)
}),
)
- .map_err(Into::into)
}
pub fn face_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {
@@ -102,7 +101,6 @@ impl MlDb {
[minimum_ml_version],
pair,
)
- .map_err(Into::into)
}
pub fn get_face_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
@@ -110,7 +108,6 @@ impl MlDb {
"SELECT COUNT(DISTINCT file_id) as count FROM faces WHERE ml_version >= ?",
[minimum_ml_version],
)
- .map_err(Into::into)
}
pub fn get_face_embeddings_for_cluster(
@@ -133,7 +130,6 @@ impl MlDb {
let mut parameters: Vec<&dyn ToSql> = vec![&cluster_id];
parameters.extend(optional_parameter(&limit));
self.read_column(&sql, parameters.as_slice())
- .map_err(Into::into)
}
pub fn get_face_embeddings_for_clusters(
@@ -372,17 +368,14 @@ impl MlDb {
"SELECT COUNT(*) as count FROM faces WHERE score > ? AND blur > ?",
(MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
)
- .map_err(Into::into)
}
pub fn get_errored_face_count(&self) -> Result<i64> {
self.read_value("SELECT COUNT(*) as count FROM faces WHERE score < 0", ())
- .map_err(Into::into)
}
pub fn get_errored_file_ids(&self) -> Result<HashSet<i64>> {
self.read_column("SELECT DISTINCT file_id FROM faces WHERE score < 0", ())
- .map_err(Into::into)
}
pub fn prune_resolved_face_error_results(&self, file_ids: &[i64]) -> Result<()> {
@@ -404,7 +397,6 @@ impl MlDb {
"#,
file_ids,
)
- .map_err(Into::into)
}
pub fn get_file_ids_with_error_results(&self, file_ids: &[i64]) -> Result<HashSet<i64>> {
@@ -450,7 +442,6 @@ impl MlDb {
pub fn delete_face_index_for_files(&self, file_ids: &[i64]) -> Result<()> {
self.execute_chunked_in("DELETE FROM faces WHERE file_id IN ({})", file_ids)
- .map_err(Into::into)
}
pub fn delete_unclustered_face_index_for_files(&self, file_ids: &[i64]) -> Result<()> {
@@ -469,7 +460,6 @@ impl MlDb {
"#,
file_ids,
)
- .map_err(Into::into)
}
pub fn get_clustered_or_faceless_file_count(&self) -> Result<i64> {
@@ -505,7 +495,6 @@ impl MlDb {
"#,
(MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
)
- .map_err(Into::into)
}
pub fn get_all_file_ids_of_face_ids_not_in_any_cluster(&self) -> Result<HashSet<i64>> {
@@ -519,7 +508,6 @@ impl MlDb {
"#,
(),
)
- .map_err(Into::into)
}
pub fn get_all_files_associated_with_all_clusters(
@@ -538,7 +526,6 @@ impl MlDb {
bind_placeholders(except_clusters.len())
);
self.read_column(&sql, params_from_iter(except_clusters))
- .map_err(Into::into)
}
pub fn get_fully_indexed_file_ids(&self, include_pets: bool) -> Result<HashSet<i64>> {
@@ -559,11 +546,14 @@ impl MlDb {
parameters.push(PET_ML_VERSION);
}
self.read_column(&sql, params_from_iter(parameters))
- .map_err(Into::into)
}
}
-pub fn is_bad_face_for_clustering(face_score: f64, blur_value: f64, is_sideways: bool) -> bool {
+pub(super) fn is_bad_face_for_clustering(
+ face_score: f64,
+ blur_value: f64,
+ is_sideways: bool,
+) -> bool {
face_score < MINIMUM_QUALITY_FACE_SCORE
|| blur_value < LAPLACIAN_SOFT_THRESHOLD
|| (blur_value < LAPLACIAN_VERY_SOFT_THRESHOLD && face_score < MEDIUM_QUALITY_FACE_SCORE)
diff --git a/rust/crates/photos/src/ml_db/queries/filedata.rs b/rust/crates/photos/src/ml_db/queries/filedata.rs
index 31b1ad4ab8..938d8dd3e0 100644
--- a/rust/crates/photos/src/ml_db/queries/filedata.rs
+++ b/rust/crates/photos/src/ml_db/queries/filedata.rs
@@ -44,7 +44,6 @@ impl MlDb {
)
}),
)
- .map_err(Into::into)
}
pub fn get_file_ids_vid_preview(&self) -> Result<HashMap<i64, PreviewInfo>> {
@@ -61,7 +60,6 @@ impl MlDb {
))
},
)
- .map_err(Into::into)
}
pub fn get_file_ids_with_fd_data(&self, data_type: Option<&str>) -> Result<HashSet<i64>> {
@@ -70,7 +68,6 @@ impl MlDb {
Some(_) => "SELECT file_id FROM filedata WHERE type = ?",
};
self.read_column(sql, optional_parameter(&data_type).as_slice())
- .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rs
index 83561630e3..3af6c6634a 100644
--- a/rust/crates/photos/src/ml_db/queries/helpers.rs
+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs
@@ -2,9 +2,9 @@ use std::collections::HashMap;
use std::hash::Hash;
use std::num::NonZeroUsize;
-use crate::db::{FromSql, Params, Result, Row, ToSql, TransactionBehavior, params_from_iter};
+use crate::db::{self, FromSql, Params, Row, ToSql, TransactionBehavior, params_from_iter};
-use crate::ml_db::MlDb;
+use crate::ml_db::{MlDb, Result};
pub(super) const MAX_SQL_BIND_PARAMS_PER_QUERY: usize = 10000;
@@ -13,13 +13,15 @@ impl MlDb {
&self,
sql: &str,
parameters: P,
- map: impl FnMut(&Row<'_>) -> Result<T>,
+ map: impl FnMut(&Row<'_>) -> db::Result<T>,
) -> Result<C> {
- self.db.read(|connection| {
- let mut statement = connection.prepare_cached(sql)?;
- let rows = statement.query_map(parameters, map)?;
- rows.collect::<Result<C>>()
- })
+ self.db
+ .read(|connection| {
+ let mut statement = connection.prepare_cached(sql)?;
+ let rows = statement.query_map(parameters, map)?;
+ rows.collect::<db::Result<C>>()
+ })
+ .map_err(Into::into)
}
pub(in crate::ml_db) fn read_column<C: FromIterator<T>, T: FromSql, P: Params>(
@@ -49,23 +51,28 @@ impl MlDb {
sql: &str,
parameters: impl Params,
) -> Result<T> {
- self.db.read(|connection| {
- connection
- .prepare_cached(sql)?
- .query_row(parameters, |row| Ok(row.get(0)?))
- })
+ self.db
+ .read(|connection| {
+ connection
+ .prepare_cached(sql)?
+ .query_row(parameters, |row| Ok(row.get(0)?))
+ })
+ .map_err(Into::into)
}
- pub(in crate::ml_db) fn read_optional<T: FromSql>(
+ pub(in crate::ml_db) fn read_optional<T>(
&self,
sql: &str,
parameters: impl Params,
+ map: impl FnOnce(&Row<'_>) -> db::Result<T>,
) -> Result<Option<T>> {
- self.db.read(|connection| {
- connection
- .prepare_cached(sql)?
- .query_optional(parameters, |row| Ok(row.get(0)?))
- })
+ self.db
+ .read(|connection| {
+ connection
+ .prepare_cached(sql)?
+ .query_optional(parameters, map)
+ })
+ .map_err(Into::into)
}
pub(in crate::ml_db) fn read_chunked_in<C: FromIterator<T>, T, I: ToSql>(
@@ -73,7 +80,7 @@ impl MlDb {
sql: &str,
ids: &[I],
chunk_size: NonZeroUsize,
- mut map: impl FnMut(&Row<'_>) -> Result<T>,
+ mut map: impl FnMut(&Row<'_>) -> db::Result<T>,
) -> Result<C> {
let mut rows = Vec::new();
for chunk in ids.chunks(chunk_size.get()) {
@@ -87,6 +94,7 @@ impl MlDb {
pub(in crate::ml_db) fn execute(&self, sql: &str, parameters: impl Params) -> Result<usize> {
self.db
.write(|connection| connection.prepare_cached(sql)?.execute(parameters))
+ .map_err(Into::into)
}
pub(in crate::ml_db) fn execute_chunked_in<I: ToSql>(
@@ -97,25 +105,29 @@ impl MlDb {
if ids.is_empty() {
return Ok(());
}
- self.db.write_transaction(|transaction| {
- for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
- let sql = expand_in_clause(sql, chunk.len());
- transaction.execute(&sql, params_from_iter(chunk))?;
- }
- Ok(())
- })
+ self.db
+ .write_transaction(|transaction| {
+ for chunk in ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {
+ let sql = expand_in_clause(sql, chunk.len());
+ transaction.execute(&sql, params_from_iter(chunk))?;
+ }
+ Ok(())
+ })
+ .map_err(Into::into)
}
pub(in crate::ml_db) fn execute_statements<'a>(
&self,
statements: impl IntoIterator<Item = &'a str>,
) -> Result<()> {
- self.db.write_transaction(|transaction| {
- for statement in statements {
- transaction.execute_batch(statement)?;
- }
- Ok(())
- })
+ self.db
+ .write_transaction(|transaction| {
+ for statement in statements {
+ transaction.execute_batch(statement)?;
+ }
+ Ok(())
+ })
+ .map_err(Into::into)
}
pub(in crate::ml_db) fn write_batch_atomic<P: Params>(
@@ -123,13 +135,15 @@ impl MlDb {
sql: &str,
parameter_sets: impl IntoIterator<Item = P>,
) -> Result<()> {
- self.db.write_transaction(|transaction| {
- let mut statement = transaction.prepare_cached(sql)?;
- for parameters in parameter_sets {
- statement.execute(parameters)?;
- }
- Ok(())
- })
+ self.db
+ .write_transaction(|transaction| {
+ let mut statement = transaction.prepare_cached(sql)?;
+ for parameters in parameter_sets {
+ statement.execute(parameters)?;
+ }
+ Ok(())
+ })
+ .map_err(Into::into)
}
pub(in crate::ml_db) fn write_batches_committing_each<P: Params>(
@@ -138,21 +152,23 @@ impl MlDb {
batch_size: NonZeroUsize,
parameter_sets: impl IntoIterator<Item = P>,
) -> Result<()> {
- self.db.write(|connection| {
- let mut parameter_sets = parameter_sets.into_iter().peekable();
- while parameter_sets.peek().is_some() {
- let transaction =
- connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
- {
- let mut statement = transaction.prepare_cached(sql)?;
- for parameters in parameter_sets.by_ref().take(batch_size.get()) {
- statement.execute(parameters)?;
+ self.db
+ .write(|connection| {
+ let mut parameter_sets = parameter_sets.into_iter().peekable();
+ while parameter_sets.peek().is_some() {
+ let transaction =
+ connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ {
+ let mut statement = transaction.prepare_cached(sql)?;
+ for parameters in parameter_sets.by_ref().take(batch_size.get()) {
+ statement.execute(parameters)?;
+ }
}
+ transaction.commit()?;
}
- transaction.commit()?;
- }
- Ok(())
- })
+ Ok(())
+ })
+ .map_err(Into::into)
}
}
@@ -164,7 +180,7 @@ fn expand_in_clause(sql: &str, count: usize) -> String {
sql.replacen("{}", &bind_placeholders(count), 1)
}
-pub(super) fn pair<A: FromSql, B: FromSql>(row: &Row<'_>) -> Result<(A, B)> {
+pub(super) fn pair<A: FromSql, B: FromSql>(row: &Row<'_>) -> db::Result<(A, B)> {
Ok((row.get(0)?, row.get(1)?))
}
diff --git a/rust/crates/photos/src/ml_db/queries/persons.rs b/rust/crates/photos/src/ml_db/queries/persons.rs
index 19a55f7692..b3f0b485ab 100644
--- a/rust/crates/photos/src/ml_db/queries/persons.rs
+++ b/rust/crates/photos/src/ml_db/queries/persons.rs
@@ -34,7 +34,6 @@ impl MlDb {
"#,
(),
)
- .map_err(Into::into)
}
pub fn get_person_ignored_clusters(&self, person_id: &str) -> Result<HashSet<String>> {
@@ -57,7 +56,6 @@ impl MlDb {
pub fn get_person_to_rejected_suggestions(&self) -> Result<HashMap<String, HashSet<String>>> {
self.read_grouped("SELECT person_id, cluster_id FROM not_person_feedback", ())
- .map_err(Into::into)
}
pub fn get_person_cluster_ids(&self, person_id: &str) -> Result<HashSet<String>> {
@@ -65,7 +63,6 @@ impl MlDb {
"SELECT cluster_id FROM cluster_person WHERE person_id = ?",
[person_id],
)
- .map_err(Into::into)
}
pub fn get_persons_cluster_ids(&self, person_ids: &[String]) -> Result<HashSet<String>> {
@@ -75,7 +72,6 @@ impl MlDb {
const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
|row| Ok(row.get(0)?),
)
- .map_err(Into::into)
}
pub fn get_person_to_cluster_id_to_face_ids(&self) -> Result<PersonToClusterIdToFaceIds> {
@@ -103,7 +99,6 @@ impl MlDb {
pub fn get_person_to_cluster_ids(&self) -> Result<HashMap<String, HashSet<String>>> {
self.read_grouped("SELECT person_id, cluster_id FROM cluster_person", ())
- .map_err(Into::into)
}
pub fn get_face_id_to_person_id_for_faces(
@@ -122,7 +117,6 @@ impl MlDb {
const { NonZeroUsize::new(MAX_SQL_BIND_PARAMS_PER_QUERY).unwrap() },
pair,
)
- .map_err(Into::into)
}
pub fn get_cluster_id_to_face_ids_for_person(
@@ -139,7 +133,6 @@ impl MlDb {
"#,
[person_id],
)
- .map_err(Into::into)
}
pub fn get_face_ids_for_person(&self, person_id: &str) -> Result<HashSet<String>> {
@@ -153,7 +146,6 @@ impl MlDb {
"#,
[person_id],
)
- .map_err(Into::into)
}
pub fn get_face_ids_for_person_ordered_by_score(
@@ -175,7 +167,6 @@ impl MlDb {
"#,
(person_id, limit),
)
- .map_err(Into::into)
}
pub fn remove_person(&self, person_id: &str) -> Result<()> {
@@ -209,7 +200,6 @@ impl MlDb {
.iter()
.map(|(cluster_id, person_id)| (person_id, cluster_id)),
)
- .map_err(Into::into)
}
pub fn capture_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {
@@ -227,7 +217,6 @@ impl MlDb {
.iter()
.map(|(cluster_id, person_id)| (person_id, cluster_id)),
)
- .map_err(Into::into)
}
pub fn remove_not_person_feedback(&self, person_id: &str, cluster_id: &str) -> Result<()> {
@@ -261,7 +250,6 @@ impl MlDb {
[person_id],
pair,
)
- .map_err(Into::into)
.and_then(file_id_to_cluster_ids)
}
@@ -271,7 +259,6 @@ impl MlDb {
(),
|row| Ok((row.get(1)?, row.get(0)?)),
)
- .map_err(Into::into)
}
pub fn drop_faces_feedback_tables(&self) -> Result<()> {
@@ -281,7 +268,6 @@ impl MlDb {
CREATE_CLUSTER_PERSON_TABLE,
CREATE_NOT_PERSON_FEEDBACK_TABLE,
])
- .map_err(Into::into)
}
pub fn get_file_ids_of_person_id(&self, person_id: &str) -> Result<Vec<i64>> {
@@ -297,7 +283,6 @@ impl MlDb {
"#,
[person_id],
)
- .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/queries/pets.rs b/rust/crates/photos/src/ml_db/queries/pets.rs
index 72ccb46de9..63309f1407 100644
--- a/rust/crates/photos/src/ml_db/queries/pets.rs
+++ b/rust/crates/photos/src/ml_db/queries/pets.rs
@@ -105,7 +105,6 @@ impl MlDb {
)
}),
)
- .map_err(Into::into)
}
pub fn bulk_insert_pet_bodies(&self, pet_bodies: &[PetBodyRow]) -> Result<()> {
@@ -126,7 +125,6 @@ impl MlDb {
)
}),
)
- .map_err(Into::into)
}
pub fn update_pet_face_vector_ids(
@@ -143,7 +141,6 @@ impl MlDb {
.iter()
.map(|(pet_face_id, vector_id)| (vector_id, pet_face_id)),
)
- .map_err(Into::into)
}
pub fn update_pet_body_vector_ids(
@@ -160,7 +157,6 @@ impl MlDb {
.iter()
.map(|(pet_body_id, vector_id)| (vector_id, pet_body_id)),
)
- .map_err(Into::into)
}
pub fn get_pet_faces_for_file_id(&self, file_upload_id: i64) -> Result<Vec<PetFaceRow>> {
@@ -169,7 +165,6 @@ impl MlDb {
[file_upload_id],
read_pet_face,
)
- .map_err(Into::into)
}
pub fn get_pet_bodies_for_file_id(&self, file_upload_id: i64) -> Result<Vec<PetBodyRow>> {
@@ -178,7 +173,6 @@ impl MlDb {
[file_upload_id],
read_pet_body,
)
- .map_err(Into::into)
}
pub fn pet_indexed_file_ids(&self, minimum_ml_version: i64) -> Result<HashMap<i64, i64>> {
@@ -187,7 +181,6 @@ impl MlDb {
[minimum_ml_version],
pair,
)
- .map_err(Into::into)
}
pub fn get_pet_indexed_file_count(&self, minimum_ml_version: i64) -> Result<i64> {
@@ -195,7 +188,6 @@ impl MlDb {
"SELECT COUNT(DISTINCT file_id) as count FROM pet_faces WHERE ml_version >= ?",
[minimum_ml_version],
)
- .map_err(Into::into)
}
pub fn get_pet_rows_for_files(&self, file_ids: &[i64]) -> Result<PetRowsForFiles> {
@@ -326,7 +318,6 @@ impl MlDb {
const { NonZeroUsize::new(800).unwrap() },
pair,
)
- .map_err(Into::into)
}
}
diff --git a/rust/crates/photos/src/ml_db/vector_encoding.rs b/rust/crates/photos/src/ml_db/vector_encoding.rs
index c09db144eb..2bd743f3a5 100644
--- a/rust/crates/photos/src/ml_db/vector_encoding.rs
+++ b/rust/crates/photos/src/ml_db/vector_encoding.rs
@@ -4,7 +4,7 @@ const EVECTOR_VALUES_PACKED_KEY: u32 = 0x0A;
const EVECTOR_VALUES_FIXED64_KEY: u32 = 0x09;
const MAX_GROUP_DEPTH: usize = 100;
-pub fn encode_evector(values: &[f64]) -> Vec<u8> {
+pub(super) fn encode_evector(values: &[f64]) -> Vec<u8> {
if values.is_empty() {
return Vec::new();
}
@@ -17,7 +17,7 @@ pub fn encode_evector(values: &[f64]) -> Vec<u8> {
bytes
}
-pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {
+pub(super) fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {
let mut values = Vec::new();
let mut offset = 0;
while offset < bytes.len() {
@@ -44,11 +44,11 @@ pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {
Ok(values)
}
-pub fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {
+pub(super) fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {
values.into_iter().flat_map(f32::to_le_bytes).collect()
}
-pub fn decode_f32(bytes: &[u8]) -> Vec<f32> {
+pub(super) fn decode_f32(bytes: &[u8]) -> Vec<f32> {
bytes
.as_chunks::<4>()
.0