Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Consolidate ML queries and harden vector decoding

laurenspriem173207071067
diff --git a/rust/crates/photos/src/ml_db/caches.rs b/rust/crates/photos/src/ml_db/caches.rs
index dead632342..9bdb18e3bb 100644
--- a/rust/crates/photos/src/ml_db/caches.rs
+++ b/rust/crates/photos/src/ml_db/caches.rs
@@ -1,7 +1,7 @@
use std::time::{SystemTime, UNIX_EPOCH};
-use super::clip::CLIP_ML_VERSION;
-use super::codec::{decode_f32, encode_f32};
+use super::queries::CLIP_ML_VERSION;
+use super::vector_encoding::{decode_f32, encode_f32};
use super::{MlDb, Result};
const THREE_MONTHS_MILLIS: i64 = 90 * 24 * 60 * 60 * 1000;
diff --git a/rust/crates/photos/src/ml_db/codec.rs b/rust/crates/photos/src/ml_db/codec.rs
deleted file mode 100644
index b6811a6586..0000000000
--- a/rust/crates/photos/src/ml_db/codec.rs
+++ /dev/null
@@ -1,188 +0,0 @@
-use super::{Error, Result};
-
-const EVECTOR_VALUES_PACKED_TAG: u64 = 0x0A;
-const EVECTOR_VALUES_FIXED64_TAG: u64 = 0x09;
-
-pub fn encode_evector(values: &[f64]) -> Vec<u8> {
- if values.is_empty() {
- return Vec::new();
- }
- let mut bytes = Vec::with_capacity(values.len() * 8 + 10);
- bytes.push(EVECTOR_VALUES_PACKED_TAG as u8);
- push_varint(&mut bytes, (values.len() * 8) as u64);
- for value in values {
- bytes.extend_from_slice(&value.to_le_bytes());
- }
- bytes
-}
-
-pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {
- let mut values = Vec::new();
- let mut offset = 0;
- while offset < bytes.len() {
- let tag = read_varint(bytes, &mut offset)?;
- match (tag, tag & 7) {
- (EVECTOR_VALUES_PACKED_TAG, _) => {
- let length = read_length(bytes, &mut offset)?;
- if length % 8 != 0 {
- return Err(malformed());
- }
- for chunk in bytes[offset..offset + length].as_chunks::<8>().0 {
- values.push(f64::from_le_bytes(*chunk));
- }
- offset += length;
- }
- (EVECTOR_VALUES_FIXED64_TAG, _) => {
- let end = advance(bytes, &mut offset, 8)?;
- values.push(f64_from_le(&bytes[end - 8..end]));
- }
- (_, 0) => {
- read_varint(bytes, &mut offset)?;
- }
- (_, 1) => {
- advance(bytes, &mut offset, 8)?;
- }
- (_, 2) => {
- let length = read_length(bytes, &mut offset)?;
- offset += length;
- }
- (_, 5) => {
- advance(bytes, &mut offset, 4)?;
- }
- _ => return Err(malformed()),
- }
- }
- Ok(values)
-}
-
-pub fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {
- values
- .into_iter()
- .flat_map(|value| value.to_le_bytes())
- .collect()
-}
-
-pub fn decode_f32(bytes: &[u8]) -> Vec<f32> {
- bytes
- .as_chunks::<4>()
- .0
- .iter()
- .map(|chunk| f32::from_le_bytes(*chunk))
- .collect()
-}
-
-fn f64_from_le(chunk: &[u8]) -> f64 {
- let mut buffer = [0u8; 8];
- buffer.copy_from_slice(chunk);
- f64::from_le_bytes(buffer)
-}
-
-fn push_varint(bytes: &mut Vec<u8>, mut value: u64) {
- while value >= 0x80 {
- bytes.push((value as u8) | 0x80);
- value >>= 7;
- }
- bytes.push(value as u8);
-}
-
-fn read_varint(bytes: &[u8], offset: &mut usize) -> Result<u64> {
- let mut value = 0u64;
- for shift in (0..64).step_by(7) {
- let byte = *bytes.get(*offset).ok_or_else(malformed)?;
- *offset += 1;
- value |= u64::from(byte & 0x7F) << shift;
- if byte & 0x80 == 0 {
- return Ok(value);
- }
- }
- Err(malformed())
-}
-
-fn read_length(bytes: &[u8], offset: &mut usize) -> Result<usize> {
- let length = usize::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;
- if bytes.len() - *offset < length {
- return Err(malformed());
- }
- Ok(length)
-}
-
-fn advance(bytes: &[u8], offset: &mut usize, count: usize) -> Result<usize> {
- if bytes.len() - *offset < count {
- return Err(malformed());
- }
- *offset += count;
- Ok(*offset)
-}
-
-fn malformed() -> Error {
- Error::Codec("malformed EVector bytes".to_string())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn evector_matches_protobuf_packed_encoding() {
- let bytes = encode_evector(&[1.0, 2.0]);
- assert_eq!(
- bytes,
- [
- 0x0A, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x40
- ]
- );
- assert_eq!(decode_evector(&bytes).unwrap(), vec![1.0, 2.0]);
- }
-
- #[test]
- fn empty_evector_is_zero_bytes() {
- assert!(encode_evector(&[]).is_empty());
- assert!(decode_evector(&[]).unwrap().is_empty());
- }
-
- #[test]
- fn evector_round_trips_long_vectors() {
- let values: Vec<f64> = (0..192).map(|i| i as f64 * 0.25 - 3.0).collect();
- let bytes = encode_evector(&values);
- assert_eq!(bytes[0], 0x0A);
- assert_eq!(&bytes[1..3], &[0x80, 0x0C]);
- assert_eq!(decode_evector(&bytes).unwrap(), values);
- }
-
- #[test]
- fn evector_accepts_unpacked_and_unknown_fields() {
- let mut bytes = vec![0x09];
- bytes.extend_from_slice(&3.5f64.to_le_bytes());
- bytes.extend_from_slice(&[0x10, 0x05]);
- bytes.extend_from_slice(&[0x1A, 0x02, 0xAA, 0xBB]);
- bytes.extend_from_slice(&[0x25, 1, 2, 3, 4]);
- bytes.extend_from_slice(&[0x09]);
- bytes.extend_from_slice(&(-1.0f64).to_le_bytes());
- assert_eq!(decode_evector(&bytes).unwrap(), vec![3.5, -1.0]);
- }
-
- #[test]
- fn evector_rejects_truncated_bytes() {
- assert!(decode_evector(&[0x0A, 0x10, 0x00]).is_err());
- assert!(decode_evector(&[0x0A, 0x04, 0, 0, 0, 0]).is_err());
- assert!(decode_evector(&[0x0F]).is_err());
- }
-
- #[test]
- fn f32_matches_little_endian_layout() {
- assert_eq!(encode_f32([1.0f32]), [0x00, 0x00, 0x80, 0x3F]);
- assert_eq!(decode_f32(&[0x00, 0x00, 0x80, 0x3F]), vec![1.0f32]);
- assert!(encode_f32([]).is_empty());
- assert!(decode_f32(&[]).is_empty());
- }
-
- #[test]
- fn f32_round_trips_and_ignores_trailing_bytes() {
- let values = vec![0.5f32, -2.25, 1e-7];
- let mut bytes = encode_f32(values.iter().copied());
- assert_eq!(decode_f32(&bytes), values);
- bytes.push(0xFF);
- assert_eq!(decode_f32(&bytes), values);
- }
-}
diff --git a/rust/crates/photos/src/ml_db/constants.rs b/rust/crates/photos/src/ml_db/constants.rs
deleted file mode 100644
index 00e646f591..0000000000
--- a/rust/crates/photos/src/ml_db/constants.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-pub const FACE_ML_VERSION: i64 = 1;
-pub const PET_ML_VERSION: i64 = 1;
-
-pub const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;
-pub const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;
-pub const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;
-pub const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;
-pub const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;
-
-pub fn is_bad_face_for_clustering(face_score: f64, blur_value: f64, is_sideways: bool) -> bool {
- face_score < MINIMUM_QUALITY_FACE_SCORE
- || blur_value < LAPLACIAN_SOFT_THRESHOLD
- || (blur_value < LAPLACIAN_VERY_SOFT_THRESHOLD && face_score < MEDIUM_QUALITY_FACE_SCORE)
- || is_sideways
-}
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index da0932ec43..12fc4d893c 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -1,30 +1,23 @@
mod caches;
-mod clip;
-mod codec;
-mod constants;
mod filedata;
+mod queries;
mod schema;
-mod types;
+mod vector_encoding;
use std::path::Path;
use crate::db::Database;
-pub use clip::{
- CLIP_EMBEDDING_BYTES_LENGTH, CLIP_EMBEDDING_DIMENSIONS, CLIP_ML_VERSION, ClipEmbedding,
- ClipRow, EmbeddingVector,
-};
-pub use codec::{decode_evector, decode_f32, encode_evector, encode_f32};
-pub use constants::{
- FACE_ML_VERSION, LAPLACIAN_HARD_THRESHOLD, LAPLACIAN_SOFT_THRESHOLD,
- LAPLACIAN_VERY_SOFT_THRESHOLD, MEDIUM_QUALITY_FACE_SCORE, MINIMUM_QUALITY_FACE_SCORE,
- PET_ML_VERSION, is_bad_face_for_clustering,
-};
pub use filedata::{FdStatus, PreviewInfo};
-pub use types::{
- ClusterCentroidRow, ClusterSummary, FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding,
- PetBodyRow, PetBodyVectorRow, PetFaceRow, PetFaceVectorRow, PetRowsForFiles,
+pub use queries::{
+ CLIP_EMBEDDING_BYTES_LENGTH, CLIP_EMBEDDING_DIMENSIONS, CLIP_ML_VERSION, ClipEmbedding,
+ ClipRow, ClusterCentroidRow, ClusterSummary, EmbeddingVector, FACE_ML_VERSION,
+ FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding, LAPLACIAN_HARD_THRESHOLD,
+ LAPLACIAN_SOFT_THRESHOLD, LAPLACIAN_VERY_SOFT_THRESHOLD, MEDIUM_QUALITY_FACE_SCORE,
+ MINIMUM_QUALITY_FACE_SCORE, PET_ML_VERSION, PetBodyRow, PetBodyVectorRow, PetFaceRow,
+ PetFaceVectorRow, PetRowsForFiles, is_bad_face_for_clustering,
};
+pub use vector_encoding::{decode_evector, decode_f32, encode_evector, encode_f32};
#[derive(Debug, thiserror::Error)]
pub enum Error {
@@ -52,42 +45,15 @@ impl MlDb {
db: Database::open(path, &schema::MIGRATION_SCRIPTS)?,
})
}
-
- pub fn clear_non_pet_tables(&self) -> Result<()> {
- self.db
- .execute_statements([
- schema::DELETE_FACES,
- schema::DELETE_FACE_CLUSTERS,
- schema::DELETE_CLUSTER_PERSON,
- schema::DELETE_CLUSTER_SUMMARY,
- schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
- schema::DELETE_NOT_PERSON_FEEDBACK,
- schema::DELETE_CLIP_EMBEDDINGS,
- schema::DELETE_FILE_DATA,
- ])
- .map_err(Into::into)
- }
-
- pub fn clear_pet_tables(&self) -> Result<()> {
- self.db
- .execute_statements([
- schema::DELETE_PET_FACES,
- schema::DELETE_PET_BODIES,
- schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
- schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
- ])
- .map_err(Into::into)
- }
}
#[cfg(test)]
mod tests {
- use std::collections::HashSet;
use std::fmt::Debug;
use std::path::Path;
- use super::clip::tests::full_clip;
- use super::{Error, MlDb, TARGET_VERSION, caches, clip, filedata};
+ use super::queries::tests::full_clip;
+ use super::{Error, MlDb, TARGET_VERSION};
use crate::db::Connection;
use tempfile::TempDir;
@@ -131,14 +97,6 @@ mod tests {
}
}
- fn seeded() -> (TempDir, MlDb) {
- let (directory, db) = open();
- clip::tests::seed(&db);
- filedata::tests::seed(&db);
- caches::tests::seed(&db);
- (directory, db)
- }
-
#[test]
fn ml_db_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
@@ -227,51 +185,4 @@ mod tests {
assert_eq!(user_version(&path), 15);
assert_eq!(db.count_clip_rows().unwrap(), 0);
}
-
- #[test]
- fn clear_non_pet_tables_leaves_pets_and_caches() {
- let (_directory, db) = seeded();
- db.clear_non_pet_tables().unwrap();
- assert_eq!(db.count_clip_rows().unwrap(), 0);
- assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());
- assert_eq!(
- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
- Some("1_0".to_string())
- );
- assert_eq!(
- db.get_repeated_text_embedding_cache("dog").unwrap(),
- Some(vec![0.5f32, -1.0])
- );
- }
-
- #[test]
- fn clear_pet_tables_leaves_non_pet_tables() {
- let (directory, db) = seeded();
- let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();
- connection
- .execute_batch(
- "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');
- INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",
- )
- .unwrap();
- db.clear_pet_tables().unwrap();
- let row_count = |table: &str| -> i64 {
- connection
- .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {
- row.get(0)
- })
- .unwrap()
- };
- assert_eq!(row_count("pet_face_vector_id_map"), 0);
- assert_eq!(row_count("pet_body_vector_id_map"), 0);
- assert_eq!(db.count_clip_rows().unwrap(), 4);
- assert_eq!(
- db.get_file_ids_with_fd_data(None).unwrap(),
- HashSet::from([1, 2, 3])
- );
- assert_eq!(
- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
- Some("1_0".to_string())
- );
- }
}
diff --git a/rust/crates/photos/src/ml_db/clip.rs b/rust/crates/photos/src/ml_db/queries.rs
similarity index 56%
rename from rust/crates/photos/src/ml_db/clip.rs
rename to rust/crates/photos/src/ml_db/queries.rs
index f742fc35c1..53842d13b3 100644
--- a/rust/crates/photos/src/ml_db/clip.rs
+++ b/rust/crates/photos/src/ml_db/queries.rs
@@ -2,14 +2,23 @@ use std::collections::HashMap;
use crate::db::pair;
-use super::codec::{decode_f32, encode_f32};
-use super::schema::DELETE_CLIP_EMBEDDINGS;
+use super::schema;
+use super::vector_encoding::{decode_f32, encode_f32};
use super::{MlDb, Result};
+pub const FACE_ML_VERSION: i64 = 1;
pub const CLIP_ML_VERSION: i64 = 1;
+pub const PET_ML_VERSION: i64 = 1;
+
pub const CLIP_EMBEDDING_DIMENSIONS: usize = 512;
pub const CLIP_EMBEDDING_BYTES_LENGTH: i64 = CLIP_EMBEDDING_DIMENSIONS as i64 * 4;
+pub const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;
+pub const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;
+pub const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;
+pub const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;
+pub const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;
+
#[derive(Clone, Debug, PartialEq)]
pub struct ClipEmbedding {
pub file_id: i64,
@@ -29,7 +38,124 @@ pub struct ClipRow {
pub embedding: Vec<u8>,
}
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceRow {
+ pub file_id: i64,
+ pub face_id: String,
+ pub detection_json: String,
+ pub embedding: Vec<f64>,
+ pub score: f64,
+ pub blur: f64,
+ pub is_sideways: bool,
+ pub image_height: i64,
+ pub image_width: i64,
+ pub ml_version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceWithoutEmbedding {
+ pub face_id: String,
+ pub file_id: i64,
+ pub score: f64,
+ pub detection_json: String,
+ pub blur: f64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceDbInfoForClustering {
+ pub face_id: String,
+ pub cluster_id: Option<String>,
+ pub embedding_bytes: Vec<u8>,
+ pub face_score: f64,
+ pub blur_value: f64,
+ pub is_sideways: bool,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetFaceRow {
+ pub file_id: i64,
+ pub pet_face_id: String,
+ pub detection_json: String,
+ pub face_vector_id: Option<i64>,
+ pub species: i64,
+ pub face_score: f64,
+ pub image_height: i64,
+ pub image_width: i64,
+ pub ml_version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetBodyRow {
+ pub file_id: i64,
+ pub pet_body_id: String,
+ pub detection_json: String,
+ pub body_vector_id: Option<i64>,
+ pub species: i64,
+ pub score: f64,
+ pub image_height: i64,
+ pub image_width: i64,
+ pub ml_version: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetFaceVectorRow {
+ pub pet_face_id: String,
+ pub face_vector_id: Option<i64>,
+ pub species: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetBodyVectorRow {
+ pub pet_body_id: String,
+ pub body_vector_id: Option<i64>,
+ pub species: i64,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct PetRowsForFiles {
+ pub faces: Vec<PetFaceVectorRow>,
+ pub bodies: Vec<PetBodyVectorRow>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClusterSummary {
+ pub avg: Vec<u8>,
+ pub count: i64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClusterCentroidRow {
+ pub cluster_id: String,
+ pub avg: Vec<u8>,
+}
+
impl MlDb {
+ pub fn clear_non_pet_tables(&self) -> Result<()> {
+ self.db
+ .execute_statements([
+ schema::DELETE_FACES,
+ schema::DELETE_FACE_CLUSTERS,
+ schema::DELETE_CLUSTER_PERSON,
+ schema::DELETE_CLUSTER_SUMMARY,
+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ schema::DELETE_NOT_PERSON_FEEDBACK,
+ schema::DELETE_CLIP_EMBEDDINGS,
+ schema::DELETE_FILE_DATA,
+ ])
+ .map_err(Into::into)
+ }
+
+ pub fn clear_pet_tables(&self) -> Result<()> {
+ self.db
+ .execute_statements([
+ schema::DELETE_PET_FACES,
+ schema::DELETE_PET_BODIES,
+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
+ ])
+ .map_err(Into::into)
+ }
+
pub fn get_all_clip_vectors(&self) -> Result<Vec<EmbeddingVector>> {
let rows: Vec<(i64, Vec<u8>)> =
self.db
@@ -92,7 +218,7 @@ impl MlDb {
pub fn delete_all_clip_rows(&self) -> Result<()> {
self.db
- .execute_statements([DELETE_CLIP_EMBEDDINGS])
+ .execute_statements([schema::DELETE_CLIP_EMBEDDINGS])
.map_err(Into::into)
}
@@ -118,6 +244,13 @@ impl MlDb {
}
}
+pub fn is_bad_face_for_clustering(face_score: f64, blur_value: f64, is_sideways: bool) -> bool {
+ face_score < MINIMUM_QUALITY_FACE_SCORE
+ || blur_value < LAPLACIAN_SOFT_THRESHOLD
+ || (blur_value < LAPLACIAN_VERY_SOFT_THRESHOLD && face_score < MEDIUM_QUALITY_FACE_SCORE)
+ || is_sideways
+}
+
fn clip_row(embedding: &ClipEmbedding) -> (i64, Vec<u8>, i64) {
(
embedding.file_id,
@@ -128,11 +261,13 @@ fn clip_row(embedding: &ClipEmbedding) -> (i64, Vec<u8>, i64) {
#[cfg(test)]
pub(super) mod tests {
- use std::collections::HashMap;
+ use std::collections::{HashMap, HashSet};
use super::{CLIP_EMBEDDING_DIMENSIONS, ClipEmbedding, ClipRow, MlDb};
- use crate::ml_db::codec::encode_f32;
+ use crate::db::Connection;
use crate::ml_db::tests::{cases, check, open};
+ use crate::ml_db::vector_encoding::encode_f32;
+ use crate::ml_db::{caches, filedata};
use tempfile::TempDir;
fn clip(file_id: i64, embedding: Vec<f64>) -> ClipEmbedding {
@@ -147,7 +282,7 @@ pub(super) mod tests {
clip(file_id, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])
}
- pub(in crate::ml_db) fn seed(db: &MlDb) {
+ fn seed(db: &MlDb) {
db.insert_clip_rows(&[
full_clip(1),
ClipEmbedding {
@@ -166,6 +301,13 @@ pub(super) mod tests {
(directory, db)
}
+ fn seeded_all() -> (TempDir, MlDb) {
+ let (directory, db) = seeded();
+ filedata::tests::seed(&db);
+ caches::tests::seed(&db);
+ (directory, db)
+ }
+
#[test]
fn seeded_counts() {
let (_directory, db) = seeded();
@@ -248,4 +390,51 @@ pub(super) mod tests {
db.delete_all_clip_rows().unwrap();
assert_eq!(db.count_clip_rows().unwrap(), 0);
}
+
+ #[test]
+ fn clear_non_pet_tables_leaves_pets_and_caches() {
+ let (_directory, db) = seeded_all();
+ db.clear_non_pet_tables().unwrap();
+ assert_eq!(db.count_clip_rows().unwrap(), 0);
+ assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());
+ assert_eq!(
+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
+ Some("1_0".to_string())
+ );
+ assert_eq!(
+ db.get_repeated_text_embedding_cache("dog").unwrap(),
+ Some(vec![0.5f32, -1.0])
+ );
+ }
+
+ #[test]
+ fn clear_pet_tables_leaves_non_pet_tables() {
+ let (directory, db) = seeded_all();
+ let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();
+ connection
+ .execute_batch(
+ "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');
+ INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",
+ )
+ .unwrap();
+ db.clear_pet_tables().unwrap();
+ let row_count = |table: &str| -> i64 {
+ connection
+ .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {
+ row.get(0)
+ })
+ .unwrap()
+ };
+ assert_eq!(row_count("pet_face_vector_id_map"), 0);
+ assert_eq!(row_count("pet_body_vector_id_map"), 0);
+ assert_eq!(db.count_clip_rows().unwrap(), 4);
+ assert_eq!(
+ db.get_file_ids_with_fd_data(None).unwrap(),
+ HashSet::from([1, 2, 3])
+ );
+ assert_eq!(
+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),
+ Some("1_0".to_string())
+ );
+ }
}
diff --git a/rust/crates/photos/src/ml_db/types.rs b/rust/crates/photos/src/ml_db/types.rs
deleted file mode 100644
index b7a1a21581..0000000000
--- a/rust/crates/photos/src/ml_db/types.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-#[derive(Clone, Debug, PartialEq)]
-pub struct FaceRow {
- pub file_id: i64,
- pub face_id: String,
- pub detection_json: String,
- pub embedding: Vec<f64>,
- pub score: f64,
- pub blur: f64,
- pub is_sideways: bool,
- pub image_height: i64,
- pub image_width: i64,
- pub ml_version: i64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct FaceWithoutEmbedding {
- pub face_id: String,
- pub file_id: i64,
- pub score: f64,
- pub detection_json: String,
- pub blur: f64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct FaceDbInfoForClustering {
- pub face_id: String,
- pub cluster_id: Option<String>,
- pub embedding_bytes: Vec<u8>,
- pub face_score: f64,
- pub blur_value: f64,
- pub is_sideways: bool,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct PetFaceRow {
- pub file_id: i64,
- pub pet_face_id: String,
- pub detection_json: String,
- pub face_vector_id: Option<i64>,
- pub species: i64,
- pub face_score: f64,
- pub image_height: i64,
- pub image_width: i64,
- pub ml_version: i64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct PetBodyRow {
- pub file_id: i64,
- pub pet_body_id: String,
- pub detection_json: String,
- pub body_vector_id: Option<i64>,
- pub species: i64,
- pub score: f64,
- pub image_height: i64,
- pub image_width: i64,
- pub ml_version: i64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct PetFaceVectorRow {
- pub pet_face_id: String,
- pub face_vector_id: Option<i64>,
- pub species: i64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct PetBodyVectorRow {
- pub pet_body_id: String,
- pub body_vector_id: Option<i64>,
- pub species: i64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct PetRowsForFiles {
- pub faces: Vec<PetFaceVectorRow>,
- pub bodies: Vec<PetBodyVectorRow>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ClusterSummary {
- pub avg: Vec<u8>,
- pub count: i64,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ClusterCentroidRow {
- pub cluster_id: String,
- pub avg: Vec<u8>,
-}
diff --git a/rust/crates/photos/src/ml_db/vector_encoding.rs b/rust/crates/photos/src/ml_db/vector_encoding.rs
new file mode 100644
index 0000000000..fa3b012ea9
--- /dev/null
+++ b/rust/crates/photos/src/ml_db/vector_encoding.rs
@@ -0,0 +1,348 @@
+use super::{Error, Result};
+
+const EVECTOR_VALUES_PACKED_KEY: u32 = 0x0A;
+const EVECTOR_VALUES_FIXED64_KEY: u32 = 0x09;
+const MAX_GROUP_DEPTH: usize = 100;
+
+pub fn encode_evector(values: &[f64]) -> Vec<u8> {
+ if values.is_empty() {
+ return Vec::new();
+ }
+ let mut bytes = Vec::with_capacity(values.len() * 8 + 10);
+ bytes.push(EVECTOR_VALUES_PACKED_KEY as u8);
+ push_varint(&mut bytes, (values.len() * 8) as u64);
+ for value in values {
+ bytes.extend_from_slice(&value.to_le_bytes());
+ }
+ bytes
+}
+
+pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {
+ let mut values = Vec::new();
+ let mut offset = 0;
+ while offset < bytes.len() {
+ let key = read_key(bytes, &mut offset)?;
+ match key {
+ EVECTOR_VALUES_PACKED_KEY => {
+ let length = read_length(bytes, &mut offset)?;
+ if length % 8 != 0 {
+ return Err(malformed());
+ }
+ for chunk in bytes[offset..offset + length].as_chunks::<8>().0 {
+ values.push(f64::from_le_bytes(*chunk));
+ }
+ offset += length;
+ }
+ EVECTOR_VALUES_FIXED64_KEY => {
+ let end = advance(bytes, &mut offset, 8)?;
+ values.push(f64_from_le(&bytes[end - 8..end]));
+ }
+ _ if key >> 3 == 1 => return Err(malformed()),
+ _ => skip_field(key, bytes, &mut offset, MAX_GROUP_DEPTH)?,
+ }
+ }
+ Ok(values)
+}
+
+pub fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {
+ values
+ .into_iter()
+ .flat_map(|value| value.to_le_bytes())
+ .collect()
+}
+
+pub fn decode_f32(bytes: &[u8]) -> Vec<f32> {
+ bytes
+ .as_chunks::<4>()
+ .0
+ .iter()
+ .map(|chunk| f32::from_le_bytes(*chunk))
+ .collect()
+}
+
+fn f64_from_le(chunk: &[u8]) -> f64 {
+ let mut buffer = [0u8; 8];
+ buffer.copy_from_slice(chunk);
+ f64::from_le_bytes(buffer)
+}
+
+fn push_varint(bytes: &mut Vec<u8>, mut value: u64) {
+ while value >= 0x80 {
+ bytes.push((value as u8) | 0x80);
+ value >>= 7;
+ }
+ bytes.push(value as u8);
+}
+
+fn read_key(bytes: &[u8], offset: &mut usize) -> Result<u32> {
+ let key = u32::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;
+ if key >> 3 == 0 || key & 7 > 5 {
+ return Err(malformed());
+ }
+ Ok(key)
+}
+
+fn skip_field(key: u32, bytes: &[u8], offset: &mut usize, remaining_depth: usize) -> Result<()> {
+ if remaining_depth == 0 {
+ return Err(malformed());
+ }
+ match key & 7 {
+ 0 => {
+ read_varint(bytes, offset)?;
+ }
+ 1 => {
+ advance(bytes, offset, 8)?;
+ }
+ 2 => {
+ let length = read_length(bytes, offset)?;
+ *offset += length;
+ }
+ 3 => loop {
+ let inner_key = read_key(bytes, offset)?;
+ if inner_key & 7 == 4 {
+ if inner_key >> 3 != key >> 3 {
+ return Err(malformed());
+ }
+ break;
+ }
+ skip_field(inner_key, bytes, offset, remaining_depth - 1)?;
+ },
+ 5 => {
+ advance(bytes, offset, 4)?;
+ }
+ _ => return Err(malformed()),
+ }
+ Ok(())
+}
+
+fn read_varint(bytes: &[u8], offset: &mut usize) -> Result<u64> {
+ let mut value = 0u64;
+ for shift in (0..64).step_by(7) {
+ let byte = *bytes.get(*offset).ok_or_else(malformed)?;
+ *offset += 1;
+ if shift == 63 && byte > 1 {
+ return Err(malformed());
+ }
+ value |= u64::from(byte & 0x7F) << shift;
+ if byte & 0x80 == 0 {
+ return Ok(value);
+ }
+ }
+ Err(malformed())
+}
+
+fn read_length(bytes: &[u8], offset: &mut usize) -> Result<usize> {
+ let length = usize::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;
+ if bytes.len() - *offset < length {
+ return Err(malformed());
+ }
+ Ok(length)
+}
+
+fn advance(bytes: &[u8], offset: &mut usize, count: usize) -> Result<usize> {
+ if bytes.len() - *offset < count {
+ return Err(malformed());
+ }
+ *offset += count;
+ Ok(*offset)
+}
+
+fn malformed() -> Error {
+ Error::Codec("malformed EVector bytes".to_string())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn evector_matches_protobuf_packed_encoding() {
+ let bytes = encode_evector(&[1.0, 2.0]);
+ assert_eq!(
+ bytes,
+ [
+ 0x0A, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x40
+ ]
+ );
+ assert_eq!(decode_evector(&bytes).unwrap(), vec![1.0, 2.0]);
+ }
+
+ #[test]
+ fn empty_evector_is_zero_bytes() {
+ assert!(encode_evector(&[]).is_empty());
+ assert!(decode_evector(&[]).unwrap().is_empty());
+ }
+
+ #[test]
+ fn evector_round_trips_long_vectors() {
+ let values: Vec<f64> = (0..192).map(|i| i as f64 * 0.25 - 3.0).collect();
+ let bytes = encode_evector(&values);
+ assert_eq!(bytes[0], 0x0A);
+ assert_eq!(&bytes[1..3], &[0x80, 0x0C]);
+ assert_eq!(decode_evector(&bytes).unwrap(), values);
+ }
+
+ #[test]
+ fn evector_accepts_unpacked_and_unknown_fields() {
+ let mut bytes = vec![0x09];
+ bytes.extend_from_slice(&3.5f64.to_le_bytes());
+ bytes.extend_from_slice(&[0x10, 0x05]);
+ bytes.extend_from_slice(&[0x1A, 0x02, 0xAA, 0xBB]);
+ bytes.extend_from_slice(&[0x25, 1, 2, 3, 4]);
+ bytes.extend_from_slice(&[0x09]);
+ bytes.extend_from_slice(&(-1.0f64).to_le_bytes());
+ assert_eq!(decode_evector(&bytes).unwrap(), vec![3.5, -1.0]);
+ }
+
+ #[test]
+ fn evector_rejects_truncated_bytes() {
+ assert!(decode_evector(&[0x0A, 0x10, 0x00]).is_err());
+ assert!(decode_evector(&[0x0A, 0x04, 0, 0, 0, 0]).is_err());
+ assert!(decode_evector(&[0x0F]).is_err());
+ for bytes in [
+ &[0x09, 0, 0, 0, 0, 0, 0, 0][..],
+ &[0x10],
+ &[0x11, 0, 0, 0, 0, 0, 0, 0],
+ &[0x12, 2, 0],
+ &[0x15, 0, 0, 0],
+ ] {
+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");
+ }
+ }
+
+ #[test]
+ fn evector_concatenates_packed_and_unpacked_values() {
+ let mut bytes = encode_evector(&[1.0, 2.0]);
+ bytes.extend([0x10, 0x05, 0x09]);
+ bytes.extend(3.0f64.to_le_bytes());
+ bytes.extend([0x0A, 0]);
+ bytes.extend(encode_evector(&[4.0, 5.0]));
+ assert_eq!(decode_evector(&bytes).unwrap(), [1.0, 2.0, 3.0, 4.0, 5.0]);
+ }
+
+ #[test]
+ fn evector_preserves_float_bits() {
+ let bits = [
+ 0,
+ 1,
+ 0x8000_0000_0000_0000,
+ 0x7FF0_0000_0000_0000,
+ 0xFFF0_0000_0000_0000,
+ 0x7FF8_0000_0000_0042,
+ ];
+ let values: Vec<f64> = bits.into_iter().map(f64::from_bits).collect();
+ let decoded = decode_evector(&encode_evector(&values)).unwrap();
+ assert_eq!(
+ decoded.into_iter().map(f64::to_bits).collect::<Vec<_>>(),
+ bits
+ );
+ }
+
+ #[test]
+ fn evector_rejects_invalid_keys() {
+ for bytes in [
+ &[0, 0][..],
+ &[0x02, 0],
+ &[0x16],
+ &[0x17],
+ &[0x80, 0x80, 0x80, 0x80, 0x10, 0],
+ ] {
+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");
+ }
+ assert!(
+ decode_evector(&[0xF8, 0xFF, 0xFF, 0xFF, 0x0F, 0])
+ .unwrap()
+ .is_empty()
+ );
+ }
+
+ #[test]
+ fn evector_rejects_incorrect_value_wire_types() {
+ for bytes in [&[0x08, 0][..], &[0x0B, 0x0C], &[0x0C], &[0x0D, 0, 0, 0, 0]] {
+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");
+ }
+ }
+
+ #[test]
+ fn evector_checks_varint_limits() {
+ let mut maximum = vec![0x10];
+ maximum.extend([0xFF; 9]);
+ maximum.push(1);
+ assert!(decode_evector(&maximum).unwrap().is_empty());
+
+ let mut overflowing = vec![0x80; 9];
+ overflowing.push(2);
+ for invalid in [vec![0x80; 9], vec![0x80; 10], overflowing] {
+ for prefix in [&[][..], &[0x0A][..], &[0x10][..], &[0x12][..]] {
+ let mut bytes = prefix.to_vec();
+ bytes.extend(&invalid);
+ assert!(decode_evector(&bytes).is_err(), "{bytes:?}");
+ }
+ }
+
+ for key in [0x0A, 0x12] {
+ maximum[0] = key;
+ assert!(decode_evector(&maximum).is_err());
+ }
+ }
+
+ #[test]
+ fn evector_skips_nested_unknown_groups() {
+ let mut bytes = encode_evector(&[1.0]);
+ bytes.extend([0x13, 0x09]);
+ bytes.extend(99.0f64.to_le_bytes());
+ bytes.extend([0x1B, 0x08, 1, 0x25, 1, 2, 3, 4, 0x29]);
+ bytes.extend([0; 8]);
+ bytes.extend([0x32, 2, 0xAA, 0xBB, 0x1C, 0x14]);
+ bytes.extend(encode_evector(&[2.0]));
+ assert_eq!(decode_evector(&bytes).unwrap(), [1.0, 2.0]);
+ }
+
+ #[test]
+ fn evector_rejects_invalid_groups() {
+ for bytes in [
+ &[0x14][..],
+ &[0x13],
+ &[0x13, 0x1C],
+ &[0x13, 0x1B, 0x14, 0x1C],
+ &[0x13, 0x10, 0, 0x14, 0x14],
+ &[0x13, 0, 0, 0x14],
+ ] {
+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");
+ }
+ }
+
+ #[test]
+ fn evector_limits_group_nesting() {
+ let nested = |depth, with_value| {
+ let mut bytes = vec![0x13; depth];
+ if with_value {
+ bytes.extend([0x10, 0]);
+ }
+ bytes.extend(vec![0x14; depth]);
+ bytes
+ };
+ assert!(decode_evector(&nested(99, true)).unwrap().is_empty());
+ assert!(decode_evector(&nested(100, false)).unwrap().is_empty());
+ assert!(decode_evector(&nested(100, true)).is_err());
+ assert!(decode_evector(&nested(101, false)).is_err());
+ }
+
+ #[test]
+ fn f32_matches_little_endian_layout() {
+ assert_eq!(encode_f32([1.0f32]), [0x00, 0x00, 0x80, 0x3F]);
+ assert_eq!(decode_f32(&[0x00, 0x00, 0x80, 0x3F]), vec![1.0f32]);
+ assert!(encode_f32([]).is_empty());
+ assert!(decode_f32(&[]).is_empty());
+ }
+
+ #[test]
+ fn f32_round_trips_and_ignores_trailing_bytes() {
+ let values = vec![0.5f32, -2.25, 1e-7];
+ let mut bytes = encode_f32(values.iter().copied());
+ assert_eq!(decode_f32(&bytes), values);
+ bytes.push(0xFF);
+ assert_eq!(decode_f32(&bytes), values);
+ }
+}