π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Consolidate ML queries and harden vector decoding
laurenspriem173207071067
diff --git a/rust/crates/photos/src/ml_db/caches.rs b/rust/crates/photos/src/ml_db/caches.rsindex dead632342..9bdb18e3bb 100644--- a/rust/crates/photos/src/ml_db/caches.rs+++ b/rust/crates/photos/src/ml_db/caches.rs@@ -1,7 +1,7 @@use std::time::{SystemTime, UNIX_EPOCH};-use super::clip::CLIP_ML_VERSION;-use super::codec::{decode_f32, encode_f32};+use super::queries::CLIP_ML_VERSION;+use super::vector_encoding::{decode_f32, encode_f32};use super::{MlDb, Result};const THREE_MONTHS_MILLIS: i64 = 90 * 24 * 60 * 60 * 1000;diff --git a/rust/crates/photos/src/ml_db/codec.rs b/rust/crates/photos/src/ml_db/codec.rsdeleted file mode 100644index b6811a6586..0000000000--- a/rust/crates/photos/src/ml_db/codec.rs+++ /dev/null@@ -1,188 +0,0 @@-use super::{Error, Result};--const EVECTOR_VALUES_PACKED_TAG: u64 = 0x0A;-const EVECTOR_VALUES_FIXED64_TAG: u64 = 0x09;--pub fn encode_evector(values: &[f64]) -> Vec<u8> {- if values.is_empty() {- return Vec::new();- }- let mut bytes = Vec::with_capacity(values.len() * 8 + 10);- bytes.push(EVECTOR_VALUES_PACKED_TAG as u8);- push_varint(&mut bytes, (values.len() * 8) as u64);- for value in values {- bytes.extend_from_slice(&value.to_le_bytes());- }- bytes-}--pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {- let mut values = Vec::new();- let mut offset = 0;- while offset < bytes.len() {- let tag = read_varint(bytes, &mut offset)?;- match (tag, tag & 7) {- (EVECTOR_VALUES_PACKED_TAG, _) => {- let length = read_length(bytes, &mut offset)?;- if length % 8 != 0 {- return Err(malformed());- }- for chunk in bytes[offset..offset + length].as_chunks::<8>().0 {- values.push(f64::from_le_bytes(*chunk));- }- offset += length;- }- (EVECTOR_VALUES_FIXED64_TAG, _) => {- let end = advance(bytes, &mut offset, 8)?;- values.push(f64_from_le(&bytes[end - 8..end]));- }- (_, 0) => {- read_varint(bytes, &mut offset)?;- }- (_, 1) => {- advance(bytes, &mut offset, 8)?;- }- (_, 2) => {- let length = read_length(bytes, &mut offset)?;- offset += length;- }- (_, 5) => {- advance(bytes, &mut offset, 4)?;- }- _ => return Err(malformed()),- }- }- Ok(values)-}--pub fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {- values- .into_iter()- .flat_map(|value| value.to_le_bytes())- .collect()-}--pub fn decode_f32(bytes: &[u8]) -> Vec<f32> {- bytes- .as_chunks::<4>()- .0- .iter()- .map(|chunk| f32::from_le_bytes(*chunk))- .collect()-}--fn f64_from_le(chunk: &[u8]) -> f64 {- let mut buffer = [0u8; 8];- buffer.copy_from_slice(chunk);- f64::from_le_bytes(buffer)-}--fn push_varint(bytes: &mut Vec<u8>, mut value: u64) {- while value >= 0x80 {- bytes.push((value as u8) | 0x80);- value >>= 7;- }- bytes.push(value as u8);-}--fn read_varint(bytes: &[u8], offset: &mut usize) -> Result<u64> {- let mut value = 0u64;- for shift in (0..64).step_by(7) {- let byte = *bytes.get(*offset).ok_or_else(malformed)?;- *offset += 1;- value |= u64::from(byte & 0x7F) << shift;- if byte & 0x80 == 0 {- return Ok(value);- }- }- Err(malformed())-}--fn read_length(bytes: &[u8], offset: &mut usize) -> Result<usize> {- let length = usize::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;- if bytes.len() - *offset < length {- return Err(malformed());- }- Ok(length)-}--fn advance(bytes: &[u8], offset: &mut usize, count: usize) -> Result<usize> {- if bytes.len() - *offset < count {- return Err(malformed());- }- *offset += count;- Ok(*offset)-}--fn malformed() -> Error {- Error::Codec("malformed EVector bytes".to_string())-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn evector_matches_protobuf_packed_encoding() {- let bytes = encode_evector(&[1.0, 2.0]);- assert_eq!(- bytes,- [- 0x0A, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00,- 0x00, 0x00, 0x00, 0x40- ]- );- assert_eq!(decode_evector(&bytes).unwrap(), vec![1.0, 2.0]);- }-- #[test]- fn empty_evector_is_zero_bytes() {- assert!(encode_evector(&[]).is_empty());- assert!(decode_evector(&[]).unwrap().is_empty());- }-- #[test]- fn evector_round_trips_long_vectors() {- let values: Vec<f64> = (0..192).map(|i| i as f64 * 0.25 - 3.0).collect();- let bytes = encode_evector(&values);- assert_eq!(bytes[0], 0x0A);- assert_eq!(&bytes[1..3], &[0x80, 0x0C]);- assert_eq!(decode_evector(&bytes).unwrap(), values);- }-- #[test]- fn evector_accepts_unpacked_and_unknown_fields() {- let mut bytes = vec![0x09];- bytes.extend_from_slice(&3.5f64.to_le_bytes());- bytes.extend_from_slice(&[0x10, 0x05]);- bytes.extend_from_slice(&[0x1A, 0x02, 0xAA, 0xBB]);- bytes.extend_from_slice(&[0x25, 1, 2, 3, 4]);- bytes.extend_from_slice(&[0x09]);- bytes.extend_from_slice(&(-1.0f64).to_le_bytes());- assert_eq!(decode_evector(&bytes).unwrap(), vec![3.5, -1.0]);- }-- #[test]- fn evector_rejects_truncated_bytes() {- assert!(decode_evector(&[0x0A, 0x10, 0x00]).is_err());- assert!(decode_evector(&[0x0A, 0x04, 0, 0, 0, 0]).is_err());- assert!(decode_evector(&[0x0F]).is_err());- }-- #[test]- fn f32_matches_little_endian_layout() {- assert_eq!(encode_f32([1.0f32]), [0x00, 0x00, 0x80, 0x3F]);- assert_eq!(decode_f32(&[0x00, 0x00, 0x80, 0x3F]), vec![1.0f32]);- assert!(encode_f32([]).is_empty());- assert!(decode_f32(&[]).is_empty());- }-- #[test]- fn f32_round_trips_and_ignores_trailing_bytes() {- let values = vec![0.5f32, -2.25, 1e-7];- let mut bytes = encode_f32(values.iter().copied());- assert_eq!(decode_f32(&bytes), values);- bytes.push(0xFF);- assert_eq!(decode_f32(&bytes), values);- }-}diff --git a/rust/crates/photos/src/ml_db/constants.rs b/rust/crates/photos/src/ml_db/constants.rsdeleted file mode 100644index 00e646f591..0000000000--- a/rust/crates/photos/src/ml_db/constants.rs+++ /dev/null@@ -1,15 +0,0 @@-pub const FACE_ML_VERSION: i64 = 1;-pub const PET_ML_VERSION: i64 = 1;--pub const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;-pub const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;-pub const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;-pub const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;-pub const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;--pub fn is_bad_face_for_clustering(face_score: f64, blur_value: f64, is_sideways: bool) -> bool {- face_score < MINIMUM_QUALITY_FACE_SCORE- || blur_value < LAPLACIAN_SOFT_THRESHOLD- || (blur_value < LAPLACIAN_VERY_SOFT_THRESHOLD && face_score < MEDIUM_QUALITY_FACE_SCORE)- || is_sideways-}diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rsindex da0932ec43..12fc4d893c 100644--- a/rust/crates/photos/src/ml_db/mod.rs+++ b/rust/crates/photos/src/ml_db/mod.rs@@ -1,30 +1,23 @@mod caches;-mod clip;-mod codec;-mod constants;mod filedata;+mod queries;mod schema;-mod types;+mod vector_encoding;use std::path::Path;use crate::db::Database;-pub use clip::{- CLIP_EMBEDDING_BYTES_LENGTH, CLIP_EMBEDDING_DIMENSIONS, CLIP_ML_VERSION, ClipEmbedding,- ClipRow, EmbeddingVector,-};-pub use codec::{decode_evector, decode_f32, encode_evector, encode_f32};-pub use constants::{- FACE_ML_VERSION, LAPLACIAN_HARD_THRESHOLD, LAPLACIAN_SOFT_THRESHOLD,- LAPLACIAN_VERY_SOFT_THRESHOLD, MEDIUM_QUALITY_FACE_SCORE, MINIMUM_QUALITY_FACE_SCORE,- PET_ML_VERSION, is_bad_face_for_clustering,-};pub use filedata::{FdStatus, PreviewInfo};-pub use types::{- ClusterCentroidRow, ClusterSummary, FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding,- PetBodyRow, PetBodyVectorRow, PetFaceRow, PetFaceVectorRow, PetRowsForFiles,+pub use queries::{+ CLIP_EMBEDDING_BYTES_LENGTH, CLIP_EMBEDDING_DIMENSIONS, CLIP_ML_VERSION, ClipEmbedding,+ ClipRow, ClusterCentroidRow, ClusterSummary, EmbeddingVector, FACE_ML_VERSION,+ FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding, LAPLACIAN_HARD_THRESHOLD,+ LAPLACIAN_SOFT_THRESHOLD, LAPLACIAN_VERY_SOFT_THRESHOLD, MEDIUM_QUALITY_FACE_SCORE,+ MINIMUM_QUALITY_FACE_SCORE, PET_ML_VERSION, PetBodyRow, PetBodyVectorRow, PetFaceRow,+ PetFaceVectorRow, PetRowsForFiles, is_bad_face_for_clustering,};+pub use vector_encoding::{decode_evector, decode_f32, encode_evector, encode_f32};#[derive(Debug, thiserror::Error)]pub enum Error {@@ -52,42 +45,15 @@ impl MlDb {db: Database::open(path, &schema::MIGRATION_SCRIPTS)?,})}-- pub fn clear_non_pet_tables(&self) -> Result<()> {- self.db- .execute_statements([- schema::DELETE_FACES,- schema::DELETE_FACE_CLUSTERS,- schema::DELETE_CLUSTER_PERSON,- schema::DELETE_CLUSTER_SUMMARY,- schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,- schema::DELETE_NOT_PERSON_FEEDBACK,- schema::DELETE_CLIP_EMBEDDINGS,- schema::DELETE_FILE_DATA,- ])- .map_err(Into::into)- }-- pub fn clear_pet_tables(&self) -> Result<()> {- self.db- .execute_statements([- schema::DELETE_PET_FACES,- schema::DELETE_PET_BODIES,- schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,- schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,- ])- .map_err(Into::into)- }}#[cfg(test)]mod tests {- use std::collections::HashSet;use std::fmt::Debug;use std::path::Path;- use super::clip::tests::full_clip;- use super::{Error, MlDb, TARGET_VERSION, caches, clip, filedata};+ use super::queries::tests::full_clip;+ use super::{Error, MlDb, TARGET_VERSION};use crate::db::Connection;use tempfile::TempDir;@@ -131,14 +97,6 @@ mod tests {}}- fn seeded() -> (TempDir, MlDb) {- let (directory, db) = open();- clip::tests::seed(&db);- filedata::tests::seed(&db);- caches::tests::seed(&db);- (directory, db)- }-#[test]fn ml_db_is_send_and_sync() {fn assert_send_sync<T: Send + Sync>() {}@@ -227,51 +185,4 @@ mod tests {assert_eq!(user_version(&path), 15);assert_eq!(db.count_clip_rows().unwrap(), 0);}-- #[test]- fn clear_non_pet_tables_leaves_pets_and_caches() {- let (_directory, db) = seeded();- db.clear_non_pet_tables().unwrap();- assert_eq!(db.count_clip_rows().unwrap(), 0);- assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());- assert_eq!(- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),- Some("1_0".to_string())- );- assert_eq!(- db.get_repeated_text_embedding_cache("dog").unwrap(),- Some(vec![0.5f32, -1.0])- );- }-- #[test]- fn clear_pet_tables_leaves_non_pet_tables() {- let (directory, db) = seeded();- let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();- connection- .execute_batch(- "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');- INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",- )- .unwrap();- db.clear_pet_tables().unwrap();- let row_count = |table: &str| -> i64 {- connection- .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {- row.get(0)- })- .unwrap()- };- assert_eq!(row_count("pet_face_vector_id_map"), 0);- assert_eq!(row_count("pet_body_vector_id_map"), 0);- assert_eq!(db.count_clip_rows().unwrap(), 4);- assert_eq!(- db.get_file_ids_with_fd_data(None).unwrap(),- HashSet::from([1, 2, 3])- );- assert_eq!(- db.get_face_id_used_for_person_or_cluster("p1").unwrap(),- Some("1_0".to_string())- );- }}diff --git a/rust/crates/photos/src/ml_db/clip.rs b/rust/crates/photos/src/ml_db/queries.rssimilarity index 56%rename from rust/crates/photos/src/ml_db/clip.rsrename to rust/crates/photos/src/ml_db/queries.rsindex f742fc35c1..53842d13b3 100644--- a/rust/crates/photos/src/ml_db/clip.rs+++ b/rust/crates/photos/src/ml_db/queries.rs@@ -2,14 +2,23 @@ use std::collections::HashMap;use crate::db::pair;-use super::codec::{decode_f32, encode_f32};-use super::schema::DELETE_CLIP_EMBEDDINGS;+use super::schema;+use super::vector_encoding::{decode_f32, encode_f32};use super::{MlDb, Result};+pub const FACE_ML_VERSION: i64 = 1;pub const CLIP_ML_VERSION: i64 = 1;+pub const PET_ML_VERSION: i64 = 1;+pub const CLIP_EMBEDDING_DIMENSIONS: usize = 512;pub const CLIP_EMBEDDING_BYTES_LENGTH: i64 = CLIP_EMBEDDING_DIMENSIONS as i64 * 4;+pub const LAPLACIAN_HARD_THRESHOLD: f64 = 10.0;+pub const LAPLACIAN_SOFT_THRESHOLD: f64 = 50.0;+pub const LAPLACIAN_VERY_SOFT_THRESHOLD: f64 = 200.0;+pub const MINIMUM_QUALITY_FACE_SCORE: f64 = 0.80;+pub const MEDIUM_QUALITY_FACE_SCORE: f64 = 0.85;+#[derive(Clone, Debug, PartialEq)]pub struct ClipEmbedding {pub file_id: i64,@@ -29,7 +38,124 @@ pub struct ClipRow {pub embedding: Vec<u8>,}+#[derive(Clone, Debug, PartialEq)]+pub struct FaceRow {+ pub file_id: i64,+ pub face_id: String,+ pub detection_json: String,+ pub embedding: Vec<f64>,+ pub score: f64,+ pub blur: f64,+ pub is_sideways: bool,+ pub image_height: i64,+ pub image_width: i64,+ pub ml_version: i64,+}++#[derive(Clone, Debug, PartialEq)]+pub struct FaceWithoutEmbedding {+ pub face_id: String,+ pub file_id: i64,+ pub score: f64,+ pub detection_json: String,+ pub blur: f64,+}++#[derive(Clone, Debug, PartialEq)]+pub struct FaceDbInfoForClustering {+ pub face_id: String,+ pub cluster_id: Option<String>,+ pub embedding_bytes: Vec<u8>,+ pub face_score: f64,+ pub blur_value: f64,+ pub is_sideways: bool,+}++#[derive(Clone, Debug, PartialEq)]+pub struct PetFaceRow {+ pub file_id: i64,+ pub pet_face_id: String,+ pub detection_json: String,+ pub face_vector_id: Option<i64>,+ pub species: i64,+ pub face_score: f64,+ pub image_height: i64,+ pub image_width: i64,+ pub ml_version: i64,+}++#[derive(Clone, Debug, PartialEq)]+pub struct PetBodyRow {+ pub file_id: i64,+ pub pet_body_id: String,+ pub detection_json: String,+ pub body_vector_id: Option<i64>,+ pub species: i64,+ pub score: f64,+ pub image_height: i64,+ pub image_width: i64,+ pub ml_version: i64,+}++#[derive(Clone, Debug, PartialEq)]+pub struct PetFaceVectorRow {+ pub pet_face_id: String,+ pub face_vector_id: Option<i64>,+ pub species: i64,+}++#[derive(Clone, Debug, PartialEq)]+pub struct PetBodyVectorRow {+ pub pet_body_id: String,+ pub body_vector_id: Option<i64>,+ pub species: i64,+}++#[derive(Clone, Debug, PartialEq)]+pub struct PetRowsForFiles {+ pub faces: Vec<PetFaceVectorRow>,+ pub bodies: Vec<PetBodyVectorRow>,+}++#[derive(Clone, Debug, PartialEq, Eq)]+pub struct ClusterSummary {+ pub avg: Vec<u8>,+ pub count: i64,+}++#[derive(Clone, Debug, PartialEq, Eq)]+pub struct ClusterCentroidRow {+ pub cluster_id: String,+ pub avg: Vec<u8>,+}+impl MlDb {+ pub fn clear_non_pet_tables(&self) -> Result<()> {+ self.db+ .execute_statements([+ schema::DELETE_FACES,+ schema::DELETE_FACE_CLUSTERS,+ schema::DELETE_CLUSTER_PERSON,+ schema::DELETE_CLUSTER_SUMMARY,+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,+ schema::DELETE_NOT_PERSON_FEEDBACK,+ schema::DELETE_CLIP_EMBEDDINGS,+ schema::DELETE_FILE_DATA,+ ])+ .map_err(Into::into)+ }++ pub fn clear_pet_tables(&self) -> Result<()> {+ self.db+ .execute_statements([+ schema::DELETE_PET_FACES,+ schema::DELETE_PET_BODIES,+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,+ ])+ .map_err(Into::into)+ }+pub fn get_all_clip_vectors(&self) -> Result<Vec<EmbeddingVector>> {let rows: Vec<(i64, Vec<u8>)> =self.db@@ -92,7 +218,7 @@ impl MlDb {pub fn delete_all_clip_rows(&self) -> Result<()> {self.db- .execute_statements([DELETE_CLIP_EMBEDDINGS])+ .execute_statements([schema::DELETE_CLIP_EMBEDDINGS]).map_err(Into::into)}@@ -118,6 +244,13 @@ impl MlDb {}}+pub fn is_bad_face_for_clustering(face_score: f64, blur_value: f64, is_sideways: bool) -> bool {+ face_score < MINIMUM_QUALITY_FACE_SCORE+ || blur_value < LAPLACIAN_SOFT_THRESHOLD+ || (blur_value < LAPLACIAN_VERY_SOFT_THRESHOLD && face_score < MEDIUM_QUALITY_FACE_SCORE)+ || is_sideways+}+fn clip_row(embedding: &ClipEmbedding) -> (i64, Vec<u8>, i64) {(embedding.file_id,@@ -128,11 +261,13 @@ fn clip_row(embedding: &ClipEmbedding) -> (i64, Vec<u8>, i64) {#[cfg(test)]pub(super) mod tests {- use std::collections::HashMap;+ use std::collections::{HashMap, HashSet};use super::{CLIP_EMBEDDING_DIMENSIONS, ClipEmbedding, ClipRow, MlDb};- use crate::ml_db::codec::encode_f32;+ use crate::db::Connection;use crate::ml_db::tests::{cases, check, open};+ use crate::ml_db::vector_encoding::encode_f32;+ use crate::ml_db::{caches, filedata};use tempfile::TempDir;fn clip(file_id: i64, embedding: Vec<f64>) -> ClipEmbedding {@@ -147,7 +282,7 @@ pub(super) mod tests {clip(file_id, vec![0.25; CLIP_EMBEDDING_DIMENSIONS])}- pub(in crate::ml_db) fn seed(db: &MlDb) {+ fn seed(db: &MlDb) {db.insert_clip_rows(&[full_clip(1),ClipEmbedding {@@ -166,6 +301,13 @@ pub(super) mod tests {(directory, db)}+ fn seeded_all() -> (TempDir, MlDb) {+ let (directory, db) = seeded();+ filedata::tests::seed(&db);+ caches::tests::seed(&db);+ (directory, db)+ }+#[test]fn seeded_counts() {let (_directory, db) = seeded();@@ -248,4 +390,51 @@ pub(super) mod tests {db.delete_all_clip_rows().unwrap();assert_eq!(db.count_clip_rows().unwrap(), 0);}++ #[test]+ fn clear_non_pet_tables_leaves_pets_and_caches() {+ let (_directory, db) = seeded_all();+ db.clear_non_pet_tables().unwrap();+ assert_eq!(db.count_clip_rows().unwrap(), 0);+ assert!(db.get_file_ids_with_fd_data(None).unwrap().is_empty());+ assert_eq!(+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),+ Some("1_0".to_string())+ );+ assert_eq!(+ db.get_repeated_text_embedding_cache("dog").unwrap(),+ Some(vec![0.5f32, -1.0])+ );+ }++ #[test]+ fn clear_pet_tables_leaves_non_pet_tables() {+ let (directory, db) = seeded_all();+ let connection = Connection::open(directory.path().join("ente.ml.db")).unwrap();+ connection+ .execute_batch(+ "INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('1_pet_0');+ INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('1_body_0');",+ )+ .unwrap();+ db.clear_pet_tables().unwrap();+ let row_count = |table: &str| -> i64 {+ connection+ .query_row(&format!("SELECT COUNT(*) FROM {table}"), (), |row| {+ row.get(0)+ })+ .unwrap()+ };+ assert_eq!(row_count("pet_face_vector_id_map"), 0);+ assert_eq!(row_count("pet_body_vector_id_map"), 0);+ assert_eq!(db.count_clip_rows().unwrap(), 4);+ assert_eq!(+ db.get_file_ids_with_fd_data(None).unwrap(),+ HashSet::from([1, 2, 3])+ );+ assert_eq!(+ db.get_face_id_used_for_person_or_cluster("p1").unwrap(),+ Some("1_0".to_string())+ );+ }}diff --git a/rust/crates/photos/src/ml_db/types.rs b/rust/crates/photos/src/ml_db/types.rsdeleted file mode 100644index b7a1a21581..0000000000--- a/rust/crates/photos/src/ml_db/types.rs+++ /dev/null@@ -1,90 +0,0 @@-#[derive(Clone, Debug, PartialEq)]-pub struct FaceRow {- pub file_id: i64,- pub face_id: String,- pub detection_json: String,- pub embedding: Vec<f64>,- pub score: f64,- pub blur: f64,- pub is_sideways: bool,- pub image_height: i64,- pub image_width: i64,- pub ml_version: i64,-}--#[derive(Clone, Debug, PartialEq)]-pub struct FaceWithoutEmbedding {- pub face_id: String,- pub file_id: i64,- pub score: f64,- pub detection_json: String,- pub blur: f64,-}--#[derive(Clone, Debug, PartialEq)]-pub struct FaceDbInfoForClustering {- pub face_id: String,- pub cluster_id: Option<String>,- pub embedding_bytes: Vec<u8>,- pub face_score: f64,- pub blur_value: f64,- pub is_sideways: bool,-}--#[derive(Clone, Debug, PartialEq)]-pub struct PetFaceRow {- pub file_id: i64,- pub pet_face_id: String,- pub detection_json: String,- pub face_vector_id: Option<i64>,- pub species: i64,- pub face_score: f64,- pub image_height: i64,- pub image_width: i64,- pub ml_version: i64,-}--#[derive(Clone, Debug, PartialEq)]-pub struct PetBodyRow {- pub file_id: i64,- pub pet_body_id: String,- pub detection_json: String,- pub body_vector_id: Option<i64>,- pub species: i64,- pub score: f64,- pub image_height: i64,- pub image_width: i64,- pub ml_version: i64,-}--#[derive(Clone, Debug, PartialEq)]-pub struct PetFaceVectorRow {- pub pet_face_id: String,- pub face_vector_id: Option<i64>,- pub species: i64,-}--#[derive(Clone, Debug, PartialEq)]-pub struct PetBodyVectorRow {- pub pet_body_id: String,- pub body_vector_id: Option<i64>,- pub species: i64,-}--#[derive(Clone, Debug, PartialEq)]-pub struct PetRowsForFiles {- pub faces: Vec<PetFaceVectorRow>,- pub bodies: Vec<PetBodyVectorRow>,-}--#[derive(Clone, Debug, PartialEq, Eq)]-pub struct ClusterSummary {- pub avg: Vec<u8>,- pub count: i64,-}--#[derive(Clone, Debug, PartialEq, Eq)]-pub struct ClusterCentroidRow {- pub cluster_id: String,- pub avg: Vec<u8>,-}diff --git a/rust/crates/photos/src/ml_db/vector_encoding.rs b/rust/crates/photos/src/ml_db/vector_encoding.rsnew file mode 100644index 0000000000..fa3b012ea9--- /dev/null+++ b/rust/crates/photos/src/ml_db/vector_encoding.rs@@ -0,0 +1,348 @@+use super::{Error, Result};++const EVECTOR_VALUES_PACKED_KEY: u32 = 0x0A;+const EVECTOR_VALUES_FIXED64_KEY: u32 = 0x09;+const MAX_GROUP_DEPTH: usize = 100;++pub fn encode_evector(values: &[f64]) -> Vec<u8> {+ if values.is_empty() {+ return Vec::new();+ }+ let mut bytes = Vec::with_capacity(values.len() * 8 + 10);+ bytes.push(EVECTOR_VALUES_PACKED_KEY as u8);+ push_varint(&mut bytes, (values.len() * 8) as u64);+ for value in values {+ bytes.extend_from_slice(&value.to_le_bytes());+ }+ bytes+}++pub fn decode_evector(bytes: &[u8]) -> Result<Vec<f64>> {+ let mut values = Vec::new();+ let mut offset = 0;+ while offset < bytes.len() {+ let key = read_key(bytes, &mut offset)?;+ match key {+ EVECTOR_VALUES_PACKED_KEY => {+ let length = read_length(bytes, &mut offset)?;+ if length % 8 != 0 {+ return Err(malformed());+ }+ for chunk in bytes[offset..offset + length].as_chunks::<8>().0 {+ values.push(f64::from_le_bytes(*chunk));+ }+ offset += length;+ }+ EVECTOR_VALUES_FIXED64_KEY => {+ let end = advance(bytes, &mut offset, 8)?;+ values.push(f64_from_le(&bytes[end - 8..end]));+ }+ _ if key >> 3 == 1 => return Err(malformed()),+ _ => skip_field(key, bytes, &mut offset, MAX_GROUP_DEPTH)?,+ }+ }+ Ok(values)+}++pub fn encode_f32(values: impl IntoIterator<Item = f32>) -> Vec<u8> {+ values+ .into_iter()+ .flat_map(|value| value.to_le_bytes())+ .collect()+}++pub fn decode_f32(bytes: &[u8]) -> Vec<f32> {+ bytes+ .as_chunks::<4>()+ .0+ .iter()+ .map(|chunk| f32::from_le_bytes(*chunk))+ .collect()+}++fn f64_from_le(chunk: &[u8]) -> f64 {+ let mut buffer = [0u8; 8];+ buffer.copy_from_slice(chunk);+ f64::from_le_bytes(buffer)+}++fn push_varint(bytes: &mut Vec<u8>, mut value: u64) {+ while value >= 0x80 {+ bytes.push((value as u8) | 0x80);+ value >>= 7;+ }+ bytes.push(value as u8);+}++fn read_key(bytes: &[u8], offset: &mut usize) -> Result<u32> {+ let key = u32::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;+ if key >> 3 == 0 || key & 7 > 5 {+ return Err(malformed());+ }+ Ok(key)+}++fn skip_field(key: u32, bytes: &[u8], offset: &mut usize, remaining_depth: usize) -> Result<()> {+ if remaining_depth == 0 {+ return Err(malformed());+ }+ match key & 7 {+ 0 => {+ read_varint(bytes, offset)?;+ }+ 1 => {+ advance(bytes, offset, 8)?;+ }+ 2 => {+ let length = read_length(bytes, offset)?;+ *offset += length;+ }+ 3 => loop {+ let inner_key = read_key(bytes, offset)?;+ if inner_key & 7 == 4 {+ if inner_key >> 3 != key >> 3 {+ return Err(malformed());+ }+ break;+ }+ skip_field(inner_key, bytes, offset, remaining_depth - 1)?;+ },+ 5 => {+ advance(bytes, offset, 4)?;+ }+ _ => return Err(malformed()),+ }+ Ok(())+}++fn read_varint(bytes: &[u8], offset: &mut usize) -> Result<u64> {+ let mut value = 0u64;+ for shift in (0..64).step_by(7) {+ let byte = *bytes.get(*offset).ok_or_else(malformed)?;+ *offset += 1;+ if shift == 63 && byte > 1 {+ return Err(malformed());+ }+ value |= u64::from(byte & 0x7F) << shift;+ if byte & 0x80 == 0 {+ return Ok(value);+ }+ }+ Err(malformed())+}++fn read_length(bytes: &[u8], offset: &mut usize) -> Result<usize> {+ let length = usize::try_from(read_varint(bytes, offset)?).map_err(|_| malformed())?;+ if bytes.len() - *offset < length {+ return Err(malformed());+ }+ Ok(length)+}++fn advance(bytes: &[u8], offset: &mut usize, count: usize) -> Result<usize> {+ if bytes.len() - *offset < count {+ return Err(malformed());+ }+ *offset += count;+ Ok(*offset)+}++fn malformed() -> Error {+ Error::Codec("malformed EVector bytes".to_string())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn evector_matches_protobuf_packed_encoding() {+ let bytes = encode_evector(&[1.0, 2.0]);+ assert_eq!(+ bytes,+ [+ 0x0A, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00,+ 0x00, 0x00, 0x00, 0x40+ ]+ );+ assert_eq!(decode_evector(&bytes).unwrap(), vec![1.0, 2.0]);+ }++ #[test]+ fn empty_evector_is_zero_bytes() {+ assert!(encode_evector(&[]).is_empty());+ assert!(decode_evector(&[]).unwrap().is_empty());+ }++ #[test]+ fn evector_round_trips_long_vectors() {+ let values: Vec<f64> = (0..192).map(|i| i as f64 * 0.25 - 3.0).collect();+ let bytes = encode_evector(&values);+ assert_eq!(bytes[0], 0x0A);+ assert_eq!(&bytes[1..3], &[0x80, 0x0C]);+ assert_eq!(decode_evector(&bytes).unwrap(), values);+ }++ #[test]+ fn evector_accepts_unpacked_and_unknown_fields() {+ let mut bytes = vec![0x09];+ bytes.extend_from_slice(&3.5f64.to_le_bytes());+ bytes.extend_from_slice(&[0x10, 0x05]);+ bytes.extend_from_slice(&[0x1A, 0x02, 0xAA, 0xBB]);+ bytes.extend_from_slice(&[0x25, 1, 2, 3, 4]);+ bytes.extend_from_slice(&[0x09]);+ bytes.extend_from_slice(&(-1.0f64).to_le_bytes());+ assert_eq!(decode_evector(&bytes).unwrap(), vec![3.5, -1.0]);+ }++ #[test]+ fn evector_rejects_truncated_bytes() {+ assert!(decode_evector(&[0x0A, 0x10, 0x00]).is_err());+ assert!(decode_evector(&[0x0A, 0x04, 0, 0, 0, 0]).is_err());+ assert!(decode_evector(&[0x0F]).is_err());+ for bytes in [+ &[0x09, 0, 0, 0, 0, 0, 0, 0][..],+ &[0x10],+ &[0x11, 0, 0, 0, 0, 0, 0, 0],+ &[0x12, 2, 0],+ &[0x15, 0, 0, 0],+ ] {+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");+ }+ }++ #[test]+ fn evector_concatenates_packed_and_unpacked_values() {+ let mut bytes = encode_evector(&[1.0, 2.0]);+ bytes.extend([0x10, 0x05, 0x09]);+ bytes.extend(3.0f64.to_le_bytes());+ bytes.extend([0x0A, 0]);+ bytes.extend(encode_evector(&[4.0, 5.0]));+ assert_eq!(decode_evector(&bytes).unwrap(), [1.0, 2.0, 3.0, 4.0, 5.0]);+ }++ #[test]+ fn evector_preserves_float_bits() {+ let bits = [+ 0,+ 1,+ 0x8000_0000_0000_0000,+ 0x7FF0_0000_0000_0000,+ 0xFFF0_0000_0000_0000,+ 0x7FF8_0000_0000_0042,+ ];+ let values: Vec<f64> = bits.into_iter().map(f64::from_bits).collect();+ let decoded = decode_evector(&encode_evector(&values)).unwrap();+ assert_eq!(+ decoded.into_iter().map(f64::to_bits).collect::<Vec<_>>(),+ bits+ );+ }++ #[test]+ fn evector_rejects_invalid_keys() {+ for bytes in [+ &[0, 0][..],+ &[0x02, 0],+ &[0x16],+ &[0x17],+ &[0x80, 0x80, 0x80, 0x80, 0x10, 0],+ ] {+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");+ }+ assert!(+ decode_evector(&[0xF8, 0xFF, 0xFF, 0xFF, 0x0F, 0])+ .unwrap()+ .is_empty()+ );+ }++ #[test]+ fn evector_rejects_incorrect_value_wire_types() {+ for bytes in [&[0x08, 0][..], &[0x0B, 0x0C], &[0x0C], &[0x0D, 0, 0, 0, 0]] {+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");+ }+ }++ #[test]+ fn evector_checks_varint_limits() {+ let mut maximum = vec![0x10];+ maximum.extend([0xFF; 9]);+ maximum.push(1);+ assert!(decode_evector(&maximum).unwrap().is_empty());++ let mut overflowing = vec![0x80; 9];+ overflowing.push(2);+ for invalid in [vec![0x80; 9], vec![0x80; 10], overflowing] {+ for prefix in [&[][..], &[0x0A][..], &[0x10][..], &[0x12][..]] {+ let mut bytes = prefix.to_vec();+ bytes.extend(&invalid);+ assert!(decode_evector(&bytes).is_err(), "{bytes:?}");+ }+ }++ for key in [0x0A, 0x12] {+ maximum[0] = key;+ assert!(decode_evector(&maximum).is_err());+ }+ }++ #[test]+ fn evector_skips_nested_unknown_groups() {+ let mut bytes = encode_evector(&[1.0]);+ bytes.extend([0x13, 0x09]);+ bytes.extend(99.0f64.to_le_bytes());+ bytes.extend([0x1B, 0x08, 1, 0x25, 1, 2, 3, 4, 0x29]);+ bytes.extend([0; 8]);+ bytes.extend([0x32, 2, 0xAA, 0xBB, 0x1C, 0x14]);+ bytes.extend(encode_evector(&[2.0]));+ assert_eq!(decode_evector(&bytes).unwrap(), [1.0, 2.0]);+ }++ #[test]+ fn evector_rejects_invalid_groups() {+ for bytes in [+ &[0x14][..],+ &[0x13],+ &[0x13, 0x1C],+ &[0x13, 0x1B, 0x14, 0x1C],+ &[0x13, 0x10, 0, 0x14, 0x14],+ &[0x13, 0, 0, 0x14],+ ] {+ assert!(decode_evector(bytes).is_err(), "{bytes:?}");+ }+ }++ #[test]+ fn evector_limits_group_nesting() {+ let nested = |depth, with_value| {+ let mut bytes = vec![0x13; depth];+ if with_value {+ bytes.extend([0x10, 0]);+ }+ bytes.extend(vec![0x14; depth]);+ bytes+ };+ assert!(decode_evector(&nested(99, true)).unwrap().is_empty());+ assert!(decode_evector(&nested(100, false)).unwrap().is_empty());+ assert!(decode_evector(&nested(100, true)).is_err());+ assert!(decode_evector(&nested(101, false)).is_err());+ }++ #[test]+ fn f32_matches_little_endian_layout() {+ assert_eq!(encode_f32([1.0f32]), [0x00, 0x00, 0x80, 0x3F]);+ assert_eq!(decode_f32(&[0x00, 0x00, 0x80, 0x3F]), vec![1.0f32]);+ assert!(encode_f32([]).is_empty());+ assert!(decode_f32(&[]).is_empty());+ }++ #[test]+ fn f32_round_trips_and_ignores_trailing_bytes() {+ let values = vec![0.5f32, -2.25, 1e-7];+ let mut bytes = encode_f32(values.iter().copied());+ assert_eq!(decode_f32(&bytes), values);+ bytes.push(0xFF);+ assert_eq!(decode_f32(&bytes), values);+ }+}