π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Centralize ML model runtime metadata
laurenspriem19c9c694580b
diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rsindex 3e2171ecc5..3bfcdd1b5e 100644--- a/rust/crates/photos/src/ml/assets.rs+++ b/rust/crates/photos/src/ml/assets.rs@@ -3,7 +3,9 @@ use std::path::{Path, PathBuf};use ente_assets::{Asset, AssetFile, AssetStore};-use super::runtime::ModelPaths;+use super::model::{self, ModelPaths};++pub use super::model::Model;const MODELS: &str = "models";@@ -22,33 +24,6 @@ const RETIRED_LEGACY_MODEL_FILES: &[&str] = &["mobilefacenet_opset15.onnx",];-#[derive(Clone, Copy, Debug, PartialEq, Eq)]-pub enum Model {- FaceDetection,- FaceEmbedding,- ClipImage,- ClipText,- PetFaceDetection,- PetFaceEmbeddingDog,- PetFaceEmbeddingCat,- PetBodyDetection,- PetBodyEmbeddingDog,- PetBodyEmbeddingCat,-}--const ALL_MODELS: [Model; 10] = [- Model::FaceDetection,- Model::FaceEmbedding,- Model::ClipImage,- Model::ClipText,- Model::PetFaceDetection,- Model::PetFaceEmbeddingDog,- Model::PetFaceEmbeddingCat,- Model::PetBodyDetection,- Model::PetBodyEmbeddingDog,- Model::PetBodyEmbeddingCat,-];-pub struct ClipTextPaths {pub model: PathBuf,pub vocab: PathBuf,@@ -71,24 +46,7 @@ pub fn model_asset(model: Model) -> Asset {}pub fn indexing_models(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Model> {- let mut models = Vec::new();- if run_faces {- models.extend([Model::FaceDetection, Model::FaceEmbedding]);- }- if run_clip {- models.push(Model::ClipImage);- }- if run_pets {- models.extend([- Model::PetFaceDetection,- Model::PetFaceEmbeddingDog,- Model::PetFaceEmbeddingCat,- Model::PetBodyDetection,- Model::PetBodyEmbeddingDog,- Model::PetBodyEmbeddingCat,- ]);- }- models+ model::enabled_indexing_models(run_faces, run_clip, run_pets).collect()}pub fn indexing_model_paths(@@ -97,18 +55,11 @@ pub fn indexing_model_paths(run_clip: bool,run_pets: bool,) -> ModelPaths {- ModelPaths {- face_detection: model_path_if(store, Model::FaceDetection, run_faces),- face_embedding: model_path_if(store, Model::FaceEmbedding, run_faces),- clip_image: model_path_if(store, Model::ClipImage, run_clip),- clip_text: String::new(),- pet_face_detection: model_path_if(store, Model::PetFaceDetection, run_pets),- pet_face_embedding_dog: model_path_if(store, Model::PetFaceEmbeddingDog, run_pets),- pet_face_embedding_cat: model_path_if(store, Model::PetFaceEmbeddingCat, run_pets),- pet_body_detection: model_path_if(store, Model::PetBodyDetection, run_pets),- pet_body_embedding_dog: model_path_if(store, Model::PetBodyEmbeddingDog, run_pets),- pet_body_embedding_cat: model_path_if(store, Model::PetBodyEmbeddingCat, run_pets),+ let mut model_paths = ModelPaths::default();+ for model in model::enabled_indexing_models(run_faces, run_clip, run_pets) {+ *model_paths.get_mut(model) = model_path(store, model);}+ model_paths}pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {@@ -127,7 +78,7 @@ pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {let mut warnings = Vec::new();if legacy_dir.exists() {- for model in ALL_MODELS {+ for model in Model::ALL {if let Err(error) = migrate_desktop_model(store, legacy_dir, model) {warnings.push(format!("Failed to migrate {}: {error}", model.spec().key));}@@ -255,10 +206,7 @@ struct ModelFileSpec {sha256: &'static str,}-fn model_path_if(store: &AssetStore, model: Model, enabled: bool) -> String {- if !enabled {- return String::new();- }+fn model_path(store: &AssetStore, model: Model) -> String {let asset = model_asset(model);store.file_path(&asset, model.spec().files[0].name)@@ -332,7 +280,7 @@ mod tests {fn catalog_keys_are_unique_and_prefix_free() {let root = Path::new("assets");let store = AssetStore::new(root);- let paths = ALL_MODELS+ let paths = Model::ALL.map(|model| store.asset_dir(&model_asset(model))).to_vec();for (index, path) in paths.iter().enumerate() {@@ -348,7 +296,7 @@ mod tests {#[test]fn retired_keys_are_not_in_the_catalog() {for key in RETIRED_MODEL_KEYS {- assert!(ALL_MODELS.iter().all(|model| model.spec().key != *key));+ assert!(Model::ALL.iter().all(|model| model.spec().key != *key));}}@@ -361,7 +309,7 @@ mod tests {for model in lock["models"].as_object().unwrap().values() {let name = model["file_name"].as_str().unwrap();let sha256 = model["sha256"].as_str().unwrap();- let catalog_file = ALL_MODELS+ let catalog_file = Model::ALL.iter().flat_map(|model| model.spec().files).find(|file| file.name == name)@@ -384,7 +332,7 @@ mod tests {let asset = model_asset(Model::FaceEmbedding);assert!(store.is_downloaded(&asset));assert_eq!(- fs::metadata(model_path_if(&store, Model::FaceEmbedding, true))+ fs::metadata(model_path(&store, Model::FaceEmbedding)).unwrap().len(),Model::FaceEmbedding.spec().files[0].sizediff --git a/rust/crates/photos/src/ml/clip/image.rs b/rust/crates/photos/src/ml/clip/image.rsindex 8ca06f56db..c8590231e5 100644--- a/rust/crates/photos/src/ml/clip/image.rs+++ b/rust/crates/photos/src/ml/clip/image.rs@@ -1,5 +1,6 @@use crate::ml::{error::MlResult,+ model::Model,onnx, preprocess,runtime::MlRuntimeView,types::{ClipResult, DecodedImage},@@ -12,7 +13,7 @@ pub(crate) fn run_clip_image(decoded: &DecodedImage,) -> MlResult<ClipResult> {let input = onnx::PreparedF32Input::new(preprocess::preprocess_clip(decoded)?);- let (shape, output) = runtime.with_clip_image_session(|session| {+ let (shape, output) = runtime.run(Model::ClipImage, |session| {onnx::run_f32(session,&input,diff --git a/rust/crates/photos/src/ml/clip/text.rs b/rust/crates/photos/src/ml/clip/text.rsindex 6e19d42aab..35df370759 100644--- a/rust/crates/photos/src/ml/clip/text.rs+++ b/rust/crates/photos/src/ml/clip/text.rs@@ -1,6 +1,7 @@use crate::ml::{clip::tokenizer,error::{MlError, MlResult},+ model::Model,onnx,runtime::MlRuntimeView,types::ClipResult,@@ -16,7 +17,7 @@ fn run_clip_text(runtime: &MlRuntimeView<'_>, token_ids: &[i32]) -> MlResult<Cli)));}- let (shape, output) = runtime.with_clip_text_session(|session| {+ let (shape, output) = runtime.run(Model::ClipText, |session| {onnx::run_i32_f32(session, token_ids, [1, CLIP_TEXT_TOKEN_COUNT as i64])})?;diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rsindex 71ab73c31a..4e76238e75 100644--- a/rust/crates/photos/src/ml/face/detect.rs+++ b/rust/crates/photos/src/ml/face/detect.rs@@ -1,5 +1,6 @@use crate::ml::{error::{MlError, MlResult},+ model::Model,onnx,postprocess::{NmsDetection, greedy_non_max_suppression},preprocess::{YOLO_INPUT_SIZE, YoloInput},@@ -24,7 +25,7 @@ pub(crate) fn run_face_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,) -> MlResult<Vec<FaceDetection>> {- runtime.with_face_detection_session(|session| {+ runtime.run(Model::FaceDetection, |session| {onnx::with_prepared_float_output(session,&input.tensor,diff --git a/rust/crates/photos/src/ml/face/embed.rs b/rust/crates/photos/src/ml/face/embed.rsindex 4a045b82cb..1bbc8f6806 100644--- a/rust/crates/photos/src/ml/face/embed.rs+++ b/rust/crates/photos/src/ml/face/embed.rs@@ -1,5 +1,6 @@use crate::ml::{error::{MlError, MlResult},+ model::Model,onnx,postprocess::l2_normalize,runtime::MlRuntimeView,@@ -38,7 +39,7 @@ pub(crate) fn run_face_embedding(}let input = onnx::PreparedF32Input::new(aligned);- let (shape, mut embedding) = runtime.with_face_embedding_session(|session| {+ let (shape, mut embedding) = runtime.run(Model::FaceEmbedding, |session| {onnx::run_f32(session,&input,diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex 4e9b2f786c..0acbf635bb 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -6,6 +6,7 @@ use crate::ml::{run_face_alignment, run_face_detection, run_face_embedding,thumbnail::{FaceBox, generate_face_thumbnails},},+ model::{self, Model},pet::{run_pet_body_detection, run_pet_body_embedding, run_pet_face_alignment,run_pet_face_detection, run_pet_face_embedding,@@ -240,9 +241,10 @@ pub fn run_clip_text(req: RunClipTextRequest) -> MlResult<RunClipTextResult> {} = req;if model_path.trim().is_empty() {- return Err(MlError::InvalidRequest(- "missing model path: clipTextModelPath".to_string(),- ));+ return Err(MlError::InvalidRequest(format!(+ "missing model path: {}",+ Model::ClipText.path_label()+ )));}if vocab_path.trim().is_empty() {return Err(MlError::InvalidRequest(@@ -250,18 +252,8 @@ pub fn run_clip_text(req: RunClipTextRequest) -> MlResult<RunClipTextResult> {));}- let model_paths = ModelPaths {- face_detection: String::new(),- face_embedding: String::new(),- clip_image: String::new(),- clip_text: model_path,- pet_face_detection: String::new(),- pet_face_embedding_dog: String::new(),- pet_face_embedding_cat: String::new(),- pet_body_detection: String::new(),- pet_body_embedding_dog: String::new(),- pet_body_embedding_cat: String::new(),- };+ let mut model_paths = ModelPaths::default();+ *model_paths.get_mut(Model::ClipText) = model_path;runtime::with_runtime(&model_paths, |runtime| {let clip = run_clip_text_query(runtime, &text, &vocab_path)?;@@ -291,40 +283,10 @@ pub fn tokenize_clip_text(text: &str, vocab_path: &str) -> MlResult<Vec<i32>> {}fn validate_request_model_paths(req: &AnalyzeImageRequest) -> MlResult<()> {- let model_paths = &req.model_paths;-- let mut missing = Vec::new();- if req.run_faces {- if model_paths.face_detection.trim().is_empty() {- missing.push("faceDetectionModelPath");- }- if model_paths.face_embedding.trim().is_empty() {- missing.push("faceEmbeddingModelPath");- }- }- if req.run_clip && model_paths.clip_image.trim().is_empty() {- missing.push("clipImageModelPath");- }- if req.run_pets {- if model_paths.pet_face_detection.trim().is_empty() {- missing.push("petFaceDetectionModelPath");- }- if model_paths.pet_body_detection.trim().is_empty() {- missing.push("petBodyDetectionModelPath");- }- if model_paths.pet_face_embedding_dog.trim().is_empty() {- missing.push("petFaceEmbeddingDogModelPath");- }- if model_paths.pet_face_embedding_cat.trim().is_empty() {- missing.push("petFaceEmbeddingCatModelPath");- }- if model_paths.pet_body_embedding_dog.trim().is_empty() {- missing.push("petBodyEmbeddingDogModelPath");- }- if model_paths.pet_body_embedding_cat.trim().is_empty() {- missing.push("petBodyEmbeddingCatModelPath");- }- }+ let missing = model::required_indexing_models(req.run_faces, req.run_clip, req.run_pets)+ .filter(|&model| req.model_paths.get(model).trim().is_empty())+ .map(Model::path_label)+ .collect::<Vec<_>>();if missing.is_empty() {return Ok(());}diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rsindex 4068c879cc..b3fd32e012 100644--- a/rust/crates/photos/src/ml/mod.rs+++ b/rust/crates/photos/src/ml/mod.rs@@ -7,6 +7,7 @@ pub mod golden;mod golden_data;pub mod golden_tooling;pub mod indexing;+mod model;mod onnx;mod pet;mod postprocess;diff --git a/rust/crates/photos/src/ml/model.rs b/rust/crates/photos/src/ml/model.rsnew file mode 100644index 0000000000..b8770a8412--- /dev/null+++ b/rust/crates/photos/src/ml/model.rs@@ -0,0 +1,271 @@+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+#[repr(usize)]+pub enum Model {+ FaceDetection,+ FaceEmbedding,+ ClipImage,+ ClipText,+ PetFaceDetection,+ PetFaceEmbeddingDog,+ PetFaceEmbeddingCat,+ PetBodyDetection,+ PetBodyEmbeddingDog,+ PetBodyEmbeddingCat,+}++impl Model {+ pub(crate) const COUNT: usize = Self::PetBodyEmbeddingCat as usize + 1;++ pub(crate) const ALL: [Self; Self::COUNT] = [+ Self::FaceDetection,+ Self::FaceEmbedding,+ Self::ClipImage,+ Self::ClipText,+ Self::PetFaceDetection,+ Self::PetFaceEmbeddingDog,+ Self::PetFaceEmbeddingCat,+ Self::PetBodyDetection,+ Self::PetBodyEmbeddingDog,+ Self::PetBodyEmbeddingCat,+ ];++ pub(crate) const INDEXING: [Self; 9] = [+ Self::FaceDetection,+ Self::FaceEmbedding,+ Self::ClipImage,+ Self::PetFaceDetection,+ Self::PetFaceEmbeddingDog,+ Self::PetFaceEmbeddingCat,+ Self::PetBodyDetection,+ Self::PetBodyEmbeddingDog,+ Self::PetBodyEmbeddingCat,+ ];++ const INDEXING_PATH_VALIDATION: [Self; 9] = [+ Self::FaceDetection,+ Self::FaceEmbedding,+ Self::ClipImage,+ Self::PetFaceDetection,+ Self::PetBodyDetection,+ Self::PetFaceEmbeddingDog,+ Self::PetFaceEmbeddingCat,+ Self::PetBodyEmbeddingDog,+ Self::PetBodyEmbeddingCat,+ ];++ pub(crate) const fn index(self) -> usize {+ self as usize+ }++ pub(crate) const fn path_label(self) -> &'static str {+ match self {+ Self::FaceDetection => "faceDetectionModelPath",+ Self::FaceEmbedding => "faceEmbeddingModelPath",+ Self::ClipImage => "clipImageModelPath",+ Self::ClipText => "clipTextModelPath",+ Self::PetFaceDetection => "petFaceDetectionModelPath",+ Self::PetFaceEmbeddingDog => "petFaceEmbeddingDogModelPath",+ Self::PetFaceEmbeddingCat => "petFaceEmbeddingCatModelPath",+ Self::PetBodyDetection => "petBodyDetectionModelPath",+ Self::PetBodyEmbeddingDog => "petBodyEmbeddingDogModelPath",+ Self::PetBodyEmbeddingCat => "petBodyEmbeddingCatModelPath",+ }+ }++ pub(crate) const fn namespace(self) -> &'static str {+ match self {+ Self::FaceDetection => "face-detection",+ Self::FaceEmbedding => "face-embedding",+ Self::ClipImage => "clip-image",+ Self::ClipText => "clip-text",+ Self::PetFaceDetection => "pet-face-detection",+ Self::PetFaceEmbeddingDog => "pet-face-embedding-dog",+ Self::PetFaceEmbeddingCat => "pet-face-embedding-cat",+ Self::PetBodyDetection => "pet-body-detection",+ Self::PetBodyEmbeddingDog => "pet-body-embedding-dog",+ Self::PetBodyEmbeddingCat => "pet-body-embedding-cat",+ }+ }++ pub(crate) fn missing_path_error(self) -> String {+ let requirement = match self {+ Self::FaceDetection | Self::FaceEmbedding => " is required when runFaces is true",+ Self::ClipImage => " is required when runClip is true",+ Self::ClipText => " is required when running clip text",+ Self::PetFaceDetection | Self::PetBodyDetection => " is required when runPets is true",+ Self::PetFaceEmbeddingDog+ | Self::PetFaceEmbeddingCat+ | Self::PetBodyEmbeddingDog+ | Self::PetBodyEmbeddingCat => " is required",+ };+ format!("missing model path: {}{requirement}", self.path_label())+ }+}++#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct ModelPaths {+ pub face_detection: String,+ pub face_embedding: String,+ pub clip_image: String,+ pub clip_text: String,+ pub pet_face_detection: String,+ pub pet_face_embedding_dog: String,+ pub pet_face_embedding_cat: String,+ pub pet_body_detection: String,+ pub pet_body_embedding_dog: String,+ pub pet_body_embedding_cat: String,+}++impl ModelPaths {+ pub(crate) fn get(&self, model: Model) -> &str {+ match model {+ Model::FaceDetection => &self.face_detection,+ Model::FaceEmbedding => &self.face_embedding,+ Model::ClipImage => &self.clip_image,+ Model::ClipText => &self.clip_text,+ Model::PetFaceDetection => &self.pet_face_detection,+ Model::PetFaceEmbeddingDog => &self.pet_face_embedding_dog,+ Model::PetFaceEmbeddingCat => &self.pet_face_embedding_cat,+ Model::PetBodyDetection => &self.pet_body_detection,+ Model::PetBodyEmbeddingDog => &self.pet_body_embedding_dog,+ Model::PetBodyEmbeddingCat => &self.pet_body_embedding_cat,+ }+ }++ pub(crate) fn get_mut(&mut self, model: Model) -> &mut String {+ match model {+ Model::FaceDetection => &mut self.face_detection,+ Model::FaceEmbedding => &mut self.face_embedding,+ Model::ClipImage => &mut self.clip_image,+ Model::ClipText => &mut self.clip_text,+ Model::PetFaceDetection => &mut self.pet_face_detection,+ Model::PetFaceEmbeddingDog => &mut self.pet_face_embedding_dog,+ Model::PetFaceEmbeddingCat => &mut self.pet_face_embedding_cat,+ Model::PetBodyDetection => &mut self.pet_body_detection,+ Model::PetBodyEmbeddingDog => &mut self.pet_body_embedding_dog,+ Model::PetBodyEmbeddingCat => &mut self.pet_body_embedding_cat,+ }+ }+}++pub(crate) fn enabled_indexing_models(+ run_faces: bool,+ run_clip: bool,+ run_pets: bool,+) -> impl Iterator<Item = Model> {+ Model::INDEXING+ .into_iter()+ .filter(move |&model| indexing_model_is_enabled(model, run_faces, run_clip, run_pets))+}++pub(crate) fn required_indexing_models(+ run_faces: bool,+ run_clip: bool,+ run_pets: bool,+) -> impl Iterator<Item = Model> {+ Model::INDEXING_PATH_VALIDATION+ .into_iter()+ .filter(move |&model| indexing_model_is_enabled(model, run_faces, run_clip, run_pets))+}++fn indexing_model_is_enabled(+ model: Model,+ run_faces: bool,+ run_clip: bool,+ run_pets: bool,+) -> bool {+ match model {+ Model::FaceDetection | Model::FaceEmbedding => run_faces,+ Model::ClipImage => run_clip,+ Model::PetFaceDetection+ | Model::PetFaceEmbeddingDog+ | Model::PetFaceEmbeddingCat+ | Model::PetBodyDetection+ | Model::PetBodyEmbeddingDog+ | Model::PetBodyEmbeddingCat => run_pets,+ Model::ClipText => false,+ }+}++#[cfg(test)]+mod tests {+ use std::collections::HashSet;++ use super::*;++ #[test]+ fn model_catalog_is_array_indexed_and_unique() {+ let mut path_labels = HashSet::new();+ let mut namespaces = HashSet::new();++ for (index, model) in Model::ALL.into_iter().enumerate() {+ assert_eq!(model.index(), index);+ assert!(path_labels.insert(model.path_label()));+ assert!(namespaces.insert(model.namespace()));+ }+ }++ #[test]+ fn model_paths_are_addressable_by_model() {+ let mut paths = ModelPaths::default();++ for model in Model::ALL {+ *paths.get_mut(model) = model.namespace().to_string();+ }++ for model in Model::ALL {+ assert_eq!(paths.get(model), model.namespace());+ }+ }++ #[test]+ fn indexing_catalog_contains_every_model_except_clip_text() {+ let indexes = Model::INDEXING+ .into_iter()+ .map(Model::index)+ .collect::<HashSet<_>>();++ assert_eq!(indexes.len(), Model::COUNT - 1);+ for model in Model::ALL {+ assert_eq!(indexes.contains(&model.index()), model != Model::ClipText);+ }+ assert_eq!(+ Model::INDEXING_PATH_VALIDATION+ .into_iter()+ .map(Model::index)+ .collect::<HashSet<_>>(),+ indexes+ );++ assert_eq!(+ enabled_indexing_models(true, false, false).collect::<Vec<_>>(),+ [Model::FaceDetection, Model::FaceEmbedding]+ );+ assert_eq!(+ enabled_indexing_models(false, true, false).collect::<Vec<_>>(),+ [Model::ClipImage]+ );+ assert_eq!(+ enabled_indexing_models(false, false, true).collect::<Vec<_>>(),+ [+ Model::PetFaceDetection,+ Model::PetFaceEmbeddingDog,+ Model::PetFaceEmbeddingCat,+ Model::PetBodyDetection,+ Model::PetBodyEmbeddingDog,+ Model::PetBodyEmbeddingCat,+ ]+ );+ assert_eq!(+ required_indexing_models(false, false, true).collect::<Vec<_>>(),+ [+ Model::PetFaceDetection,+ Model::PetBodyDetection,+ Model::PetFaceEmbeddingDog,+ Model::PetFaceEmbeddingCat,+ Model::PetBodyEmbeddingDog,+ Model::PetBodyEmbeddingCat,+ ]+ );+ }+}diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rsindex 52814e5548..152db8a975 100644--- a/rust/crates/photos/src/ml/pet/detect.rs+++ b/rust/crates/photos/src/ml/pet/detect.rs@@ -1,5 +1,6 @@use crate::ml::{error::{MlError, MlResult},+ model::Model,onnx,postprocess::{NmsDetection, greedy_non_max_suppression},preprocess::{YOLO_INPUT_SIZE, YoloInput},@@ -48,7 +49,7 @@ pub(crate) fn run_pet_face_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,) -> MlResult<Vec<PetFaceDetection>> {- runtime.with_pet_face_detection_session(|session| {+ runtime.run(Model::PetFaceDetection, |session| {onnx::with_prepared_float_output(session,&input.tensor,@@ -182,7 +183,7 @@ pub(crate) fn run_pet_body_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,) -> MlResult<Vec<PetBodyDetection>> {- runtime.with_pet_body_detection_session(|session| {+ runtime.run(Model::PetBodyDetection, |session| {onnx::with_prepared_float_output(session,&input.tensor,diff --git a/rust/crates/photos/src/ml/pet/embed.rs b/rust/crates/photos/src/ml/pet/embed.rsindex 745e377159..e56b105603 100644--- a/rust/crates/photos/src/ml/pet/embed.rs+++ b/rust/crates/photos/src/ml/pet/embed.rs@@ -1,5 +1,6 @@use crate::ml::{error::{MlError, MlResult},+ model::Model,onnx,postprocess::l2_normalize,runtime::MlRuntimeView,@@ -52,15 +53,13 @@ pub(crate) fn run_pet_face_embedding(PET_EMBEDDING_INPUT_SIZE as i64,PET_EMBEDDING_INPUT_SIZE as i64,];- let (shape, output) = if species == PET_SPECIES_DOG {- runtime.with_pet_face_embedding_dog_session(|session| {- onnx::run_f32(session, &input, input_shape)- })?+ let model = if species == PET_SPECIES_DOG {+ Model::PetFaceEmbeddingDog} else {- runtime.with_pet_face_embedding_cat_session(|session| {- onnx::run_f32(session, &input, input_shape)- })?+ Model::PetFaceEmbeddingCat};+ let (shape, output) =+ runtime.run(model, |session| onnx::run_f32(session, &input, input_shape))?;let embedding_size =validate_embedding_batch_output("face", &shape, output.len(), batch.indices.len())?;@@ -134,15 +133,13 @@ pub(crate) fn run_pet_body_embedding(PET_EMBEDDING_INPUT_SIZE as i64,PET_EMBEDDING_INPUT_SIZE as i64,];- let (shape, output) = if is_cat {- runtime.with_pet_body_embedding_cat_session(|session| {- onnx::run_f32(session, &input, input_shape)- })?+ let model = if is_cat {+ Model::PetBodyEmbeddingCat} else {- runtime.with_pet_body_embedding_dog_session(|session| {- onnx::run_f32(session, &input, input_shape)- })?+ Model::PetBodyEmbeddingDog};+ let (shape, output) =+ runtime.run(model, |session| onnx::run_f32(session, &input, input_shape))?;let embedding_size =validate_embedding_batch_output("body", &shape, output.len(), batch.indices.len())?;diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex 6ac9ecfdf9..065a78609e 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -9,22 +9,11 @@ use ort::session::Session;use crate::ml::{error::{MlError, MlResult},+ model::Model,onnx,};-#[derive(Clone, Debug, Default, PartialEq, Eq)]-pub struct ModelPaths {- pub face_detection: String,- pub face_embedding: String,- pub clip_image: String,- pub clip_text: String,- pub pet_face_detection: String,- pub pet_face_embedding_dog: String,- pub pet_face_embedding_cat: String,- pub pet_body_detection: String,- pub pet_body_embedding_dog: String,- pub pet_body_embedding_cat: String,-}+pub use crate::ml::model::ModelPaths;#[derive(Debug)]struct ModelSlotState {@@ -35,8 +24,8 @@ struct ModelSlotState {#[derive(Debug)]struct ModelSlot {+ model: Model,default_execution_mode: onnx::ExecutionMode,- model_namespace: &'static str,state: Mutex<ModelSlotState>,}@@ -63,10 +52,10 @@ pub(crate) struct MlRuntimeView<'a> {}impl ModelSlot {- fn new(default_execution_mode: onnx::ExecutionMode, model_namespace: &'static str) -> Self {+ fn new(model: Model) -> Self {Self {- default_execution_mode,- model_namespace,+ model,+ default_execution_mode: default_execution_mode(model),state: Mutex::new(ModelSlotState {path: String::new(),provider_plan: None,@@ -108,17 +97,16 @@ impl ModelSlot {fn run<T>(&self,path: &str,- error_msg: &str,mut operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,) -> MlResult<(T, onnx::ExecutionProvider)> {if path.trim().is_empty() {- return Err(MlError::InvalidRequest(error_msg.to_string()));+ return Err(MlError::InvalidRequest(self.model.missing_path_error()));}loop {let mut state = self.lock_state();Self::set_config_locked(&mut state, path);- self.ensure_loaded_locked(&mut state, error_msg)?;+ self.ensure_loaded_locked(&mut state)?;let execution_provider = state.provider_plan@@ -182,9 +170,9 @@ impl ModelSlot {Self::clear_transient_runtime_state_locked(state);}- fn ensure_loaded_locked(&self, state: &mut ModelSlotState, error_msg: &str) -> MlResult<()> {+ fn ensure_loaded_locked(&self, state: &mut ModelSlotState) -> MlResult<()> {if state.path.trim().is_empty() {- return Err(MlError::InvalidRequest(error_msg.to_string()));+ return Err(MlError::InvalidRequest(self.model.missing_path_error()));}if state.session.is_some() {return Ok(());@@ -202,7 +190,7 @@ impl ModelSlot {self.default_execution_mode);let t = std::time::Instant::now();- let session = onnx::build_session(&state.path, provider_plan, self.model_namespace)?;+ let session = onnx::build_session(&state.path, provider_plan, self.model.namespace())?;let execution_provider = provider_plan.selected_provider().expect("successful session build must select an execution provider");@@ -217,97 +205,40 @@ impl ModelSlot {#[derive(Debug)]struct MlRuntime {- face_detection: ModelSlot,- face_embedding: ModelSlot,- clip_image: ModelSlot,- clip_text: ModelSlot,- pet_face_detection: ModelSlot,- pet_face_embedding_dog: ModelSlot,- pet_face_embedding_cat: ModelSlot,- pet_body_detection: ModelSlot,- pet_body_embedding_dog: ModelSlot,- pet_body_embedding_cat: ModelSlot,+ slots: [ModelSlot; Model::COUNT],}static GLOBAL_RUNTIME: Lazy<MlRuntime> = Lazy::new(MlRuntime::new);impl MlRuntime {fn new() -> Self {- let platform_default = onnx::ExecutionMode::PlatformDefault;- let cpu_only = onnx::ExecutionMode::CpuOnly;-Self {- face_detection: ModelSlot::new(platform_default, "face-detection"),- face_embedding: ModelSlot::new(platform_default, "face-embedding"),- clip_image: ModelSlot::new(platform_default, "clip-image"),- // The quantized CLIP text graph is heavily partitioned by both- // CoreML and WebGPU, making their mixed CPU/GPU execution slower- // than running the complete model on CPU.- clip_text: ModelSlot::new(cpu_only, "clip-text"),- // Pet models stay CPU-only due to device-specific FP16 failures.- pet_face_detection: ModelSlot::new(cpu_only, "pet-face-detection"),- pet_face_embedding_dog: ModelSlot::new(cpu_only, "pet-face-embedding-dog"),- pet_face_embedding_cat: ModelSlot::new(cpu_only, "pet-face-embedding-cat"),- pet_body_detection: ModelSlot::new(cpu_only, "pet-body-detection"),- pet_body_embedding_dog: ModelSlot::new(cpu_only, "pet-body-embedding-dog"),- pet_body_embedding_cat: ModelSlot::new(cpu_only, "pet-body-embedding-cat"),+ slots: Model::ALL.map(ModelSlot::new),}}+ fn slot(&self, model: Model) -> &ModelSlot {+ &self.slots[model.index()]+ }+fn configure_requested_models(&self, model_paths: &ModelPaths) {- self.face_detection- .configure_if_requested(&model_paths.face_detection);- self.face_embedding- .configure_if_requested(&model_paths.face_embedding);- self.clip_image- .configure_if_requested(&model_paths.clip_image);- self.clip_text- .configure_if_requested(&model_paths.clip_text);- self.pet_face_detection- .configure_if_requested(&model_paths.pet_face_detection);- self.pet_face_embedding_dog- .configure_if_requested(&model_paths.pet_face_embedding_dog);- self.pet_face_embedding_cat- .configure_if_requested(&model_paths.pet_face_embedding_cat);- self.pet_body_detection- .configure_if_requested(&model_paths.pet_body_detection);- self.pet_body_embedding_dog- .configure_if_requested(&model_paths.pet_body_embedding_dog);- self.pet_body_embedding_cat- .configure_if_requested(&model_paths.pet_body_embedding_cat);+ for model in Model::ALL {+ self.slot(model)+ .configure_if_requested(model_paths.get(model));+ }}fn prepare_indexing_models(&self, model_paths: &ModelPaths) {- self.face_detection- .sync_indexing_residency(&model_paths.face_detection);- self.face_embedding- .sync_indexing_residency(&model_paths.face_embedding);- self.clip_image- .sync_indexing_residency(&model_paths.clip_image);- self.pet_face_detection- .sync_indexing_residency(&model_paths.pet_face_detection);- self.pet_face_embedding_dog- .sync_indexing_residency(&model_paths.pet_face_embedding_dog);- self.pet_face_embedding_cat- .sync_indexing_residency(&model_paths.pet_face_embedding_cat);- self.pet_body_detection- .sync_indexing_residency(&model_paths.pet_body_detection);- self.pet_body_embedding_dog- .sync_indexing_residency(&model_paths.pet_body_embedding_dog);- self.pet_body_embedding_cat- .sync_indexing_residency(&model_paths.pet_body_embedding_cat);+ for model in Model::INDEXING {+ self.slot(model)+ .sync_indexing_residency(model_paths.get(model));+ }}fn release_indexing_models(&self) {- self.face_detection.release_residency();- self.face_embedding.release_residency();- self.clip_image.release_residency();- self.pet_face_detection.release_residency();- self.pet_face_embedding_dog.release_residency();- self.pet_face_embedding_cat.release_residency();- self.pet_body_detection.release_residency();- self.pet_body_embedding_dog.release_residency();- self.pet_body_embedding_cat.release_residency();+ for model in Model::INDEXING {+ self.slot(model).release_residency();+ }}fn view<'a>(&'a self, model_paths: &'a ModelPaths) -> MlRuntimeView<'a> {@@ -324,138 +255,36 @@ impl<'a> MlRuntimeView<'a> {self.used_providers.get()}- fn run_tracked<T>(+ pub(crate) fn run<T>(&self,- slot: &ModelSlot,- path: &str,- error_msg: &str,+ model: Model,operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,) -> MlResult<T> {- let (value, execution_provider) = slot.run(path, error_msg, operation)?;+ let (value, execution_provider) = self+ .runtime+ .slot(model)+ .run(self.model_paths.get(model), operation)?;let mut used = self.used_providers.get();used.record(execution_provider);self.used_providers.set(used);Ok(value)}+}- pub(crate) fn with_face_detection_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.face_detection,- &self.model_paths.face_detection,- "missing model path: faceDetectionModelPath is required when runFaces is true",- operation,- )- }-- pub(crate) fn with_face_embedding_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.face_embedding,- &self.model_paths.face_embedding,- "missing model path: faceEmbeddingModelPath is required when runFaces is true",- operation,- )- }-- pub(crate) fn with_clip_image_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.clip_image,- &self.model_paths.clip_image,- "missing model path: clipImageModelPath is required when runClip is true",- operation,- )- }-- pub(crate) fn with_clip_text_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.clip_text,- &self.model_paths.clip_text,- "missing model path: clipTextModelPath is required when running clip text",- operation,- )- }-- pub(crate) fn with_pet_face_detection_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.pet_face_detection,- &self.model_paths.pet_face_detection,- "missing model path: petFaceDetectionModelPath is required when runPets is true",- operation,- )- }-- pub(crate) fn with_pet_face_embedding_dog_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.pet_face_embedding_dog,- &self.model_paths.pet_face_embedding_dog,- "missing model path: petFaceEmbeddingDogModelPath is required",- operation,- )- }-- pub(crate) fn with_pet_face_embedding_cat_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.pet_face_embedding_cat,- &self.model_paths.pet_face_embedding_cat,- "missing model path: petFaceEmbeddingCatModelPath is required",- operation,- )- }-- pub(crate) fn with_pet_body_detection_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.pet_body_detection,- &self.model_paths.pet_body_detection,- "missing model path: petBodyDetectionModelPath is required when runPets is true",- operation,- )- }-- pub(crate) fn with_pet_body_embedding_dog_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.pet_body_embedding_dog,- &self.model_paths.pet_body_embedding_dog,- "missing model path: petBodyEmbeddingDogModelPath is required",- operation,- )- }-- pub(crate) fn with_pet_body_embedding_cat_session<T>(- &self,- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,- ) -> MlResult<T> {- self.run_tracked(- &self.runtime.pet_body_embedding_cat,- &self.model_paths.pet_body_embedding_cat,- "missing model path: petBodyEmbeddingCatModelPath is required",- operation,- )+fn default_execution_mode(model: Model) -> onnx::ExecutionMode {+ match model {+ Model::FaceDetection | Model::FaceEmbedding | Model::ClipImage => {+ onnx::ExecutionMode::PlatformDefault+ }+ // Accelerators heavily partition the quantized CLIP text graph.+ Model::ClipText => onnx::ExecutionMode::CpuOnly,+ // Pet models have device-specific FP16 failures.+ Model::PetFaceDetection+ | Model::PetFaceEmbeddingDog+ | Model::PetFaceEmbeddingCat+ | Model::PetBodyDetection+ | Model::PetBodyEmbeddingDog+ | Model::PetBodyEmbeddingCat => onnx::ExecutionMode::CpuOnly,}}@@ -499,7 +328,7 @@ mod tests {..ModelPaths::default()});- let clip_text = runtime.clip_text.lock_state();+ let clip_text = runtime.slot(Model::ClipText).lock_state();assert_eq!(clip_text.path, "clip_text.onnx");}@@ -517,7 +346,7 @@ mod tests {let runtime = MlRuntime::new();{- let mut clip_text = runtime.clip_text.lock_state();+ let mut clip_text = runtime.slot(Model::ClipText).lock_state();clip_text.path = "clip_text.onnx".to_string();clip_text.provider_plan = Some(onnx::ProviderPlan::new(onnx::ExecutionMode::CpuOnly,@@ -525,7 +354,7 @@ mod tests {));}{- let mut face_detection = runtime.face_detection.lock_state();+ let mut face_detection = runtime.slot(Model::FaceDetection).lock_state();face_detection.path = "face.onnx".to_string();face_detection.provider_plan = Some(onnx::ProviderPlan::new(onnx::ExecutionMode::CpuOnly,@@ -535,11 +364,11 @@ mod tests {runtime.release_indexing_models();- let clip_text = runtime.clip_text.lock_state();+ let clip_text = runtime.slot(Model::ClipText).lock_state();assert_eq!(clip_text.path, "clip_text.onnx");assert!(clip_text.provider_plan.is_some());- let face_detection = runtime.face_detection.lock_state();+ let face_detection = runtime.slot(Model::FaceDetection).lock_state();assert!(face_detection.provider_plan.is_none());}@@ -554,22 +383,22 @@ mod tests {..ModelPaths::default()});- let face_detection = runtime.face_detection.lock_state();+ let face_detection = runtime.slot(Model::FaceDetection).lock_state();assert_eq!(face_detection.path, "face.onnx");assert!(face_detection.session.is_none());- let face_embedding = runtime.face_embedding.lock_state();+ let face_embedding = runtime.slot(Model::FaceEmbedding).lock_state();assert_eq!(face_embedding.path, "embed.onnx");assert!(face_embedding.session.is_none());- let clip_image = runtime.clip_image.lock_state();+ let clip_image = runtime.slot(Model::ClipImage).lock_state();assert_eq!(clip_image.path, "clip.onnx");assert!(clip_image.session.is_none());}#[test]fn sync_indexing_residency_clears_disabled_slots() {- let slot = ModelSlot::new(onnx::ExecutionMode::PlatformDefault, "test-model");+ let slot = ModelSlot::new(Model::ClipImage);{let mut state = slot.lock_state();@@ -590,7 +419,7 @@ mod tests {#[test]fn release_residency_resets_transient_cpu_fallback_for_any_slot() {- let slot = ModelSlot::new(onnx::ExecutionMode::PlatformDefault, "test-model");+ let slot = ModelSlot::new(Model::ClipImage);{let mut state = slot.lock_state();@@ -613,44 +442,56 @@ mod tests {let runtime = MlRuntime::new();let expected_pet_mode = onnx::ExecutionMode::CpuOnly;+ for model in Model::ALL {+ assert_eq!(runtime.slot(model).model, model);+ }+assert_eq!(- runtime.face_detection.default_execution_mode,+ runtime.slot(Model::FaceDetection).default_execution_mode,onnx::ExecutionMode::PlatformDefault);assert_eq!(- runtime.face_embedding.default_execution_mode,+ runtime.slot(Model::FaceEmbedding).default_execution_mode,onnx::ExecutionMode::PlatformDefault);assert_eq!(- runtime.clip_image.default_execution_mode,+ runtime.slot(Model::ClipImage).default_execution_mode,onnx::ExecutionMode::PlatformDefault);assert_eq!(- runtime.clip_text.default_execution_mode,+ runtime.slot(Model::ClipText).default_execution_mode,onnx::ExecutionMode::CpuOnly);assert_eq!(- runtime.pet_face_detection.default_execution_mode,+ runtime.slot(Model::PetFaceDetection).default_execution_mode,expected_pet_mode);assert_eq!(- runtime.pet_face_embedding_dog.default_execution_mode,+ runtime+ .slot(Model::PetFaceEmbeddingDog)+ .default_execution_mode,expected_pet_mode);assert_eq!(- runtime.pet_face_embedding_cat.default_execution_mode,+ runtime+ .slot(Model::PetFaceEmbeddingCat)+ .default_execution_mode,expected_pet_mode);assert_eq!(- runtime.pet_body_detection.default_execution_mode,+ runtime.slot(Model::PetBodyDetection).default_execution_mode,expected_pet_mode);assert_eq!(- runtime.pet_body_embedding_dog.default_execution_mode,+ runtime+ .slot(Model::PetBodyEmbeddingDog)+ .default_execution_mode,expected_pet_mode);assert_eq!(- runtime.pet_body_embedding_cat.default_execution_mode,+ runtime+ .slot(Model::PetBodyEmbeddingCat)+ .default_execution_mode,expected_pet_mode);}