Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Centralize ML model runtime metadata

laurenspriem19c9c694580b
diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rs
index 3e2171ecc5..3bfcdd1b5e 100644
--- a/rust/crates/photos/src/ml/assets.rs
+++ b/rust/crates/photos/src/ml/assets.rs
@@ -3,7 +3,9 @@ use std::path::{Path, PathBuf};
use ente_assets::{Asset, AssetFile, AssetStore};
-use super::runtime::ModelPaths;
+use super::model::{self, ModelPaths};
+
+pub use super::model::Model;
const MODELS: &str = "models";
@@ -22,33 +24,6 @@ const RETIRED_LEGACY_MODEL_FILES: &[&str] = &[
"mobilefacenet_opset15.onnx",
];
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum Model {
- FaceDetection,
- FaceEmbedding,
- ClipImage,
- ClipText,
- PetFaceDetection,
- PetFaceEmbeddingDog,
- PetFaceEmbeddingCat,
- PetBodyDetection,
- PetBodyEmbeddingDog,
- PetBodyEmbeddingCat,
-}
-
-const ALL_MODELS: [Model; 10] = [
- Model::FaceDetection,
- Model::FaceEmbedding,
- Model::ClipImage,
- Model::ClipText,
- Model::PetFaceDetection,
- Model::PetFaceEmbeddingDog,
- Model::PetFaceEmbeddingCat,
- Model::PetBodyDetection,
- Model::PetBodyEmbeddingDog,
- Model::PetBodyEmbeddingCat,
-];
-
pub struct ClipTextPaths {
pub model: PathBuf,
pub vocab: PathBuf,
@@ -71,24 +46,7 @@ pub fn model_asset(model: Model) -> Asset {
}
pub fn indexing_models(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Model> {
- let mut models = Vec::new();
- if run_faces {
- models.extend([Model::FaceDetection, Model::FaceEmbedding]);
- }
- if run_clip {
- models.push(Model::ClipImage);
- }
- if run_pets {
- models.extend([
- Model::PetFaceDetection,
- Model::PetFaceEmbeddingDog,
- Model::PetFaceEmbeddingCat,
- Model::PetBodyDetection,
- Model::PetBodyEmbeddingDog,
- Model::PetBodyEmbeddingCat,
- ]);
- }
- models
+ model::enabled_indexing_models(run_faces, run_clip, run_pets).collect()
}
pub fn indexing_model_paths(
@@ -97,18 +55,11 @@ pub fn indexing_model_paths(
run_clip: bool,
run_pets: bool,
) -> ModelPaths {
- ModelPaths {
- face_detection: model_path_if(store, Model::FaceDetection, run_faces),
- face_embedding: model_path_if(store, Model::FaceEmbedding, run_faces),
- clip_image: model_path_if(store, Model::ClipImage, run_clip),
- clip_text: String::new(),
- pet_face_detection: model_path_if(store, Model::PetFaceDetection, run_pets),
- pet_face_embedding_dog: model_path_if(store, Model::PetFaceEmbeddingDog, run_pets),
- pet_face_embedding_cat: model_path_if(store, Model::PetFaceEmbeddingCat, run_pets),
- pet_body_detection: model_path_if(store, Model::PetBodyDetection, run_pets),
- pet_body_embedding_dog: model_path_if(store, Model::PetBodyEmbeddingDog, run_pets),
- pet_body_embedding_cat: model_path_if(store, Model::PetBodyEmbeddingCat, run_pets),
+ let mut model_paths = ModelPaths::default();
+ for model in model::enabled_indexing_models(run_faces, run_clip, run_pets) {
+ *model_paths.get_mut(model) = model_path(store, model);
}
+ model_paths
}
pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {
@@ -127,7 +78,7 @@ pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {
pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {
let mut warnings = Vec::new();
if legacy_dir.exists() {
- for model in ALL_MODELS {
+ for model in Model::ALL {
if let Err(error) = migrate_desktop_model(store, legacy_dir, model) {
warnings.push(format!("Failed to migrate {}: {error}", model.spec().key));
}
@@ -255,10 +206,7 @@ struct ModelFileSpec {
sha256: &'static str,
}
-fn model_path_if(store: &AssetStore, model: Model, enabled: bool) -> String {
- if !enabled {
- return String::new();
- }
+fn model_path(store: &AssetStore, model: Model) -> String {
let asset = model_asset(model);
store
.file_path(&asset, model.spec().files[0].name)
@@ -332,7 +280,7 @@ mod tests {
fn catalog_keys_are_unique_and_prefix_free() {
let root = Path::new("assets");
let store = AssetStore::new(root);
- let paths = ALL_MODELS
+ let paths = Model::ALL
.map(|model| store.asset_dir(&model_asset(model)))
.to_vec();
for (index, path) in paths.iter().enumerate() {
@@ -348,7 +296,7 @@ mod tests {
#[test]
fn retired_keys_are_not_in_the_catalog() {
for key in RETIRED_MODEL_KEYS {
- assert!(ALL_MODELS.iter().all(|model| model.spec().key != *key));
+ assert!(Model::ALL.iter().all(|model| model.spec().key != *key));
}
}
@@ -361,7 +309,7 @@ mod tests {
for model in lock["models"].as_object().unwrap().values() {
let name = model["file_name"].as_str().unwrap();
let sha256 = model["sha256"].as_str().unwrap();
- let catalog_file = ALL_MODELS
+ let catalog_file = Model::ALL
.iter()
.flat_map(|model| model.spec().files)
.find(|file| file.name == name)
@@ -384,7 +332,7 @@ mod tests {
let asset = model_asset(Model::FaceEmbedding);
assert!(store.is_downloaded(&asset));
assert_eq!(
- fs::metadata(model_path_if(&store, Model::FaceEmbedding, true))
+ fs::metadata(model_path(&store, Model::FaceEmbedding))
.unwrap()
.len(),
Model::FaceEmbedding.spec().files[0].size
diff --git a/rust/crates/photos/src/ml/clip/image.rs b/rust/crates/photos/src/ml/clip/image.rs
index 8ca06f56db..c8590231e5 100644
--- a/rust/crates/photos/src/ml/clip/image.rs
+++ b/rust/crates/photos/src/ml/clip/image.rs
@@ -1,5 +1,6 @@
use crate::ml::{
error::MlResult,
+ model::Model,
onnx, preprocess,
runtime::MlRuntimeView,
types::{ClipResult, DecodedImage},
@@ -12,7 +13,7 @@ pub(crate) fn run_clip_image(
decoded: &DecodedImage,
) -> MlResult<ClipResult> {
let input = onnx::PreparedF32Input::new(preprocess::preprocess_clip(decoded)?);
- let (shape, output) = runtime.with_clip_image_session(|session| {
+ let (shape, output) = runtime.run(Model::ClipImage, |session| {
onnx::run_f32(
session,
&input,
diff --git a/rust/crates/photos/src/ml/clip/text.rs b/rust/crates/photos/src/ml/clip/text.rs
index 6e19d42aab..35df370759 100644
--- a/rust/crates/photos/src/ml/clip/text.rs
+++ b/rust/crates/photos/src/ml/clip/text.rs
@@ -1,6 +1,7 @@
use crate::ml::{
clip::tokenizer,
error::{MlError, MlResult},
+ model::Model,
onnx,
runtime::MlRuntimeView,
types::ClipResult,
@@ -16,7 +17,7 @@ fn run_clip_text(runtime: &MlRuntimeView<'_>, token_ids: &[i32]) -> MlResult<Cli
)));
}
- let (shape, output) = runtime.with_clip_text_session(|session| {
+ let (shape, output) = runtime.run(Model::ClipText, |session| {
onnx::run_i32_f32(session, token_ids, [1, CLIP_TEXT_TOKEN_COUNT as i64])
})?;
diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rs
index 71ab73c31a..4e76238e75 100644
--- a/rust/crates/photos/src/ml/face/detect.rs
+++ b/rust/crates/photos/src/ml/face/detect.rs
@@ -1,5 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
+ model::Model,
onnx,
postprocess::{NmsDetection, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
@@ -24,7 +25,7 @@ pub(crate) fn run_face_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<FaceDetection>> {
- runtime.with_face_detection_session(|session| {
+ runtime.run(Model::FaceDetection, |session| {
onnx::with_prepared_float_output(
session,
&input.tensor,
diff --git a/rust/crates/photos/src/ml/face/embed.rs b/rust/crates/photos/src/ml/face/embed.rs
index 4a045b82cb..1bbc8f6806 100644
--- a/rust/crates/photos/src/ml/face/embed.rs
+++ b/rust/crates/photos/src/ml/face/embed.rs
@@ -1,5 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
+ model::Model,
onnx,
postprocess::l2_normalize,
runtime::MlRuntimeView,
@@ -38,7 +39,7 @@ pub(crate) fn run_face_embedding(
}
let input = onnx::PreparedF32Input::new(aligned);
- let (shape, mut embedding) = runtime.with_face_embedding_session(|session| {
+ let (shape, mut embedding) = runtime.run(Model::FaceEmbedding, |session| {
onnx::run_f32(
session,
&input,
diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rs
index 4e9b2f786c..0acbf635bb 100644
--- a/rust/crates/photos/src/ml/indexing.rs
+++ b/rust/crates/photos/src/ml/indexing.rs
@@ -6,6 +6,7 @@ use crate::ml::{
run_face_alignment, run_face_detection, run_face_embedding,
thumbnail::{FaceBox, generate_face_thumbnails},
},
+ model::{self, Model},
pet::{
run_pet_body_detection, run_pet_body_embedding, run_pet_face_alignment,
run_pet_face_detection, run_pet_face_embedding,
@@ -240,9 +241,10 @@ pub fn run_clip_text(req: RunClipTextRequest) -> MlResult<RunClipTextResult> {
} = req;
if model_path.trim().is_empty() {
- return Err(MlError::InvalidRequest(
- "missing model path: clipTextModelPath".to_string(),
- ));
+ return Err(MlError::InvalidRequest(format!(
+ "missing model path: {}",
+ Model::ClipText.path_label()
+ )));
}
if vocab_path.trim().is_empty() {
return Err(MlError::InvalidRequest(
@@ -250,18 +252,8 @@ pub fn run_clip_text(req: RunClipTextRequest) -> MlResult<RunClipTextResult> {
));
}
- let model_paths = ModelPaths {
- face_detection: String::new(),
- face_embedding: String::new(),
- clip_image: String::new(),
- clip_text: model_path,
- pet_face_detection: String::new(),
- pet_face_embedding_dog: String::new(),
- pet_face_embedding_cat: String::new(),
- pet_body_detection: String::new(),
- pet_body_embedding_dog: String::new(),
- pet_body_embedding_cat: String::new(),
- };
+ let mut model_paths = ModelPaths::default();
+ *model_paths.get_mut(Model::ClipText) = model_path;
runtime::with_runtime(&model_paths, |runtime| {
let clip = run_clip_text_query(runtime, &text, &vocab_path)?;
@@ -291,40 +283,10 @@ pub fn tokenize_clip_text(text: &str, vocab_path: &str) -> MlResult<Vec<i32>> {
}
fn validate_request_model_paths(req: &AnalyzeImageRequest) -> MlResult<()> {
- let model_paths = &req.model_paths;
-
- let mut missing = Vec::new();
- if req.run_faces {
- if model_paths.face_detection.trim().is_empty() {
- missing.push("faceDetectionModelPath");
- }
- if model_paths.face_embedding.trim().is_empty() {
- missing.push("faceEmbeddingModelPath");
- }
- }
- if req.run_clip && model_paths.clip_image.trim().is_empty() {
- missing.push("clipImageModelPath");
- }
- if req.run_pets {
- if model_paths.pet_face_detection.trim().is_empty() {
- missing.push("petFaceDetectionModelPath");
- }
- if model_paths.pet_body_detection.trim().is_empty() {
- missing.push("petBodyDetectionModelPath");
- }
- if model_paths.pet_face_embedding_dog.trim().is_empty() {
- missing.push("petFaceEmbeddingDogModelPath");
- }
- if model_paths.pet_face_embedding_cat.trim().is_empty() {
- missing.push("petFaceEmbeddingCatModelPath");
- }
- if model_paths.pet_body_embedding_dog.trim().is_empty() {
- missing.push("petBodyEmbeddingDogModelPath");
- }
- if model_paths.pet_body_embedding_cat.trim().is_empty() {
- missing.push("petBodyEmbeddingCatModelPath");
- }
- }
+ let missing = model::required_indexing_models(req.run_faces, req.run_clip, req.run_pets)
+ .filter(|&model| req.model_paths.get(model).trim().is_empty())
+ .map(Model::path_label)
+ .collect::<Vec<_>>();
if missing.is_empty() {
return Ok(());
}
diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rs
index 4068c879cc..b3fd32e012 100644
--- a/rust/crates/photos/src/ml/mod.rs
+++ b/rust/crates/photos/src/ml/mod.rs
@@ -7,6 +7,7 @@ pub mod golden;
mod golden_data;
pub mod golden_tooling;
pub mod indexing;
+mod model;
mod onnx;
mod pet;
mod postprocess;
diff --git a/rust/crates/photos/src/ml/model.rs b/rust/crates/photos/src/ml/model.rs
new file mode 100644
index 0000000000..b8770a8412
--- /dev/null
+++ b/rust/crates/photos/src/ml/model.rs
@@ -0,0 +1,271 @@
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[repr(usize)]
+pub enum Model {
+ FaceDetection,
+ FaceEmbedding,
+ ClipImage,
+ ClipText,
+ PetFaceDetection,
+ PetFaceEmbeddingDog,
+ PetFaceEmbeddingCat,
+ PetBodyDetection,
+ PetBodyEmbeddingDog,
+ PetBodyEmbeddingCat,
+}
+
+impl Model {
+ pub(crate) const COUNT: usize = Self::PetBodyEmbeddingCat as usize + 1;
+
+ pub(crate) const ALL: [Self; Self::COUNT] = [
+ Self::FaceDetection,
+ Self::FaceEmbedding,
+ Self::ClipImage,
+ Self::ClipText,
+ Self::PetFaceDetection,
+ Self::PetFaceEmbeddingDog,
+ Self::PetFaceEmbeddingCat,
+ Self::PetBodyDetection,
+ Self::PetBodyEmbeddingDog,
+ Self::PetBodyEmbeddingCat,
+ ];
+
+ pub(crate) const INDEXING: [Self; 9] = [
+ Self::FaceDetection,
+ Self::FaceEmbedding,
+ Self::ClipImage,
+ Self::PetFaceDetection,
+ Self::PetFaceEmbeddingDog,
+ Self::PetFaceEmbeddingCat,
+ Self::PetBodyDetection,
+ Self::PetBodyEmbeddingDog,
+ Self::PetBodyEmbeddingCat,
+ ];
+
+ const INDEXING_PATH_VALIDATION: [Self; 9] = [
+ Self::FaceDetection,
+ Self::FaceEmbedding,
+ Self::ClipImage,
+ Self::PetFaceDetection,
+ Self::PetBodyDetection,
+ Self::PetFaceEmbeddingDog,
+ Self::PetFaceEmbeddingCat,
+ Self::PetBodyEmbeddingDog,
+ Self::PetBodyEmbeddingCat,
+ ];
+
+ pub(crate) const fn index(self) -> usize {
+ self as usize
+ }
+
+ pub(crate) const fn path_label(self) -> &'static str {
+ match self {
+ Self::FaceDetection => "faceDetectionModelPath",
+ Self::FaceEmbedding => "faceEmbeddingModelPath",
+ Self::ClipImage => "clipImageModelPath",
+ Self::ClipText => "clipTextModelPath",
+ Self::PetFaceDetection => "petFaceDetectionModelPath",
+ Self::PetFaceEmbeddingDog => "petFaceEmbeddingDogModelPath",
+ Self::PetFaceEmbeddingCat => "petFaceEmbeddingCatModelPath",
+ Self::PetBodyDetection => "petBodyDetectionModelPath",
+ Self::PetBodyEmbeddingDog => "petBodyEmbeddingDogModelPath",
+ Self::PetBodyEmbeddingCat => "petBodyEmbeddingCatModelPath",
+ }
+ }
+
+ pub(crate) const fn namespace(self) -> &'static str {
+ match self {
+ Self::FaceDetection => "face-detection",
+ Self::FaceEmbedding => "face-embedding",
+ Self::ClipImage => "clip-image",
+ Self::ClipText => "clip-text",
+ Self::PetFaceDetection => "pet-face-detection",
+ Self::PetFaceEmbeddingDog => "pet-face-embedding-dog",
+ Self::PetFaceEmbeddingCat => "pet-face-embedding-cat",
+ Self::PetBodyDetection => "pet-body-detection",
+ Self::PetBodyEmbeddingDog => "pet-body-embedding-dog",
+ Self::PetBodyEmbeddingCat => "pet-body-embedding-cat",
+ }
+ }
+
+ pub(crate) fn missing_path_error(self) -> String {
+ let requirement = match self {
+ Self::FaceDetection | Self::FaceEmbedding => " is required when runFaces is true",
+ Self::ClipImage => " is required when runClip is true",
+ Self::ClipText => " is required when running clip text",
+ Self::PetFaceDetection | Self::PetBodyDetection => " is required when runPets is true",
+ Self::PetFaceEmbeddingDog
+ | Self::PetFaceEmbeddingCat
+ | Self::PetBodyEmbeddingDog
+ | Self::PetBodyEmbeddingCat => " is required",
+ };
+ format!("missing model path: {}{requirement}", self.path_label())
+ }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ModelPaths {
+ pub face_detection: String,
+ pub face_embedding: String,
+ pub clip_image: String,
+ pub clip_text: String,
+ pub pet_face_detection: String,
+ pub pet_face_embedding_dog: String,
+ pub pet_face_embedding_cat: String,
+ pub pet_body_detection: String,
+ pub pet_body_embedding_dog: String,
+ pub pet_body_embedding_cat: String,
+}
+
+impl ModelPaths {
+ pub(crate) fn get(&self, model: Model) -> &str {
+ match model {
+ Model::FaceDetection => &self.face_detection,
+ Model::FaceEmbedding => &self.face_embedding,
+ Model::ClipImage => &self.clip_image,
+ Model::ClipText => &self.clip_text,
+ Model::PetFaceDetection => &self.pet_face_detection,
+ Model::PetFaceEmbeddingDog => &self.pet_face_embedding_dog,
+ Model::PetFaceEmbeddingCat => &self.pet_face_embedding_cat,
+ Model::PetBodyDetection => &self.pet_body_detection,
+ Model::PetBodyEmbeddingDog => &self.pet_body_embedding_dog,
+ Model::PetBodyEmbeddingCat => &self.pet_body_embedding_cat,
+ }
+ }
+
+ pub(crate) fn get_mut(&mut self, model: Model) -> &mut String {
+ match model {
+ Model::FaceDetection => &mut self.face_detection,
+ Model::FaceEmbedding => &mut self.face_embedding,
+ Model::ClipImage => &mut self.clip_image,
+ Model::ClipText => &mut self.clip_text,
+ Model::PetFaceDetection => &mut self.pet_face_detection,
+ Model::PetFaceEmbeddingDog => &mut self.pet_face_embedding_dog,
+ Model::PetFaceEmbeddingCat => &mut self.pet_face_embedding_cat,
+ Model::PetBodyDetection => &mut self.pet_body_detection,
+ Model::PetBodyEmbeddingDog => &mut self.pet_body_embedding_dog,
+ Model::PetBodyEmbeddingCat => &mut self.pet_body_embedding_cat,
+ }
+ }
+}
+
+pub(crate) fn enabled_indexing_models(
+ run_faces: bool,
+ run_clip: bool,
+ run_pets: bool,
+) -> impl Iterator<Item = Model> {
+ Model::INDEXING
+ .into_iter()
+ .filter(move |&model| indexing_model_is_enabled(model, run_faces, run_clip, run_pets))
+}
+
+pub(crate) fn required_indexing_models(
+ run_faces: bool,
+ run_clip: bool,
+ run_pets: bool,
+) -> impl Iterator<Item = Model> {
+ Model::INDEXING_PATH_VALIDATION
+ .into_iter()
+ .filter(move |&model| indexing_model_is_enabled(model, run_faces, run_clip, run_pets))
+}
+
+fn indexing_model_is_enabled(
+ model: Model,
+ run_faces: bool,
+ run_clip: bool,
+ run_pets: bool,
+) -> bool {
+ match model {
+ Model::FaceDetection | Model::FaceEmbedding => run_faces,
+ Model::ClipImage => run_clip,
+ Model::PetFaceDetection
+ | Model::PetFaceEmbeddingDog
+ | Model::PetFaceEmbeddingCat
+ | Model::PetBodyDetection
+ | Model::PetBodyEmbeddingDog
+ | Model::PetBodyEmbeddingCat => run_pets,
+ Model::ClipText => false,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashSet;
+
+ use super::*;
+
+ #[test]
+ fn model_catalog_is_array_indexed_and_unique() {
+ let mut path_labels = HashSet::new();
+ let mut namespaces = HashSet::new();
+
+ for (index, model) in Model::ALL.into_iter().enumerate() {
+ assert_eq!(model.index(), index);
+ assert!(path_labels.insert(model.path_label()));
+ assert!(namespaces.insert(model.namespace()));
+ }
+ }
+
+ #[test]
+ fn model_paths_are_addressable_by_model() {
+ let mut paths = ModelPaths::default();
+
+ for model in Model::ALL {
+ *paths.get_mut(model) = model.namespace().to_string();
+ }
+
+ for model in Model::ALL {
+ assert_eq!(paths.get(model), model.namespace());
+ }
+ }
+
+ #[test]
+ fn indexing_catalog_contains_every_model_except_clip_text() {
+ let indexes = Model::INDEXING
+ .into_iter()
+ .map(Model::index)
+ .collect::<HashSet<_>>();
+
+ assert_eq!(indexes.len(), Model::COUNT - 1);
+ for model in Model::ALL {
+ assert_eq!(indexes.contains(&model.index()), model != Model::ClipText);
+ }
+ assert_eq!(
+ Model::INDEXING_PATH_VALIDATION
+ .into_iter()
+ .map(Model::index)
+ .collect::<HashSet<_>>(),
+ indexes
+ );
+
+ assert_eq!(
+ enabled_indexing_models(true, false, false).collect::<Vec<_>>(),
+ [Model::FaceDetection, Model::FaceEmbedding]
+ );
+ assert_eq!(
+ enabled_indexing_models(false, true, false).collect::<Vec<_>>(),
+ [Model::ClipImage]
+ );
+ assert_eq!(
+ enabled_indexing_models(false, false, true).collect::<Vec<_>>(),
+ [
+ Model::PetFaceDetection,
+ Model::PetFaceEmbeddingDog,
+ Model::PetFaceEmbeddingCat,
+ Model::PetBodyDetection,
+ Model::PetBodyEmbeddingDog,
+ Model::PetBodyEmbeddingCat,
+ ]
+ );
+ assert_eq!(
+ required_indexing_models(false, false, true).collect::<Vec<_>>(),
+ [
+ Model::PetFaceDetection,
+ Model::PetBodyDetection,
+ Model::PetFaceEmbeddingDog,
+ Model::PetFaceEmbeddingCat,
+ Model::PetBodyEmbeddingDog,
+ Model::PetBodyEmbeddingCat,
+ ]
+ );
+ }
+}
diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rs
index 52814e5548..152db8a975 100644
--- a/rust/crates/photos/src/ml/pet/detect.rs
+++ b/rust/crates/photos/src/ml/pet/detect.rs
@@ -1,5 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
+ model::Model,
onnx,
postprocess::{NmsDetection, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
@@ -48,7 +49,7 @@ pub(crate) fn run_pet_face_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetFaceDetection>> {
- runtime.with_pet_face_detection_session(|session| {
+ runtime.run(Model::PetFaceDetection, |session| {
onnx::with_prepared_float_output(
session,
&input.tensor,
@@ -182,7 +183,7 @@ pub(crate) fn run_pet_body_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetBodyDetection>> {
- runtime.with_pet_body_detection_session(|session| {
+ runtime.run(Model::PetBodyDetection, |session| {
onnx::with_prepared_float_output(
session,
&input.tensor,
diff --git a/rust/crates/photos/src/ml/pet/embed.rs b/rust/crates/photos/src/ml/pet/embed.rs
index 745e377159..e56b105603 100644
--- a/rust/crates/photos/src/ml/pet/embed.rs
+++ b/rust/crates/photos/src/ml/pet/embed.rs
@@ -1,5 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
+ model::Model,
onnx,
postprocess::l2_normalize,
runtime::MlRuntimeView,
@@ -52,15 +53,13 @@ pub(crate) fn run_pet_face_embedding(
PET_EMBEDDING_INPUT_SIZE as i64,
PET_EMBEDDING_INPUT_SIZE as i64,
];
- let (shape, output) = if species == PET_SPECIES_DOG {
- runtime.with_pet_face_embedding_dog_session(|session| {
- onnx::run_f32(session, &input, input_shape)
- })?
+ let model = if species == PET_SPECIES_DOG {
+ Model::PetFaceEmbeddingDog
} else {
- runtime.with_pet_face_embedding_cat_session(|session| {
- onnx::run_f32(session, &input, input_shape)
- })?
+ Model::PetFaceEmbeddingCat
};
+ let (shape, output) =
+ runtime.run(model, |session| onnx::run_f32(session, &input, input_shape))?;
let embedding_size =
validate_embedding_batch_output("face", &shape, output.len(), batch.indices.len())?;
@@ -134,15 +133,13 @@ pub(crate) fn run_pet_body_embedding(
PET_EMBEDDING_INPUT_SIZE as i64,
PET_EMBEDDING_INPUT_SIZE as i64,
];
- let (shape, output) = if is_cat {
- runtime.with_pet_body_embedding_cat_session(|session| {
- onnx::run_f32(session, &input, input_shape)
- })?
+ let model = if is_cat {
+ Model::PetBodyEmbeddingCat
} else {
- runtime.with_pet_body_embedding_dog_session(|session| {
- onnx::run_f32(session, &input, input_shape)
- })?
+ Model::PetBodyEmbeddingDog
};
+ let (shape, output) =
+ runtime.run(model, |session| onnx::run_f32(session, &input, input_shape))?;
let embedding_size =
validate_embedding_batch_output("body", &shape, output.len(), batch.indices.len())?;
diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rs
index 6ac9ecfdf9..065a78609e 100644
--- a/rust/crates/photos/src/ml/runtime.rs
+++ b/rust/crates/photos/src/ml/runtime.rs
@@ -9,22 +9,11 @@ use ort::session::Session;
use crate::ml::{
error::{MlError, MlResult},
+ model::Model,
onnx,
};
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct ModelPaths {
- pub face_detection: String,
- pub face_embedding: String,
- pub clip_image: String,
- pub clip_text: String,
- pub pet_face_detection: String,
- pub pet_face_embedding_dog: String,
- pub pet_face_embedding_cat: String,
- pub pet_body_detection: String,
- pub pet_body_embedding_dog: String,
- pub pet_body_embedding_cat: String,
-}
+pub use crate::ml::model::ModelPaths;
#[derive(Debug)]
struct ModelSlotState {
@@ -35,8 +24,8 @@ struct ModelSlotState {
#[derive(Debug)]
struct ModelSlot {
+ model: Model,
default_execution_mode: onnx::ExecutionMode,
- model_namespace: &'static str,
state: Mutex<ModelSlotState>,
}
@@ -63,10 +52,10 @@ pub(crate) struct MlRuntimeView<'a> {
}
impl ModelSlot {
- fn new(default_execution_mode: onnx::ExecutionMode, model_namespace: &'static str) -> Self {
+ fn new(model: Model) -> Self {
Self {
- default_execution_mode,
- model_namespace,
+ model,
+ default_execution_mode: default_execution_mode(model),
state: Mutex::new(ModelSlotState {
path: String::new(),
provider_plan: None,
@@ -108,17 +97,16 @@ impl ModelSlot {
fn run<T>(
&self,
path: &str,
- error_msg: &str,
mut operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
) -> MlResult<(T, onnx::ExecutionProvider)> {
if path.trim().is_empty() {
- return Err(MlError::InvalidRequest(error_msg.to_string()));
+ return Err(MlError::InvalidRequest(self.model.missing_path_error()));
}
loop {
let mut state = self.lock_state();
Self::set_config_locked(&mut state, path);
- self.ensure_loaded_locked(&mut state, error_msg)?;
+ self.ensure_loaded_locked(&mut state)?;
let execution_provider = state
.provider_plan
@@ -182,9 +170,9 @@ impl ModelSlot {
Self::clear_transient_runtime_state_locked(state);
}
- fn ensure_loaded_locked(&self, state: &mut ModelSlotState, error_msg: &str) -> MlResult<()> {
+ fn ensure_loaded_locked(&self, state: &mut ModelSlotState) -> MlResult<()> {
if state.path.trim().is_empty() {
- return Err(MlError::InvalidRequest(error_msg.to_string()));
+ return Err(MlError::InvalidRequest(self.model.missing_path_error()));
}
if state.session.is_some() {
return Ok(());
@@ -202,7 +190,7 @@ impl ModelSlot {
self.default_execution_mode
);
let t = std::time::Instant::now();
- let session = onnx::build_session(&state.path, provider_plan, self.model_namespace)?;
+ let session = onnx::build_session(&state.path, provider_plan, self.model.namespace())?;
let execution_provider = provider_plan
.selected_provider()
.expect("successful session build must select an execution provider");
@@ -217,97 +205,40 @@ impl ModelSlot {
#[derive(Debug)]
struct MlRuntime {
- face_detection: ModelSlot,
- face_embedding: ModelSlot,
- clip_image: ModelSlot,
- clip_text: ModelSlot,
- pet_face_detection: ModelSlot,
- pet_face_embedding_dog: ModelSlot,
- pet_face_embedding_cat: ModelSlot,
- pet_body_detection: ModelSlot,
- pet_body_embedding_dog: ModelSlot,
- pet_body_embedding_cat: ModelSlot,
+ slots: [ModelSlot; Model::COUNT],
}
static GLOBAL_RUNTIME: Lazy<MlRuntime> = Lazy::new(MlRuntime::new);
impl MlRuntime {
fn new() -> Self {
- let platform_default = onnx::ExecutionMode::PlatformDefault;
- let cpu_only = onnx::ExecutionMode::CpuOnly;
-
Self {
- face_detection: ModelSlot::new(platform_default, "face-detection"),
- face_embedding: ModelSlot::new(platform_default, "face-embedding"),
- clip_image: ModelSlot::new(platform_default, "clip-image"),
- // The quantized CLIP text graph is heavily partitioned by both
- // CoreML and WebGPU, making their mixed CPU/GPU execution slower
- // than running the complete model on CPU.
- clip_text: ModelSlot::new(cpu_only, "clip-text"),
- // Pet models stay CPU-only due to device-specific FP16 failures.
- pet_face_detection: ModelSlot::new(cpu_only, "pet-face-detection"),
- pet_face_embedding_dog: ModelSlot::new(cpu_only, "pet-face-embedding-dog"),
- pet_face_embedding_cat: ModelSlot::new(cpu_only, "pet-face-embedding-cat"),
- pet_body_detection: ModelSlot::new(cpu_only, "pet-body-detection"),
- pet_body_embedding_dog: ModelSlot::new(cpu_only, "pet-body-embedding-dog"),
- pet_body_embedding_cat: ModelSlot::new(cpu_only, "pet-body-embedding-cat"),
+ slots: Model::ALL.map(ModelSlot::new),
}
}
+ fn slot(&self, model: Model) -> &ModelSlot {
+ &self.slots[model.index()]
+ }
+
fn configure_requested_models(&self, model_paths: &ModelPaths) {
- self.face_detection
- .configure_if_requested(&model_paths.face_detection);
- self.face_embedding
- .configure_if_requested(&model_paths.face_embedding);
- self.clip_image
- .configure_if_requested(&model_paths.clip_image);
- self.clip_text
- .configure_if_requested(&model_paths.clip_text);
- self.pet_face_detection
- .configure_if_requested(&model_paths.pet_face_detection);
- self.pet_face_embedding_dog
- .configure_if_requested(&model_paths.pet_face_embedding_dog);
- self.pet_face_embedding_cat
- .configure_if_requested(&model_paths.pet_face_embedding_cat);
- self.pet_body_detection
- .configure_if_requested(&model_paths.pet_body_detection);
- self.pet_body_embedding_dog
- .configure_if_requested(&model_paths.pet_body_embedding_dog);
- self.pet_body_embedding_cat
- .configure_if_requested(&model_paths.pet_body_embedding_cat);
+ for model in Model::ALL {
+ self.slot(model)
+ .configure_if_requested(model_paths.get(model));
+ }
}
fn prepare_indexing_models(&self, model_paths: &ModelPaths) {
- self.face_detection
- .sync_indexing_residency(&model_paths.face_detection);
- self.face_embedding
- .sync_indexing_residency(&model_paths.face_embedding);
- self.clip_image
- .sync_indexing_residency(&model_paths.clip_image);
- self.pet_face_detection
- .sync_indexing_residency(&model_paths.pet_face_detection);
- self.pet_face_embedding_dog
- .sync_indexing_residency(&model_paths.pet_face_embedding_dog);
- self.pet_face_embedding_cat
- .sync_indexing_residency(&model_paths.pet_face_embedding_cat);
- self.pet_body_detection
- .sync_indexing_residency(&model_paths.pet_body_detection);
- self.pet_body_embedding_dog
- .sync_indexing_residency(&model_paths.pet_body_embedding_dog);
- self.pet_body_embedding_cat
- .sync_indexing_residency(&model_paths.pet_body_embedding_cat);
+ for model in Model::INDEXING {
+ self.slot(model)
+ .sync_indexing_residency(model_paths.get(model));
+ }
}
fn release_indexing_models(&self) {
- self.face_detection.release_residency();
- self.face_embedding.release_residency();
- self.clip_image.release_residency();
- self.pet_face_detection.release_residency();
- self.pet_face_embedding_dog.release_residency();
- self.pet_face_embedding_cat.release_residency();
- self.pet_body_detection.release_residency();
- self.pet_body_embedding_dog.release_residency();
- self.pet_body_embedding_cat.release_residency();
+ for model in Model::INDEXING {
+ self.slot(model).release_residency();
+ }
}
fn view<'a>(&'a self, model_paths: &'a ModelPaths) -> MlRuntimeView<'a> {
@@ -324,138 +255,36 @@ impl<'a> MlRuntimeView<'a> {
self.used_providers.get()
}
- fn run_tracked<T>(
+ pub(crate) fn run<T>(
&self,
- slot: &ModelSlot,
- path: &str,
- error_msg: &str,
+ model: Model,
operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
) -> MlResult<T> {
- let (value, execution_provider) = slot.run(path, error_msg, operation)?;
+ let (value, execution_provider) = self
+ .runtime
+ .slot(model)
+ .run(self.model_paths.get(model), operation)?;
let mut used = self.used_providers.get();
used.record(execution_provider);
self.used_providers.set(used);
Ok(value)
}
+}
- pub(crate) fn with_face_detection_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.face_detection,
- &self.model_paths.face_detection,
- "missing model path: faceDetectionModelPath is required when runFaces is true",
- operation,
- )
- }
-
- pub(crate) fn with_face_embedding_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.face_embedding,
- &self.model_paths.face_embedding,
- "missing model path: faceEmbeddingModelPath is required when runFaces is true",
- operation,
- )
- }
-
- pub(crate) fn with_clip_image_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.clip_image,
- &self.model_paths.clip_image,
- "missing model path: clipImageModelPath is required when runClip is true",
- operation,
- )
- }
-
- pub(crate) fn with_clip_text_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.clip_text,
- &self.model_paths.clip_text,
- "missing model path: clipTextModelPath is required when running clip text",
- operation,
- )
- }
-
- pub(crate) fn with_pet_face_detection_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.pet_face_detection,
- &self.model_paths.pet_face_detection,
- "missing model path: petFaceDetectionModelPath is required when runPets is true",
- operation,
- )
- }
-
- pub(crate) fn with_pet_face_embedding_dog_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.pet_face_embedding_dog,
- &self.model_paths.pet_face_embedding_dog,
- "missing model path: petFaceEmbeddingDogModelPath is required",
- operation,
- )
- }
-
- pub(crate) fn with_pet_face_embedding_cat_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.pet_face_embedding_cat,
- &self.model_paths.pet_face_embedding_cat,
- "missing model path: petFaceEmbeddingCatModelPath is required",
- operation,
- )
- }
-
- pub(crate) fn with_pet_body_detection_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.pet_body_detection,
- &self.model_paths.pet_body_detection,
- "missing model path: petBodyDetectionModelPath is required when runPets is true",
- operation,
- )
- }
-
- pub(crate) fn with_pet_body_embedding_dog_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.pet_body_embedding_dog,
- &self.model_paths.pet_body_embedding_dog,
- "missing model path: petBodyEmbeddingDogModelPath is required",
- operation,
- )
- }
-
- pub(crate) fn with_pet_body_embedding_cat_session<T>(
- &self,
- operation: impl FnMut(&mut Session) -> onnx::SessionRunResult<T>,
- ) -> MlResult<T> {
- self.run_tracked(
- &self.runtime.pet_body_embedding_cat,
- &self.model_paths.pet_body_embedding_cat,
- "missing model path: petBodyEmbeddingCatModelPath is required",
- operation,
- )
+fn default_execution_mode(model: Model) -> onnx::ExecutionMode {
+ match model {
+ Model::FaceDetection | Model::FaceEmbedding | Model::ClipImage => {
+ onnx::ExecutionMode::PlatformDefault
+ }
+ // Accelerators heavily partition the quantized CLIP text graph.
+ Model::ClipText => onnx::ExecutionMode::CpuOnly,
+ // Pet models have device-specific FP16 failures.
+ Model::PetFaceDetection
+ | Model::PetFaceEmbeddingDog
+ | Model::PetFaceEmbeddingCat
+ | Model::PetBodyDetection
+ | Model::PetBodyEmbeddingDog
+ | Model::PetBodyEmbeddingCat => onnx::ExecutionMode::CpuOnly,
}
}
@@ -499,7 +328,7 @@ mod tests {
..ModelPaths::default()
});
- let clip_text = runtime.clip_text.lock_state();
+ let clip_text = runtime.slot(Model::ClipText).lock_state();
assert_eq!(clip_text.path, "clip_text.onnx");
}
@@ -517,7 +346,7 @@ mod tests {
let runtime = MlRuntime::new();
{
- let mut clip_text = runtime.clip_text.lock_state();
+ let mut clip_text = runtime.slot(Model::ClipText).lock_state();
clip_text.path = "clip_text.onnx".to_string();
clip_text.provider_plan = Some(onnx::ProviderPlan::new(
onnx::ExecutionMode::CpuOnly,
@@ -525,7 +354,7 @@ mod tests {
));
}
{
- let mut face_detection = runtime.face_detection.lock_state();
+ let mut face_detection = runtime.slot(Model::FaceDetection).lock_state();
face_detection.path = "face.onnx".to_string();
face_detection.provider_plan = Some(onnx::ProviderPlan::new(
onnx::ExecutionMode::CpuOnly,
@@ -535,11 +364,11 @@ mod tests {
runtime.release_indexing_models();
- let clip_text = runtime.clip_text.lock_state();
+ let clip_text = runtime.slot(Model::ClipText).lock_state();
assert_eq!(clip_text.path, "clip_text.onnx");
assert!(clip_text.provider_plan.is_some());
- let face_detection = runtime.face_detection.lock_state();
+ let face_detection = runtime.slot(Model::FaceDetection).lock_state();
assert!(face_detection.provider_plan.is_none());
}
@@ -554,22 +383,22 @@ mod tests {
..ModelPaths::default()
});
- let face_detection = runtime.face_detection.lock_state();
+ let face_detection = runtime.slot(Model::FaceDetection).lock_state();
assert_eq!(face_detection.path, "face.onnx");
assert!(face_detection.session.is_none());
- let face_embedding = runtime.face_embedding.lock_state();
+ let face_embedding = runtime.slot(Model::FaceEmbedding).lock_state();
assert_eq!(face_embedding.path, "embed.onnx");
assert!(face_embedding.session.is_none());
- let clip_image = runtime.clip_image.lock_state();
+ let clip_image = runtime.slot(Model::ClipImage).lock_state();
assert_eq!(clip_image.path, "clip.onnx");
assert!(clip_image.session.is_none());
}
#[test]
fn sync_indexing_residency_clears_disabled_slots() {
- let slot = ModelSlot::new(onnx::ExecutionMode::PlatformDefault, "test-model");
+ let slot = ModelSlot::new(Model::ClipImage);
{
let mut state = slot.lock_state();
@@ -590,7 +419,7 @@ mod tests {
#[test]
fn release_residency_resets_transient_cpu_fallback_for_any_slot() {
- let slot = ModelSlot::new(onnx::ExecutionMode::PlatformDefault, "test-model");
+ let slot = ModelSlot::new(Model::ClipImage);
{
let mut state = slot.lock_state();
@@ -613,44 +442,56 @@ mod tests {
let runtime = MlRuntime::new();
let expected_pet_mode = onnx::ExecutionMode::CpuOnly;
+ for model in Model::ALL {
+ assert_eq!(runtime.slot(model).model, model);
+ }
+
assert_eq!(
- runtime.face_detection.default_execution_mode,
+ runtime.slot(Model::FaceDetection).default_execution_mode,
onnx::ExecutionMode::PlatformDefault
);
assert_eq!(
- runtime.face_embedding.default_execution_mode,
+ runtime.slot(Model::FaceEmbedding).default_execution_mode,
onnx::ExecutionMode::PlatformDefault
);
assert_eq!(
- runtime.clip_image.default_execution_mode,
+ runtime.slot(Model::ClipImage).default_execution_mode,
onnx::ExecutionMode::PlatformDefault
);
assert_eq!(
- runtime.clip_text.default_execution_mode,
+ runtime.slot(Model::ClipText).default_execution_mode,
onnx::ExecutionMode::CpuOnly
);
assert_eq!(
- runtime.pet_face_detection.default_execution_mode,
+ runtime.slot(Model::PetFaceDetection).default_execution_mode,
expected_pet_mode
);
assert_eq!(
- runtime.pet_face_embedding_dog.default_execution_mode,
+ runtime
+ .slot(Model::PetFaceEmbeddingDog)
+ .default_execution_mode,
expected_pet_mode
);
assert_eq!(
- runtime.pet_face_embedding_cat.default_execution_mode,
+ runtime
+ .slot(Model::PetFaceEmbeddingCat)
+ .default_execution_mode,
expected_pet_mode
);
assert_eq!(
- runtime.pet_body_detection.default_execution_mode,
+ runtime.slot(Model::PetBodyDetection).default_execution_mode,
expected_pet_mode
);
assert_eq!(
- runtime.pet_body_embedding_dog.default_execution_mode,
+ runtime
+ .slot(Model::PetBodyEmbeddingDog)
+ .default_execution_mode,
expected_pet_mode
);
assert_eq!(
- runtime.pet_body_embedding_cat.default_execution_mode,
+ runtime
+ .slot(Model::PetBodyEmbeddingCat)
+ .default_execution_mode,
expected_pet_mode
);
}