Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Clarify ML model and asset ownership

laurenspriem94fd463c5014
diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
index 90bc725948..5245693cac 100644
--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
@@ -1,4 +1,4 @@
-use ente_photos::ml::{error::MlError, indexing, runtime::ModelPaths, types};
+use ente_photos::ml::{ModelPaths, error::MlError, indexing, types};
#[derive(Clone, Debug)]
pub struct RustModelPaths {
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index 5903e7a58c..d723c92c85 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -1,5 +1,5 @@
use ente_photos::ml::{
- assets, error::MlError, indexing as shared_indexing, runtime::ModelPaths as SharedModelPaths,
+ ModelPaths as SharedModelPaths, assets, error::MlError, indexing as shared_indexing,
types as shared_types,
};
use napi::bindgen_prelude::*;
@@ -80,12 +80,7 @@ impl AssetStore {
run_clip: bool,
run_pets: bool,
) -> Result<ModelPaths> {
- let models = assets::indexing_models(run_faces, run_clip, run_pets);
- let model_assets = models
- .iter()
- .copied()
- .map(assets::model_asset)
- .collect::<Vec<_>>();
+ let model_assets = assets::indexing_assets(run_faces, run_clip, run_pets);
self.inner
.download(
&model_assets,
@@ -104,7 +99,7 @@ impl AssetStore {
#[napi]
pub async fn clip_text_model_paths(&self) -> Result<ClipTextModelPaths> {
- let asset = assets::model_asset(assets::Model::ClipText);
+ let asset = assets::clip_text_asset();
self.inner
.download(
std::slice::from_ref(&asset),
diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rs
index 3bfcdd1b5e..67c65bd89c 100644
--- a/rust/crates/photos/src/ml/assets.rs
+++ b/rust/crates/photos/src/ml/assets.rs
@@ -3,9 +3,7 @@ use std::path::{Path, PathBuf};
use ente_assets::{Asset, AssetFile, AssetStore};
-use super::model::{self, ModelPaths};
-
-pub use super::model::Model;
+use super::models::{self, Model, ModelPaths};
const MODELS: &str = "models";
@@ -29,8 +27,8 @@ pub struct ClipTextPaths {
pub vocab: PathBuf,
}
-pub fn model_asset(model: Model) -> Asset {
- let spec = model.spec();
+fn model_asset(model: Model) -> Asset {
+ let spec = model_asset_spec(model);
Asset::files(
model_key(spec.key),
spec.files
@@ -45,8 +43,10 @@ pub fn model_asset(model: Model) -> Asset {
.expect("valid Photos model catalog")
}
-pub fn indexing_models(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Model> {
- model::enabled_indexing_models(run_faces, run_clip, run_pets).collect()
+pub fn indexing_assets(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Asset> {
+ models::enabled_indexing_models(run_faces, run_clip, run_pets)
+ .map(model_asset)
+ .collect()
}
pub fn indexing_model_paths(
@@ -56,15 +56,15 @@ pub fn indexing_model_paths(
run_pets: bool,
) -> ModelPaths {
let mut model_paths = ModelPaths::default();
- for model in model::enabled_indexing_models(run_faces, run_clip, run_pets) {
+ for model in models::enabled_indexing_models(run_faces, run_clip, run_pets) {
*model_paths.get_mut(model) = model_path(store, model);
}
model_paths
}
pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {
- let asset = model_asset(Model::ClipText);
- let files = Model::ClipText.spec().files;
+ let asset = clip_text_asset();
+ let files = model_asset_spec(Model::ClipText).files;
ClipTextPaths {
model: store
.file_path(&asset, files[0].name)
@@ -75,12 +75,19 @@ pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {
}
}
+pub fn clip_text_asset() -> Asset {
+ model_asset(Model::ClipText)
+}
+
pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {
let mut warnings = Vec::new();
if legacy_dir.exists() {
for model in Model::ALL {
if let Err(error) = migrate_desktop_model(store, legacy_dir, model) {
- warnings.push(format!("Failed to migrate {}: {error}", model.spec().key));
+ warnings.push(format!(
+ "Failed to migrate {}: {error}",
+ model_asset_spec(model).key
+ ));
}
}
for name in RETIRED_LEGACY_MODEL_FILES {
@@ -101,106 +108,104 @@ pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<Stri
warnings
}
-impl Model {
- fn spec(self) -> ModelSpec {
- match self {
- Self::FaceDetection => ModelSpec {
- key: "yolov5s_face_640_640_static_b1",
- files: &[ModelFileSpec {
- name: "yolov5s_face_640_640_static_b1.onnx",
- size: 32_355_091,
- sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",
- }],
- },
- Self::FaceEmbedding => ModelSpec {
- key: "mobilefacenet_portable_static_b1",
- files: &[ModelFileSpec {
- name: "mobilefacenet_portable_static_b1.onnx",
- size: 5_278_803,
- sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",
- }],
- },
- Self::ClipImage => ModelSpec {
- key: "mobileclip_s2_image_gelu_opset20",
- files: &[ModelFileSpec {
- name: "mobileclip_s2_image_gelu_opset20.onnx",
- size: 143_057_352,
- sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",
- }],
- },
- Self::ClipText => ModelSpec {
- key: "mobileclip_s2_text_opset18_quant",
- files: &[
- ModelFileSpec {
- name: "mobileclip_s2_text_opset18_quant.onnx",
- size: 67_144_712,
- sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",
- },
- ModelFileSpec {
- name: "bpe_simple_vocab_16e6.txt",
- size: 3_194_984,
- sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",
- },
- ],
- },
- Self::PetFaceDetection => ModelSpec {
- key: "yolov5s_pet_face_fp16_V2",
- files: &[ModelFileSpec {
- name: "yolov5s_pet_face_fp16_V2.onnx",
- size: 14_758_020,
- sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",
- }],
- },
- Self::PetFaceEmbeddingDog => ModelSpec {
- key: "dog_face_embedding128",
- files: &[ModelFileSpec {
- name: "dog_face_embedding128.onnx",
- size: 4_141_071,
- sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",
- }],
- },
- Self::PetFaceEmbeddingCat => ModelSpec {
- key: "cat_face_embedding128",
- files: &[ModelFileSpec {
- name: "cat_face_embedding128.onnx",
- size: 4_141_071,
- sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",
- }],
- },
- Self::PetBodyDetection => ModelSpec {
- key: "yolov5s_object_fp16",
- files: &[ModelFileSpec {
- name: "yolov5s_object_fp16.onnx",
- size: 14_987_107,
- sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",
- }],
- },
- Self::PetBodyEmbeddingDog => ModelSpec {
- key: "dog_body_embedding192",
- files: &[ModelFileSpec {
- name: "dog_body_embedding192.onnx",
- size: 4_569_361,
- sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",
- }],
- },
- Self::PetBodyEmbeddingCat => ModelSpec {
- key: "cat_body_embedding192",
- files: &[ModelFileSpec {
- name: "cat_body_embedding192.onnx",
- size: 4_569_361,
- sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",
- }],
- },
- }
+fn model_asset_spec(model: Model) -> ModelAssetSpec {
+ match model {
+ Model::FaceDetection => ModelAssetSpec {
+ key: "yolov5s_face_640_640_static_b1",
+ files: &[ModelAssetFile {
+ name: "yolov5s_face_640_640_static_b1.onnx",
+ size: 32_355_091,
+ sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",
+ }],
+ },
+ Model::FaceEmbedding => ModelAssetSpec {
+ key: "mobilefacenet_portable_static_b1",
+ files: &[ModelAssetFile {
+ name: "mobilefacenet_portable_static_b1.onnx",
+ size: 5_278_803,
+ sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",
+ }],
+ },
+ Model::ClipImage => ModelAssetSpec {
+ key: "mobileclip_s2_image_gelu_opset20",
+ files: &[ModelAssetFile {
+ name: "mobileclip_s2_image_gelu_opset20.onnx",
+ size: 143_057_352,
+ sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",
+ }],
+ },
+ Model::ClipText => ModelAssetSpec {
+ key: "mobileclip_s2_text_opset18_quant",
+ files: &[
+ ModelAssetFile {
+ name: "mobileclip_s2_text_opset18_quant.onnx",
+ size: 67_144_712,
+ sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",
+ },
+ ModelAssetFile {
+ name: "bpe_simple_vocab_16e6.txt",
+ size: 3_194_984,
+ sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",
+ },
+ ],
+ },
+ Model::PetFaceDetection => ModelAssetSpec {
+ key: "yolov5s_pet_face_fp16_V2",
+ files: &[ModelAssetFile {
+ name: "yolov5s_pet_face_fp16_V2.onnx",
+ size: 14_758_020,
+ sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",
+ }],
+ },
+ Model::PetFaceEmbeddingDog => ModelAssetSpec {
+ key: "dog_face_embedding128",
+ files: &[ModelAssetFile {
+ name: "dog_face_embedding128.onnx",
+ size: 4_141_071,
+ sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",
+ }],
+ },
+ Model::PetFaceEmbeddingCat => ModelAssetSpec {
+ key: "cat_face_embedding128",
+ files: &[ModelAssetFile {
+ name: "cat_face_embedding128.onnx",
+ size: 4_141_071,
+ sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",
+ }],
+ },
+ Model::PetBodyDetection => ModelAssetSpec {
+ key: "yolov5s_object_fp16",
+ files: &[ModelAssetFile {
+ name: "yolov5s_object_fp16.onnx",
+ size: 14_987_107,
+ sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",
+ }],
+ },
+ Model::PetBodyEmbeddingDog => ModelAssetSpec {
+ key: "dog_body_embedding192",
+ files: &[ModelAssetFile {
+ name: "dog_body_embedding192.onnx",
+ size: 4_569_361,
+ sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",
+ }],
+ },
+ Model::PetBodyEmbeddingCat => ModelAssetSpec {
+ key: "cat_body_embedding192",
+ files: &[ModelAssetFile {
+ name: "cat_body_embedding192.onnx",
+ size: 4_569_361,
+ sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",
+ }],
+ },
}
}
-struct ModelSpec {
+struct ModelAssetSpec {
key: &'static str,
- files: &'static [ModelFileSpec],
+ files: &'static [ModelAssetFile],
}
-struct ModelFileSpec {
+struct ModelAssetFile {
name: &'static str,
size: u64,
sha256: &'static str,
@@ -209,7 +214,7 @@ struct ModelFileSpec {
fn model_path(store: &AssetStore, model: Model) -> String {
let asset = model_asset(model);
store
- .file_path(&asset, model.spec().files[0].name)
+ .file_path(&asset, model_asset_spec(model).files[0].name)
.expect("model file")
.to_string_lossy()
.into_owned()
@@ -224,7 +229,7 @@ fn migrate_desktop_model(
legacy_dir: &Path,
model: Model,
) -> Result<(), String> {
- let spec = model.spec();
+ let spec = model_asset_spec(model);
let asset = model_asset(model);
let legacy_files = spec
.files
@@ -296,7 +301,11 @@ mod tests {
#[test]
fn retired_keys_are_not_in_the_catalog() {
for key in RETIRED_MODEL_KEYS {
- assert!(Model::ALL.iter().all(|model| model.spec().key != *key));
+ assert!(
+ Model::ALL
+ .iter()
+ .all(|model| model_asset_spec(*model).key != *key)
+ );
}
}
@@ -311,7 +320,7 @@ mod tests {
let sha256 = model["sha256"].as_str().unwrap();
let catalog_file = Model::ALL
.iter()
- .flat_map(|model| model.spec().files)
+ .flat_map(|model| model_asset_spec(*model).files)
.find(|file| file.name == name)
.unwrap_or_else(|| panic!("{name} is missing from the Photos model catalog"));
assert_eq!(catalog_file.sha256, sha256, "{name}");
@@ -323,7 +332,7 @@ mod tests {
let temp = TempDir::new().unwrap();
let legacy = temp.path().join("models");
fs::create_dir(&legacy).unwrap();
- create_legacy_file(&legacy, &Model::FaceEmbedding.spec().files[0]);
+ create_legacy_file(&legacy, &model_asset_spec(Model::FaceEmbedding).files[0]);
fs::write(legacy.join(RETIRED_LEGACY_MODEL_FILES[0]), b"retired").unwrap();
let store = AssetStore::new(temp.path().join("assets"));
@@ -335,7 +344,7 @@ mod tests {
fs::metadata(model_path(&store, Model::FaceEmbedding))
.unwrap()
.len(),
- Model::FaceEmbedding.spec().files[0].size
+ model_asset_spec(Model::FaceEmbedding).files[0].size
);
assert!(!legacy.join(RETIRED_LEGACY_MODEL_FILES[0]).exists());
}
@@ -345,9 +354,9 @@ mod tests {
let temp = TempDir::new().unwrap();
let legacy = temp.path().join("models");
fs::create_dir(&legacy).unwrap();
- create_legacy_file(&legacy, &Model::ClipText.spec().files[0]);
+ create_legacy_file(&legacy, &model_asset_spec(Model::ClipText).files[0]);
fs::write(
- legacy.join(Model::FaceDetection.spec().files[0].name),
+ legacy.join(model_asset_spec(Model::FaceDetection).files[0].name),
b"truncated",
)
.unwrap();
@@ -360,7 +369,7 @@ mod tests {
assert!(!legacy.exists());
}
- fn create_legacy_file(directory: &Path, file: &ModelFileSpec) {
+ fn create_legacy_file(directory: &Path, file: &ModelAssetFile) {
let handle = fs::File::create(directory.join(file.name)).unwrap();
handle.set_len(file.size).unwrap();
}
diff --git a/rust/crates/photos/src/ml/clip/image.rs b/rust/crates/photos/src/ml/clip/image.rs
index c8590231e5..f20dd7ae8f 100644
--- a/rust/crates/photos/src/ml/clip/image.rs
+++ b/rust/crates/photos/src/ml/clip/image.rs
@@ -1,6 +1,6 @@
use crate::ml::{
error::MlResult,
- model::Model,
+ models::Model,
onnx, preprocess,
runtime::MlRuntimeView,
types::{ClipResult, DecodedImage},
diff --git a/rust/crates/photos/src/ml/clip/text.rs b/rust/crates/photos/src/ml/clip/text.rs
index 35df370759..01e83898b1 100644
--- a/rust/crates/photos/src/ml/clip/text.rs
+++ b/rust/crates/photos/src/ml/clip/text.rs
@@ -1,7 +1,7 @@
use crate::ml::{
clip::tokenizer,
error::{MlError, MlResult},
- model::Model,
+ models::Model,
onnx,
runtime::MlRuntimeView,
types::ClipResult,
diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rs
index 4e76238e75..81c6e68ba8 100644
--- a/rust/crates/photos/src/ml/face/detect.rs
+++ b/rust/crates/photos/src/ml/face/detect.rs
@@ -1,6 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
- model::Model,
+ models::Model,
onnx,
postprocess::{NmsDetection, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
diff --git a/rust/crates/photos/src/ml/face/embed.rs b/rust/crates/photos/src/ml/face/embed.rs
index 1bbc8f6806..48424aa9ad 100644
--- a/rust/crates/photos/src/ml/face/embed.rs
+++ b/rust/crates/photos/src/ml/face/embed.rs
@@ -1,6 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
- model::Model,
+ models::Model,
onnx,
postprocess::l2_normalize,
runtime::MlRuntimeView,
diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rs
index ab05d712b0..cd1d54239e 100644
--- a/rust/crates/photos/src/ml/indexing.rs
+++ b/rust/crates/photos/src/ml/indexing.rs
@@ -6,14 +6,13 @@ use crate::ml::{
run_face_alignment, run_face_detection, run_face_embedding,
thumbnail::{FaceBox, generate_face_thumbnails},
},
- model::{self, Model},
+ models::{self, Model, ModelPaths},
onnx,
pet::{
run_pet_body_detection, run_pet_body_embedding, run_pet_face_alignment,
run_pet_face_detection, run_pet_face_embedding,
},
- preprocess,
- runtime::{self, ModelPaths},
+ preprocess, runtime,
types::{self, ClipResult, Dimensions, FaceResult, PetBodyResult, PetFaceResult},
};
use ente_image::decode::{decode_image_from_bytes, decode_image_from_path};
@@ -283,7 +282,7 @@ pub fn tokenize_clip_text(text: &str, vocab_path: &str) -> MlResult<Vec<i32>> {
}
fn validate_request_model_paths(req: &AnalyzeImageRequest) -> MlResult<()> {
- let missing = model::required_indexing_models(req.run_faces, req.run_clip, req.run_pets)
+ let missing = models::required_indexing_models(req.run_faces, req.run_clip, req.run_pets)
.filter(|&model| req.model_paths.get(model).trim().is_empty())
.map(Model::path_label)
.collect::<Vec<_>>();
diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rs
index 405a1649e4..3fbfbc9f8f 100644
--- a/rust/crates/photos/src/ml/mod.rs
+++ b/rust/crates/photos/src/ml/mod.rs
@@ -4,7 +4,8 @@ mod diagnostics;
pub mod error;
pub mod face;
pub mod indexing;
-mod model;
+mod models;
+pub use models::ModelPaths;
mod onnx;
pub use onnx::golden_test;
pub use onnx::golden_test::tooling as golden_tooling;
diff --git a/rust/crates/photos/src/ml/model.rs b/rust/crates/photos/src/ml/models.rs
similarity index 99%
rename from rust/crates/photos/src/ml/model.rs
rename to rust/crates/photos/src/ml/models.rs
index 0a280ee974..4e4acfcd50 100644
--- a/rust/crates/photos/src/ml/model.rs
+++ b/rust/crates/photos/src/ml/models.rs
@@ -1,6 +1,6 @@
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
-pub enum Model {
+pub(crate) enum Model {
FaceDetection,
FaceEmbedding,
ClipImage,
diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rs
index 152db8a975..8695c83751 100644
--- a/rust/crates/photos/src/ml/pet/detect.rs
+++ b/rust/crates/photos/src/ml/pet/detect.rs
@@ -1,6 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
- model::Model,
+ models::Model,
onnx,
postprocess::{NmsDetection, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
diff --git a/rust/crates/photos/src/ml/pet/embed.rs b/rust/crates/photos/src/ml/pet/embed.rs
index e56b105603..e4c914e3f6 100644
--- a/rust/crates/photos/src/ml/pet/embed.rs
+++ b/rust/crates/photos/src/ml/pet/embed.rs
@@ -1,6 +1,6 @@
use crate::ml::{
error::{MlError, MlResult},
- model::Model,
+ models::Model,
onnx,
postprocess::l2_normalize,
runtime::MlRuntimeView,
diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rs
index f05e1470e6..70a9d81fec 100644
--- a/rust/crates/photos/src/ml/runtime.rs
+++ b/rust/crates/photos/src/ml/runtime.rs
@@ -7,12 +7,10 @@ use once_cell::sync::Lazy;
use crate::ml::{
error::{MlError, MlResult},
- model::Model,
+ models::{Model, ModelPaths},
onnx,
};
-pub use crate::ml::model::ModelPaths;
-
#[derive(Debug)]
struct ModelSlotState {
path: String,
diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rs
index a6b19b4b74..8aba5f83c9 100644
--- a/rust/crates/photos/tests/support/ml_indexing.rs
+++ b/rust/crates/photos/tests/support/ml_indexing.rs
@@ -7,12 +7,12 @@ use std::{
use anyhow::{Context, Result, bail};
use ente_assets::{Asset, AssetFile, AssetStore, download};
use ente_photos::ml::{
+ ModelPaths,
error::MlError,
indexing::{
AnalyzeImageRequest, AnalyzeImageResult, ImageSource, analyze_image, init_ml_runtime,
release_ml_runtime,
},
- runtime::ModelPaths,
types::FaceResult as RustFaceResult,
};
use flate2::read::GzDecoder;