π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Clarify ML model and asset ownership
laurenspriem94fd463c5014
diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rsindex 90bc725948..5245693cac 100644--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs@@ -1,4 +1,4 @@-use ente_photos::ml::{error::MlError, indexing, runtime::ModelPaths, types};+use ente_photos::ml::{ModelPaths, error::MlError, indexing, types};#[derive(Clone, Debug)]pub struct RustModelPaths {diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex 5903e7a58c..d723c92c85 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -1,5 +1,5 @@use ente_photos::ml::{- assets, error::MlError, indexing as shared_indexing, runtime::ModelPaths as SharedModelPaths,+ ModelPaths as SharedModelPaths, assets, error::MlError, indexing as shared_indexing,types as shared_types,};use napi::bindgen_prelude::*;@@ -80,12 +80,7 @@ impl AssetStore {run_clip: bool,run_pets: bool,) -> Result<ModelPaths> {- let models = assets::indexing_models(run_faces, run_clip, run_pets);- let model_assets = models- .iter()- .copied()- .map(assets::model_asset)- .collect::<Vec<_>>();+ let model_assets = assets::indexing_assets(run_faces, run_clip, run_pets);self.inner.download(&model_assets,@@ -104,7 +99,7 @@ impl AssetStore {#[napi]pub async fn clip_text_model_paths(&self) -> Result<ClipTextModelPaths> {- let asset = assets::model_asset(assets::Model::ClipText);+ let asset = assets::clip_text_asset();self.inner.download(std::slice::from_ref(&asset),diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rsindex 3bfcdd1b5e..67c65bd89c 100644--- a/rust/crates/photos/src/ml/assets.rs+++ b/rust/crates/photos/src/ml/assets.rs@@ -3,9 +3,7 @@ use std::path::{Path, PathBuf};use ente_assets::{Asset, AssetFile, AssetStore};-use super::model::{self, ModelPaths};--pub use super::model::Model;+use super::models::{self, Model, ModelPaths};const MODELS: &str = "models";@@ -29,8 +27,8 @@ pub struct ClipTextPaths {pub vocab: PathBuf,}-pub fn model_asset(model: Model) -> Asset {- let spec = model.spec();+fn model_asset(model: Model) -> Asset {+ let spec = model_asset_spec(model);Asset::files(model_key(spec.key),spec.files@@ -45,8 +43,10 @@ pub fn model_asset(model: Model) -> Asset {.expect("valid Photos model catalog")}-pub fn indexing_models(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Model> {- model::enabled_indexing_models(run_faces, run_clip, run_pets).collect()+pub fn indexing_assets(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Asset> {+ models::enabled_indexing_models(run_faces, run_clip, run_pets)+ .map(model_asset)+ .collect()}pub fn indexing_model_paths(@@ -56,15 +56,15 @@ pub fn indexing_model_paths(run_pets: bool,) -> ModelPaths {let mut model_paths = ModelPaths::default();- for model in model::enabled_indexing_models(run_faces, run_clip, run_pets) {+ for model in models::enabled_indexing_models(run_faces, run_clip, run_pets) {*model_paths.get_mut(model) = model_path(store, model);}model_paths}pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {- let asset = model_asset(Model::ClipText);- let files = Model::ClipText.spec().files;+ let asset = clip_text_asset();+ let files = model_asset_spec(Model::ClipText).files;ClipTextPaths {model: store.file_path(&asset, files[0].name)@@ -75,12 +75,19 @@ pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {}}+pub fn clip_text_asset() -> Asset {+ model_asset(Model::ClipText)+}+pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {let mut warnings = Vec::new();if legacy_dir.exists() {for model in Model::ALL {if let Err(error) = migrate_desktop_model(store, legacy_dir, model) {- warnings.push(format!("Failed to migrate {}: {error}", model.spec().key));+ warnings.push(format!(+ "Failed to migrate {}: {error}",+ model_asset_spec(model).key+ ));}}for name in RETIRED_LEGACY_MODEL_FILES {@@ -101,106 +108,104 @@ pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<Striwarnings}-impl Model {- fn spec(self) -> ModelSpec {- match self {- Self::FaceDetection => ModelSpec {- key: "yolov5s_face_640_640_static_b1",- files: &[ModelFileSpec {- name: "yolov5s_face_640_640_static_b1.onnx",- size: 32_355_091,- sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",- }],- },- Self::FaceEmbedding => ModelSpec {- key: "mobilefacenet_portable_static_b1",- files: &[ModelFileSpec {- name: "mobilefacenet_portable_static_b1.onnx",- size: 5_278_803,- sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",- }],- },- Self::ClipImage => ModelSpec {- key: "mobileclip_s2_image_gelu_opset20",- files: &[ModelFileSpec {- name: "mobileclip_s2_image_gelu_opset20.onnx",- size: 143_057_352,- sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",- }],- },- Self::ClipText => ModelSpec {- key: "mobileclip_s2_text_opset18_quant",- files: &[- ModelFileSpec {- name: "mobileclip_s2_text_opset18_quant.onnx",- size: 67_144_712,- sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",- },- ModelFileSpec {- name: "bpe_simple_vocab_16e6.txt",- size: 3_194_984,- sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",- },- ],- },- Self::PetFaceDetection => ModelSpec {- key: "yolov5s_pet_face_fp16_V2",- files: &[ModelFileSpec {- name: "yolov5s_pet_face_fp16_V2.onnx",- size: 14_758_020,- sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",- }],- },- Self::PetFaceEmbeddingDog => ModelSpec {- key: "dog_face_embedding128",- files: &[ModelFileSpec {- name: "dog_face_embedding128.onnx",- size: 4_141_071,- sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",- }],- },- Self::PetFaceEmbeddingCat => ModelSpec {- key: "cat_face_embedding128",- files: &[ModelFileSpec {- name: "cat_face_embedding128.onnx",- size: 4_141_071,- sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",- }],- },- Self::PetBodyDetection => ModelSpec {- key: "yolov5s_object_fp16",- files: &[ModelFileSpec {- name: "yolov5s_object_fp16.onnx",- size: 14_987_107,- sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",- }],- },- Self::PetBodyEmbeddingDog => ModelSpec {- key: "dog_body_embedding192",- files: &[ModelFileSpec {- name: "dog_body_embedding192.onnx",- size: 4_569_361,- sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",- }],- },- Self::PetBodyEmbeddingCat => ModelSpec {- key: "cat_body_embedding192",- files: &[ModelFileSpec {- name: "cat_body_embedding192.onnx",- size: 4_569_361,- sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",- }],- },- }+fn model_asset_spec(model: Model) -> ModelAssetSpec {+ match model {+ Model::FaceDetection => ModelAssetSpec {+ key: "yolov5s_face_640_640_static_b1",+ files: &[ModelAssetFile {+ name: "yolov5s_face_640_640_static_b1.onnx",+ size: 32_355_091,+ sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",+ }],+ },+ Model::FaceEmbedding => ModelAssetSpec {+ key: "mobilefacenet_portable_static_b1",+ files: &[ModelAssetFile {+ name: "mobilefacenet_portable_static_b1.onnx",+ size: 5_278_803,+ sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",+ }],+ },+ Model::ClipImage => ModelAssetSpec {+ key: "mobileclip_s2_image_gelu_opset20",+ files: &[ModelAssetFile {+ name: "mobileclip_s2_image_gelu_opset20.onnx",+ size: 143_057_352,+ sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",+ }],+ },+ Model::ClipText => ModelAssetSpec {+ key: "mobileclip_s2_text_opset18_quant",+ files: &[+ ModelAssetFile {+ name: "mobileclip_s2_text_opset18_quant.onnx",+ size: 67_144_712,+ sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",+ },+ ModelAssetFile {+ name: "bpe_simple_vocab_16e6.txt",+ size: 3_194_984,+ sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",+ },+ ],+ },+ Model::PetFaceDetection => ModelAssetSpec {+ key: "yolov5s_pet_face_fp16_V2",+ files: &[ModelAssetFile {+ name: "yolov5s_pet_face_fp16_V2.onnx",+ size: 14_758_020,+ sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",+ }],+ },+ Model::PetFaceEmbeddingDog => ModelAssetSpec {+ key: "dog_face_embedding128",+ files: &[ModelAssetFile {+ name: "dog_face_embedding128.onnx",+ size: 4_141_071,+ sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",+ }],+ },+ Model::PetFaceEmbeddingCat => ModelAssetSpec {+ key: "cat_face_embedding128",+ files: &[ModelAssetFile {+ name: "cat_face_embedding128.onnx",+ size: 4_141_071,+ sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",+ }],+ },+ Model::PetBodyDetection => ModelAssetSpec {+ key: "yolov5s_object_fp16",+ files: &[ModelAssetFile {+ name: "yolov5s_object_fp16.onnx",+ size: 14_987_107,+ sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",+ }],+ },+ Model::PetBodyEmbeddingDog => ModelAssetSpec {+ key: "dog_body_embedding192",+ files: &[ModelAssetFile {+ name: "dog_body_embedding192.onnx",+ size: 4_569_361,+ sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",+ }],+ },+ Model::PetBodyEmbeddingCat => ModelAssetSpec {+ key: "cat_body_embedding192",+ files: &[ModelAssetFile {+ name: "cat_body_embedding192.onnx",+ size: 4_569_361,+ sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",+ }],+ },}}-struct ModelSpec {+struct ModelAssetSpec {key: &'static str,- files: &'static [ModelFileSpec],+ files: &'static [ModelAssetFile],}-struct ModelFileSpec {+struct ModelAssetFile {name: &'static str,size: u64,sha256: &'static str,@@ -209,7 +214,7 @@ struct ModelFileSpec {fn model_path(store: &AssetStore, model: Model) -> String {let asset = model_asset(model);store- .file_path(&asset, model.spec().files[0].name)+ .file_path(&asset, model_asset_spec(model).files[0].name).expect("model file").to_string_lossy().into_owned()@@ -224,7 +229,7 @@ fn migrate_desktop_model(legacy_dir: &Path,model: Model,) -> Result<(), String> {- let spec = model.spec();+ let spec = model_asset_spec(model);let asset = model_asset(model);let legacy_files = spec.files@@ -296,7 +301,11 @@ mod tests {#[test]fn retired_keys_are_not_in_the_catalog() {for key in RETIRED_MODEL_KEYS {- assert!(Model::ALL.iter().all(|model| model.spec().key != *key));+ assert!(+ Model::ALL+ .iter()+ .all(|model| model_asset_spec(*model).key != *key)+ );}}@@ -311,7 +320,7 @@ mod tests {let sha256 = model["sha256"].as_str().unwrap();let catalog_file = Model::ALL.iter()- .flat_map(|model| model.spec().files)+ .flat_map(|model| model_asset_spec(*model).files).find(|file| file.name == name).unwrap_or_else(|| panic!("{name} is missing from the Photos model catalog"));assert_eq!(catalog_file.sha256, sha256, "{name}");@@ -323,7 +332,7 @@ mod tests {let temp = TempDir::new().unwrap();let legacy = temp.path().join("models");fs::create_dir(&legacy).unwrap();- create_legacy_file(&legacy, &Model::FaceEmbedding.spec().files[0]);+ create_legacy_file(&legacy, &model_asset_spec(Model::FaceEmbedding).files[0]);fs::write(legacy.join(RETIRED_LEGACY_MODEL_FILES[0]), b"retired").unwrap();let store = AssetStore::new(temp.path().join("assets"));@@ -335,7 +344,7 @@ mod tests {fs::metadata(model_path(&store, Model::FaceEmbedding)).unwrap().len(),- Model::FaceEmbedding.spec().files[0].size+ model_asset_spec(Model::FaceEmbedding).files[0].size);assert!(!legacy.join(RETIRED_LEGACY_MODEL_FILES[0]).exists());}@@ -345,9 +354,9 @@ mod tests {let temp = TempDir::new().unwrap();let legacy = temp.path().join("models");fs::create_dir(&legacy).unwrap();- create_legacy_file(&legacy, &Model::ClipText.spec().files[0]);+ create_legacy_file(&legacy, &model_asset_spec(Model::ClipText).files[0]);fs::write(- legacy.join(Model::FaceDetection.spec().files[0].name),+ legacy.join(model_asset_spec(Model::FaceDetection).files[0].name),b"truncated",).unwrap();@@ -360,7 +369,7 @@ mod tests {assert!(!legacy.exists());}- fn create_legacy_file(directory: &Path, file: &ModelFileSpec) {+ fn create_legacy_file(directory: &Path, file: &ModelAssetFile) {let handle = fs::File::create(directory.join(file.name)).unwrap();handle.set_len(file.size).unwrap();}diff --git a/rust/crates/photos/src/ml/clip/image.rs b/rust/crates/photos/src/ml/clip/image.rsindex c8590231e5..f20dd7ae8f 100644--- a/rust/crates/photos/src/ml/clip/image.rs+++ b/rust/crates/photos/src/ml/clip/image.rs@@ -1,6 +1,6 @@use crate::ml::{error::MlResult,- model::Model,+ models::Model,onnx, preprocess,runtime::MlRuntimeView,types::{ClipResult, DecodedImage},diff --git a/rust/crates/photos/src/ml/clip/text.rs b/rust/crates/photos/src/ml/clip/text.rsindex 35df370759..01e83898b1 100644--- a/rust/crates/photos/src/ml/clip/text.rs+++ b/rust/crates/photos/src/ml/clip/text.rs@@ -1,7 +1,7 @@use crate::ml::{clip::tokenizer,error::{MlError, MlResult},- model::Model,+ models::Model,onnx,runtime::MlRuntimeView,types::ClipResult,diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rsindex 4e76238e75..81c6e68ba8 100644--- a/rust/crates/photos/src/ml/face/detect.rs+++ b/rust/crates/photos/src/ml/face/detect.rs@@ -1,6 +1,6 @@use crate::ml::{error::{MlError, MlResult},- model::Model,+ models::Model,onnx,postprocess::{NmsDetection, greedy_non_max_suppression},preprocess::{YOLO_INPUT_SIZE, YoloInput},diff --git a/rust/crates/photos/src/ml/face/embed.rs b/rust/crates/photos/src/ml/face/embed.rsindex 1bbc8f6806..48424aa9ad 100644--- a/rust/crates/photos/src/ml/face/embed.rs+++ b/rust/crates/photos/src/ml/face/embed.rs@@ -1,6 +1,6 @@use crate::ml::{error::{MlError, MlResult},- model::Model,+ models::Model,onnx,postprocess::l2_normalize,runtime::MlRuntimeView,diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex ab05d712b0..cd1d54239e 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -6,14 +6,13 @@ use crate::ml::{run_face_alignment, run_face_detection, run_face_embedding,thumbnail::{FaceBox, generate_face_thumbnails},},- model::{self, Model},+ models::{self, Model, ModelPaths},onnx,pet::{run_pet_body_detection, run_pet_body_embedding, run_pet_face_alignment,run_pet_face_detection, run_pet_face_embedding,},- preprocess,- runtime::{self, ModelPaths},+ preprocess, runtime,types::{self, ClipResult, Dimensions, FaceResult, PetBodyResult, PetFaceResult},};use ente_image::decode::{decode_image_from_bytes, decode_image_from_path};@@ -283,7 +282,7 @@ pub fn tokenize_clip_text(text: &str, vocab_path: &str) -> MlResult<Vec<i32>> {}fn validate_request_model_paths(req: &AnalyzeImageRequest) -> MlResult<()> {- let missing = model::required_indexing_models(req.run_faces, req.run_clip, req.run_pets)+ let missing = models::required_indexing_models(req.run_faces, req.run_clip, req.run_pets).filter(|&model| req.model_paths.get(model).trim().is_empty()).map(Model::path_label).collect::<Vec<_>>();diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rsindex 405a1649e4..3fbfbc9f8f 100644--- a/rust/crates/photos/src/ml/mod.rs+++ b/rust/crates/photos/src/ml/mod.rs@@ -4,7 +4,8 @@ mod diagnostics;pub mod error;pub mod face;pub mod indexing;-mod model;+mod models;+pub use models::ModelPaths;mod onnx;pub use onnx::golden_test;pub use onnx::golden_test::tooling as golden_tooling;diff --git a/rust/crates/photos/src/ml/model.rs b/rust/crates/photos/src/ml/models.rssimilarity index 99%rename from rust/crates/photos/src/ml/model.rsrename to rust/crates/photos/src/ml/models.rsindex 0a280ee974..4e4acfcd50 100644--- a/rust/crates/photos/src/ml/model.rs+++ b/rust/crates/photos/src/ml/models.rs@@ -1,6 +1,6 @@#[derive(Clone, Copy, Debug, PartialEq, Eq)]#[repr(usize)]-pub enum Model {+pub(crate) enum Model {FaceDetection,FaceEmbedding,ClipImage,diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rsindex 152db8a975..8695c83751 100644--- a/rust/crates/photos/src/ml/pet/detect.rs+++ b/rust/crates/photos/src/ml/pet/detect.rs@@ -1,6 +1,6 @@use crate::ml::{error::{MlError, MlResult},- model::Model,+ models::Model,onnx,postprocess::{NmsDetection, greedy_non_max_suppression},preprocess::{YOLO_INPUT_SIZE, YoloInput},diff --git a/rust/crates/photos/src/ml/pet/embed.rs b/rust/crates/photos/src/ml/pet/embed.rsindex e56b105603..e4c914e3f6 100644--- a/rust/crates/photos/src/ml/pet/embed.rs+++ b/rust/crates/photos/src/ml/pet/embed.rs@@ -1,6 +1,6 @@use crate::ml::{error::{MlError, MlResult},- model::Model,+ models::Model,onnx,postprocess::l2_normalize,runtime::MlRuntimeView,diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex f05e1470e6..70a9d81fec 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -7,12 +7,10 @@ use once_cell::sync::Lazy;use crate::ml::{error::{MlError, MlResult},- model::Model,+ models::{Model, ModelPaths},onnx,};-pub use crate::ml::model::ModelPaths;-#[derive(Debug)]struct ModelSlotState {path: String,diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rsindex a6b19b4b74..8aba5f83c9 100644--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@ -7,12 +7,12 @@ use std::{use anyhow::{Context, Result, bail};use ente_assets::{Asset, AssetFile, AssetStore, download};use ente_photos::ml::{+ ModelPaths,error::MlError,indexing::{AnalyzeImageRequest, AnalyzeImageResult, ImageSource, analyze_image, init_ml_runtime,release_ml_runtime,},- runtime::ModelPaths,types::FaceResult as RustFaceResult,};use flate2::read::GzDecoder;