π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Update ML result bitmap for CoreML and WebGPU additions
laurenspriem1d609bb4067c
diff --git a/mobile/apps/photos/lib/models/ml/ml_versions.dart b/mobile/apps/photos/lib/models/ml/ml_versions.dartindex fc13e000ee..3c84095d24 100644--- a/mobile/apps/photos/lib/models/ml/ml_versions.dart+++ b/mobile/apps/photos/lib/models/ml/ml_versions.dart@@ -9,6 +9,8 @@ const minimumClusterSize = 2;/// Bits for the `flags` bitmask on RemoteFaceEmbedding / RemoteClipEmbedding./// Once shipped, a bit's meaning is immutable β only add new bits.const int mlIndexFlagRuntimeRust = 1 << 0;+const int mlIndexFlagCoreML = 1 << 1;+const int mlIndexFlagWebGPU = 1 << 2;const embeddingFetchLimit = 200;final fileDownloadMlLimit = Platform.isIOS ? 5 : 10;diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_result.dart b/mobile/apps/photos/lib/services/machine_learning/ml_result.dartindex e2994da84f..0c4d624301 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_result.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_result.dart@@ -3,6 +3,7 @@ import "dart:convert" show jsonEncode, jsonDecode;import "package:logging/logging.dart";import "package:photos/models/ml/face/dimension.dart";import 'package:photos/models/ml/ml_typedefs.dart';+import "package:photos/models/ml/ml_versions.dart";import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';@@ -15,6 +16,10 @@ class MLResult {List<PetFaceResult>? petFaces;List<PetBodyResult>? petBodies;+ /// Bitmask attached to the remotely stored embeddings, describing the+ /// runtime and execution providers used to produce this result.+ int remoteFlags;+Dimensions decodedImageSize;bool get ranML => facesRan || clipRan || petsRan;@@ -28,11 +33,13 @@ class MLResult {this.clip,this.petFaces,this.petBodies,+ this.remoteFlags = mlIndexFlagRuntimeRust,this.decodedImageSize = const Dimensions(width: -1, height: -1),});MLResult.fromEnteFileID(int fileID, {+ this.remoteFlags = mlIndexFlagRuntimeRust,this.decodedImageSize = const Dimensions(width: -1, height: -1),}) : fileId = fileID;@@ -42,6 +49,7 @@ class MLResult {'clip': clip?.toJson(),'petFaces': petFaces?.map((pf) => pf.toJson()).toList(),'petBodies': petBodies?.map((obj) => obj.toJson()).toList(),+ 'remoteFlags': remoteFlags,'decodedImageSize': {'width': decodedImageSize.width,'height': decodedImageSize.height,@@ -79,6 +87,7 @@ class MLResult {).toList(): null,+ remoteFlags: json['remoteFlags'] as int? ?? mlIndexFlagRuntimeRust,decodedImageSize: json['decodedImageSize'] != null? Dimensions(width: json['decodedImageSize']['width'],diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dartindex dd2c119fbb..bf822b3ae8 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart@@ -774,7 +774,6 @@ class MLService {actuallyRanML = result.ranML;if (!actuallyRanML) return actuallyRanML;final bool isLocalGallery = instruction.isLocalGallery;- const int remoteFlags = mlIndexFlagRuntimeRust;// Prepare storing data on remote (online mode only)final FileDataEntity? dataEntity = isLocalGallery? null@@ -808,7 +807,7 @@ class MLService {client: client,height: result.decodedImageSize.height,width: result.decodedImageSize.width,- flags: remoteFlags,+ flags: result.remoteFlags,),);}@@ -821,7 +820,7 @@ class MLService {result.clip!.embedding,version: clipMlVersion,client: client,- flags: remoteFlags,+ flags: result.remoteFlags,),);}diff --git a/mobile/apps/photos/lib/utils/ml_util.dart b/mobile/apps/photos/lib/utils/ml_util.dartindex fa546b988d..4e429d51b0 100644--- a/mobile/apps/photos/lib/utils/ml_util.dart+++ b/mobile/apps/photos/lib/utils/ml_util.dart@@ -914,7 +914,13 @@ Future<MLResult> analyzeImageRust(Map args) async {}}- final result = MLResult.fromEnteFileID(enteFileID);+ final result = MLResult.fromEnteFileID(+ enteFileID,+ remoteFlags:+ mlIndexFlagRuntimeRust |+ (rustResult.usedCoreml ? mlIndexFlagCoreML : 0) |+ (rustResult.usedWebgpu ? mlIndexFlagWebGPU : 0),+ );result.decodedImageSize = Dimensions(width: rustResult.decodedImageSize.width,height: rustResult.decodedImageSize.height,diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rsindex fd638d81cf..5822f935ef 100644--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs@@ -116,6 +116,10 @@ pub struct AnalyzeImageResult {pub clip: Option<RustClipResult>,pub pet_faces: Option<Vec<RustPetFaceResult>>,pub pet_bodies: Option<Vec<RustPetBodyResult>>,+ /// True when any model that contributed to this result ran on the+ /// respective accelerated execution provider.+ pub used_coreml: bool,+ pub used_webgpu: bool,}#[derive(Clone, Debug)]@@ -231,6 +235,8 @@ fn to_api_analyze_image_result(result: shared_indexing::AnalyzeImageResult) -> Apet_bodies: result.pet_bodies.map(|bodies| bodies.into_iter().map(to_api_pet_body_result).collect()),+ used_coreml: result.used_coreml,+ used_webgpu: result.used_webgpu,}}diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex b281a37b80..533ef183b3 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -31,6 +31,10 @@ pub struct AnalyzeImageResult {pub clip: Option<ClipResult>,pub pet_faces: Option<Vec<PetFaceResult>>,pub pet_bodies: Option<Vec<PetBodyResult>>,+ /// True when any model that contributed to this result ran on the+ /// respective accelerated execution provider.+ pub used_coreml: bool,+ pub used_webgpu: bool,}#[derive(Clone, Debug)]@@ -145,6 +149,7 @@ pub fn analyze_image(req: AnalyzeImageRequest) -> MlResult<AnalyzeImageResult> {(None, None)};+ let used_providers = runtime.used_providers();Ok(AnalyzeImageResult {file_id,decoded_image_size: dims,@@ -152,6 +157,8 @@ pub fn analyze_image(req: AnalyzeImageRequest) -> MlResult<AnalyzeImageResult> {clip,pet_faces,pet_bodies,+ used_coreml: used_providers.coreml,+ used_webgpu: used_providers.webgpu,})})}diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rsindex ece747ab98..74715130b0 100644--- a/rust/crates/photos/src/ml/onnx.rs+++ b/rust/crates/photos/src/ml/onnx.rs@@ -97,6 +97,21 @@ pub(crate) enum ExecutionMode {CpuOnly,}+/// The preferred execution provider of the session attempt that succeeded.+///+/// Each accelerated session also registers the policy's fallback providers,+/// but this identifies the provider whose attempt produced the session. The+/// runtime aggregates it across the sessions used for a result so remotely+/// stored embeddings can be attributed to the providers that computed them.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+#[allow(dead_code)] // Accelerated variants are constructed only on their target OS.+pub(crate) enum ExecutionProvider {+ CoreMl,+ WebGpu,+ Xnnpack,+ Cpu,+}+impl ExecutionMode {pub(crate) fn fallback(self) -> Option<Self> {match self {@@ -115,17 +130,18 @@ pub(crate) fn build_session(model_path: &str,mode: ExecutionMode,coreml_cache_namespace: &str,-) -> MlResult<Session> {+) -> MlResult<(Session, ExecutionProvider)> {let attempts = provider_attempts(mode, model_path, coreml_cache_namespace);let mut errors = Vec::new();for attempt in attempts {+ let execution_provider = attempt.execution_provider;if attempt.uses_webgpu {#[cfg(target_os = "android")]{match build_webgpu_session_with_canary(model_path, coreml_cache_namespace, attempt){- Ok(session) => return Ok(session),+ Ok(session) => return Ok((session, execution_provider)),Err(error) => errors.push(format!("{error}")),}continue;@@ -145,7 +161,7 @@ pub(crate) fn build_session(cache_dir.display());}- return Ok(session);+ return Ok((session, execution_provider));}Err(error) => {if let Some(cache_dir) = coreml_cache_dir@@ -274,6 +290,7 @@ struct ProviderAttempt {disable_intra_op_spinning: bool,coreml_cache_dir: Option<PathBuf>,uses_webgpu: bool,+ execution_provider: ExecutionProvider,}impl ProviderAttempt {@@ -283,6 +300,7 @@ impl ProviderAttempt {disable_intra_op_spinning: false,coreml_cache_dir: None,uses_webgpu: false,+ execution_provider: ExecutionProvider::Cpu,}}}@@ -317,6 +335,7 @@ fn platform_default_attempts(disable_intra_op_spinning: false,coreml_cache_dir,uses_webgpu: false,+ execution_provider: ExecutionProvider::CoreMl,},ProviderAttempt::cpu_only(),]@@ -366,6 +385,7 @@ fn platform_default_attempts(disable_intra_op_spinning: true,coreml_cache_dir: None,uses_webgpu: true,+ execution_provider: ExecutionProvider::WebGpu,});}// Clean-session fallbacks for provider or driver failures that prevent@@ -530,6 +550,7 @@ fn xnnpack_attempt() -> ProviderAttempt {disable_intra_op_spinning: true,coreml_cache_dir: None,uses_webgpu: false,+ execution_provider: ExecutionProvider::Xnnpack,}}diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex 8bd50d0755..730fd8b60d 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -1,4 +1,5 @@use std::{+ cell::Cell,ops::{Deref, DerefMut},sync::{Mutex, MutexGuard},};@@ -53,6 +54,7 @@ pub struct ModelPaths {struct ModelSlotState {path: String,fallback_execution_mode: Option<onnx::ExecutionMode>,+ execution_provider: Option<onnx::ExecutionProvider>,pin_count: usize,session: Option<Session>,}@@ -68,9 +70,30 @@ pub(crate) struct ModelSessionGuard<'a> {state: MutexGuard<'a, ModelSlotState>,}+/// Which accelerated execution providers were behind the sessions used through+/// a [`MlRuntimeView`]. ORed over every session guard the view hands out, so a+/// result produced through the view can be attributed to the providers that+/// actually computed (any part of) it.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub(crate) struct UsedProviders {+ pub(crate) coreml: bool,+ pub(crate) webgpu: bool,+}++impl UsedProviders {+ fn record(&mut self, provider: onnx::ExecutionProvider) {+ match provider {+ onnx::ExecutionProvider::CoreMl => self.coreml = true,+ onnx::ExecutionProvider::WebGpu => self.webgpu = true,+ onnx::ExecutionProvider::Xnnpack | onnx::ExecutionProvider::Cpu => {}+ }+ }+}+pub(crate) struct MlRuntimeView<'a> {runtime: &'a MlRuntime,model_paths: &'a ModelPaths,+ used_providers: Cell<UsedProviders>,}impl Deref for ModelSessionGuard<'_> {@@ -104,6 +127,7 @@ impl ModelSlot {state: Mutex::new(ModelSlotState {path: String::new(),fallback_execution_mode: None,+ execution_provider: None,pin_count: 0,session: None,}),@@ -163,6 +187,7 @@ impl ModelSlot {};state.fallback_execution_mode = Some(fallback_mode);+ state.execution_provider = None;state.session = None;true}@@ -184,11 +209,13 @@ impl ModelSlot {}state.path = path.to_string();state.fallback_execution_mode = None;+ state.execution_provider = None;state.session = None;}fn clear_transient_runtime_state_locked(state: &mut ModelSlotState) {state.fallback_execution_mode = None;+ state.execution_provider = None;state.session = None;}@@ -218,14 +245,23 @@ impl ModelSlot {"loading {model_name} with {execution_mode:?} execution"));let t = std::time::Instant::now();- let session =+ let (session, execution_provider) =onnx::build_session(&state.path, execution_mode, self.coreml_cache_namespace)?;rt_log(&format!("loaded {model_name} in {:?}", t.elapsed()));+ state.execution_provider = Some(execution_provider);state.session = Some(session);Ok(())}}+impl ModelSessionGuard<'_> {+ fn execution_provider(&self) -> onnx::ExecutionProvider {+ self.state+ .execution_provider+ .expect("a loaded session must record its execution provider")+ }+}+#[derive(Debug)]struct MlRuntime {face_detection: ModelSlot,@@ -359,76 +395,110 @@ impl MlRuntime {MlRuntimeView {runtime: self,model_paths,+ used_providers: Cell::new(UsedProviders::default()),}}}-impl MlRuntimeView<'_> {+impl<'a> MlRuntimeView<'a> {+ /// The accelerated providers behind every session used through this view+ /// so far (since creation or the last [`Self::reset_used_providers`]).+ pub(crate) fn used_providers(&self) -> UsedProviders {+ self.used_providers.get()+ }++ fn reset_used_providers(&self) {+ self.used_providers.set(UsedProviders::default());+ }++ fn tracked_session(+ &self,+ slot: &'a ModelSlot,+ path: &str,+ error_msg: &str,+ ) -> MlResult<ModelSessionGuard<'a>> {+ let guard = slot.session_guard_for(path, error_msg)?;+ let mut used = self.used_providers.get();+ used.record(guard.execution_provider());+ self.used_providers.set(used);+ Ok(guard)+ }+pub(crate) fn face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.face_detection.session_guard_for(+ self.tracked_session(+ &self.runtime.face_detection,&self.model_paths.face_detection,"missing model path: faceDetectionModelPath is required when runFaces is true",)}pub(crate) fn face_embedding_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.face_embedding.session_guard_for(+ self.tracked_session(+ &self.runtime.face_embedding,&self.model_paths.face_embedding,"missing model path: faceEmbeddingModelPath is required when runFaces is true",)}pub(crate) fn clip_image_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.clip_image.session_guard_for(+ self.tracked_session(+ &self.runtime.clip_image,&self.model_paths.clip_image,"missing model path: clipImageModelPath is required when runClip is true",)}pub(crate) fn clip_text_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.clip_text.session_guard_for(+ self.tracked_session(+ &self.runtime.clip_text,&self.model_paths.clip_text,"missing model path: clipTextModelPath is required when running clip text",)}pub(crate) fn pet_face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.pet_face_detection.session_guard_for(+ self.tracked_session(+ &self.runtime.pet_face_detection,&self.model_paths.pet_face_detection,"missing model path: petFaceDetectionModelPath is required when runPets is true",)}pub(crate) fn pet_face_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.pet_face_embedding_dog.session_guard_for(+ self.tracked_session(+ &self.runtime.pet_face_embedding_dog,&self.model_paths.pet_face_embedding_dog,"missing model path: petFaceEmbeddingDogModelPath is required",)}pub(crate) fn pet_face_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.pet_face_embedding_cat.session_guard_for(+ self.tracked_session(+ &self.runtime.pet_face_embedding_cat,&self.model_paths.pet_face_embedding_cat,"missing model path: petFaceEmbeddingCatModelPath is required",)}pub(crate) fn pet_body_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.pet_body_detection.session_guard_for(+ self.tracked_session(+ &self.runtime.pet_body_detection,&self.model_paths.pet_body_detection,"missing model path: petBodyDetectionModelPath is required when runPets is true",)}pub(crate) fn pet_body_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.pet_body_embedding_dog.session_guard_for(+ self.tracked_session(+ &self.runtime.pet_body_embedding_dog,&self.model_paths.pet_body_embedding_dog,"missing model path: petBodyEmbeddingDogModelPath is required",)}pub(crate) fn pet_body_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.runtime.pet_body_embedding_cat.session_guard_for(+ self.tracked_session(+ &self.runtime.pet_body_embedding_cat,&self.model_paths.pet_body_embedding_cat,"missing model path: petBodyEmbeddingCatModelPath is required",)@@ -464,6 +534,9 @@ wherert_log(&format!("execution provider failed, retrying with the next provider fallback: {error}"));+ // Drop provider attributions from the failed attempt; only the+ // retry that produces the returned value should count.+ runtime_view.reset_used_providers();result = func(&runtime_view);}}