Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Update ML result bitmap for CoreML and WebGPU additions

laurenspriem1d609bb4067c
diff --git a/mobile/apps/photos/lib/models/ml/ml_versions.dart b/mobile/apps/photos/lib/models/ml/ml_versions.dart
index fc13e000ee..3c84095d24 100644
--- a/mobile/apps/photos/lib/models/ml/ml_versions.dart
+++ b/mobile/apps/photos/lib/models/ml/ml_versions.dart
@@ -9,6 +9,8 @@ const minimumClusterSize = 2;
/// Bits for the `flags` bitmask on RemoteFaceEmbedding / RemoteClipEmbedding.
/// Once shipped, a bit's meaning is immutable β€” only add new bits.
const int mlIndexFlagRuntimeRust = 1 << 0;
+const int mlIndexFlagCoreML = 1 << 1;
+const int mlIndexFlagWebGPU = 1 << 2;
const embeddingFetchLimit = 200;
final fileDownloadMlLimit = Platform.isIOS ? 5 : 10;
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_result.dart b/mobile/apps/photos/lib/services/machine_learning/ml_result.dart
index e2994da84f..0c4d624301 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_result.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_result.dart
@@ -3,6 +3,7 @@ import "dart:convert" show jsonEncode, jsonDecode;
import "package:logging/logging.dart";
import "package:photos/models/ml/face/dimension.dart";
import 'package:photos/models/ml/ml_typedefs.dart';
+import "package:photos/models/ml/ml_versions.dart";
import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
@@ -15,6 +16,10 @@ class MLResult {
List<PetFaceResult>? petFaces;
List<PetBodyResult>? petBodies;
+ /// Bitmask attached to the remotely stored embeddings, describing the
+ /// runtime and execution providers used to produce this result.
+ int remoteFlags;
+
Dimensions decodedImageSize;
bool get ranML => facesRan || clipRan || petsRan;
@@ -28,11 +33,13 @@ class MLResult {
this.clip,
this.petFaces,
this.petBodies,
+ this.remoteFlags = mlIndexFlagRuntimeRust,
this.decodedImageSize = const Dimensions(width: -1, height: -1),
});
MLResult.fromEnteFileID(
int fileID, {
+ this.remoteFlags = mlIndexFlagRuntimeRust,
this.decodedImageSize = const Dimensions(width: -1, height: -1),
}) : fileId = fileID;
@@ -42,6 +49,7 @@ class MLResult {
'clip': clip?.toJson(),
'petFaces': petFaces?.map((pf) => pf.toJson()).toList(),
'petBodies': petBodies?.map((obj) => obj.toJson()).toList(),
+ 'remoteFlags': remoteFlags,
'decodedImageSize': {
'width': decodedImageSize.width,
'height': decodedImageSize.height,
@@ -79,6 +87,7 @@ class MLResult {
)
.toList()
: null,
+ remoteFlags: json['remoteFlags'] as int? ?? mlIndexFlagRuntimeRust,
decodedImageSize: json['decodedImageSize'] != null
? Dimensions(
width: json['decodedImageSize']['width'],
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
index dd2c119fbb..bf822b3ae8 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
@@ -774,7 +774,6 @@ class MLService {
actuallyRanML = result.ranML;
if (!actuallyRanML) return actuallyRanML;
final bool isLocalGallery = instruction.isLocalGallery;
- const int remoteFlags = mlIndexFlagRuntimeRust;
// Prepare storing data on remote (online mode only)
final FileDataEntity? dataEntity = isLocalGallery
? null
@@ -808,7 +807,7 @@ class MLService {
client: client,
height: result.decodedImageSize.height,
width: result.decodedImageSize.width,
- flags: remoteFlags,
+ flags: result.remoteFlags,
),
);
}
@@ -821,7 +820,7 @@ class MLService {
result.clip!.embedding,
version: clipMlVersion,
client: client,
- flags: remoteFlags,
+ flags: result.remoteFlags,
),
);
}
diff --git a/mobile/apps/photos/lib/utils/ml_util.dart b/mobile/apps/photos/lib/utils/ml_util.dart
index fa546b988d..4e429d51b0 100644
--- a/mobile/apps/photos/lib/utils/ml_util.dart
+++ b/mobile/apps/photos/lib/utils/ml_util.dart
@@ -914,7 +914,13 @@ Future<MLResult> analyzeImageRust(Map args) async {
}
}
- final result = MLResult.fromEnteFileID(enteFileID);
+ final result = MLResult.fromEnteFileID(
+ enteFileID,
+ remoteFlags:
+ mlIndexFlagRuntimeRust |
+ (rustResult.usedCoreml ? mlIndexFlagCoreML : 0) |
+ (rustResult.usedWebgpu ? mlIndexFlagWebGPU : 0),
+ );
result.decodedImageSize = Dimensions(
width: rustResult.decodedImageSize.width,
height: rustResult.decodedImageSize.height,
diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
index fd638d81cf..5822f935ef 100644
--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
@@ -116,6 +116,10 @@ pub struct AnalyzeImageResult {
pub clip: Option<RustClipResult>,
pub pet_faces: Option<Vec<RustPetFaceResult>>,
pub pet_bodies: Option<Vec<RustPetBodyResult>>,
+ /// True when any model that contributed to this result ran on the
+ /// respective accelerated execution provider.
+ pub used_coreml: bool,
+ pub used_webgpu: bool,
}
#[derive(Clone, Debug)]
@@ -231,6 +235,8 @@ fn to_api_analyze_image_result(result: shared_indexing::AnalyzeImageResult) -> A
pet_bodies: result
.pet_bodies
.map(|bodies| bodies.into_iter().map(to_api_pet_body_result).collect()),
+ used_coreml: result.used_coreml,
+ used_webgpu: result.used_webgpu,
}
}
diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rs
index b281a37b80..533ef183b3 100644
--- a/rust/crates/photos/src/ml/indexing.rs
+++ b/rust/crates/photos/src/ml/indexing.rs
@@ -31,6 +31,10 @@ pub struct AnalyzeImageResult {
pub clip: Option<ClipResult>,
pub pet_faces: Option<Vec<PetFaceResult>>,
pub pet_bodies: Option<Vec<PetBodyResult>>,
+ /// True when any model that contributed to this result ran on the
+ /// respective accelerated execution provider.
+ pub used_coreml: bool,
+ pub used_webgpu: bool,
}
#[derive(Clone, Debug)]
@@ -145,6 +149,7 @@ pub fn analyze_image(req: AnalyzeImageRequest) -> MlResult<AnalyzeImageResult> {
(None, None)
};
+ let used_providers = runtime.used_providers();
Ok(AnalyzeImageResult {
file_id,
decoded_image_size: dims,
@@ -152,6 +157,8 @@ pub fn analyze_image(req: AnalyzeImageRequest) -> MlResult<AnalyzeImageResult> {
clip,
pet_faces,
pet_bodies,
+ used_coreml: used_providers.coreml,
+ used_webgpu: used_providers.webgpu,
})
})
}
diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rs
index ece747ab98..74715130b0 100644
--- a/rust/crates/photos/src/ml/onnx.rs
+++ b/rust/crates/photos/src/ml/onnx.rs
@@ -97,6 +97,21 @@ pub(crate) enum ExecutionMode {
CpuOnly,
}
+/// The preferred execution provider of the session attempt that succeeded.
+///
+/// Each accelerated session also registers the policy's fallback providers,
+/// but this identifies the provider whose attempt produced the session. The
+/// runtime aggregates it across the sessions used for a result so remotely
+/// stored embeddings can be attributed to the providers that computed them.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[allow(dead_code)] // Accelerated variants are constructed only on their target OS.
+pub(crate) enum ExecutionProvider {
+ CoreMl,
+ WebGpu,
+ Xnnpack,
+ Cpu,
+}
+
impl ExecutionMode {
pub(crate) fn fallback(self) -> Option<Self> {
match self {
@@ -115,17 +130,18 @@ pub(crate) fn build_session(
model_path: &str,
mode: ExecutionMode,
coreml_cache_namespace: &str,
-) -> MlResult<Session> {
+) -> MlResult<(Session, ExecutionProvider)> {
let attempts = provider_attempts(mode, model_path, coreml_cache_namespace);
let mut errors = Vec::new();
for attempt in attempts {
+ let execution_provider = attempt.execution_provider;
if attempt.uses_webgpu {
#[cfg(target_os = "android")]
{
match build_webgpu_session_with_canary(model_path, coreml_cache_namespace, attempt)
{
- Ok(session) => return Ok(session),
+ Ok(session) => return Ok((session, execution_provider)),
Err(error) => errors.push(format!("{error}")),
}
continue;
@@ -145,7 +161,7 @@ pub(crate) fn build_session(
cache_dir.display()
);
}
- return Ok(session);
+ return Ok((session, execution_provider));
}
Err(error) => {
if let Some(cache_dir) = coreml_cache_dir
@@ -274,6 +290,7 @@ struct ProviderAttempt {
disable_intra_op_spinning: bool,
coreml_cache_dir: Option<PathBuf>,
uses_webgpu: bool,
+ execution_provider: ExecutionProvider,
}
impl ProviderAttempt {
@@ -283,6 +300,7 @@ impl ProviderAttempt {
disable_intra_op_spinning: false,
coreml_cache_dir: None,
uses_webgpu: false,
+ execution_provider: ExecutionProvider::Cpu,
}
}
}
@@ -317,6 +335,7 @@ fn platform_default_attempts(
disable_intra_op_spinning: false,
coreml_cache_dir,
uses_webgpu: false,
+ execution_provider: ExecutionProvider::CoreMl,
},
ProviderAttempt::cpu_only(),
]
@@ -366,6 +385,7 @@ fn platform_default_attempts(
disable_intra_op_spinning: true,
coreml_cache_dir: None,
uses_webgpu: true,
+ execution_provider: ExecutionProvider::WebGpu,
});
}
// Clean-session fallbacks for provider or driver failures that prevent
@@ -530,6 +550,7 @@ fn xnnpack_attempt() -> ProviderAttempt {
disable_intra_op_spinning: true,
coreml_cache_dir: None,
uses_webgpu: false,
+ execution_provider: ExecutionProvider::Xnnpack,
}
}
diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rs
index 8bd50d0755..730fd8b60d 100644
--- a/rust/crates/photos/src/ml/runtime.rs
+++ b/rust/crates/photos/src/ml/runtime.rs
@@ -1,4 +1,5 @@
use std::{
+ cell::Cell,
ops::{Deref, DerefMut},
sync::{Mutex, MutexGuard},
};
@@ -53,6 +54,7 @@ pub struct ModelPaths {
struct ModelSlotState {
path: String,
fallback_execution_mode: Option<onnx::ExecutionMode>,
+ execution_provider: Option<onnx::ExecutionProvider>,
pin_count: usize,
session: Option<Session>,
}
@@ -68,9 +70,30 @@ pub(crate) struct ModelSessionGuard<'a> {
state: MutexGuard<'a, ModelSlotState>,
}
+/// Which accelerated execution providers were behind the sessions used through
+/// a [`MlRuntimeView`]. ORed over every session guard the view hands out, so a
+/// result produced through the view can be attributed to the providers that
+/// actually computed (any part of) it.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub(crate) struct UsedProviders {
+ pub(crate) coreml: bool,
+ pub(crate) webgpu: bool,
+}
+
+impl UsedProviders {
+ fn record(&mut self, provider: onnx::ExecutionProvider) {
+ match provider {
+ onnx::ExecutionProvider::CoreMl => self.coreml = true,
+ onnx::ExecutionProvider::WebGpu => self.webgpu = true,
+ onnx::ExecutionProvider::Xnnpack | onnx::ExecutionProvider::Cpu => {}
+ }
+ }
+}
+
pub(crate) struct MlRuntimeView<'a> {
runtime: &'a MlRuntime,
model_paths: &'a ModelPaths,
+ used_providers: Cell<UsedProviders>,
}
impl Deref for ModelSessionGuard<'_> {
@@ -104,6 +127,7 @@ impl ModelSlot {
state: Mutex::new(ModelSlotState {
path: String::new(),
fallback_execution_mode: None,
+ execution_provider: None,
pin_count: 0,
session: None,
}),
@@ -163,6 +187,7 @@ impl ModelSlot {
};
state.fallback_execution_mode = Some(fallback_mode);
+ state.execution_provider = None;
state.session = None;
true
}
@@ -184,11 +209,13 @@ impl ModelSlot {
}
state.path = path.to_string();
state.fallback_execution_mode = None;
+ state.execution_provider = None;
state.session = None;
}
fn clear_transient_runtime_state_locked(state: &mut ModelSlotState) {
state.fallback_execution_mode = None;
+ state.execution_provider = None;
state.session = None;
}
@@ -218,14 +245,23 @@ impl ModelSlot {
"loading {model_name} with {execution_mode:?} execution"
));
let t = std::time::Instant::now();
- let session =
+ let (session, execution_provider) =
onnx::build_session(&state.path, execution_mode, self.coreml_cache_namespace)?;
rt_log(&format!("loaded {model_name} in {:?}", t.elapsed()));
+ state.execution_provider = Some(execution_provider);
state.session = Some(session);
Ok(())
}
}
+impl ModelSessionGuard<'_> {
+ fn execution_provider(&self) -> onnx::ExecutionProvider {
+ self.state
+ .execution_provider
+ .expect("a loaded session must record its execution provider")
+ }
+}
+
#[derive(Debug)]
struct MlRuntime {
face_detection: ModelSlot,
@@ -359,76 +395,110 @@ impl MlRuntime {
MlRuntimeView {
runtime: self,
model_paths,
+ used_providers: Cell::new(UsedProviders::default()),
}
}
}
-impl MlRuntimeView<'_> {
+impl<'a> MlRuntimeView<'a> {
+ /// The accelerated providers behind every session used through this view
+ /// so far (since creation or the last [`Self::reset_used_providers`]).
+ pub(crate) fn used_providers(&self) -> UsedProviders {
+ self.used_providers.get()
+ }
+
+ fn reset_used_providers(&self) {
+ self.used_providers.set(UsedProviders::default());
+ }
+
+ fn tracked_session(
+ &self,
+ slot: &'a ModelSlot,
+ path: &str,
+ error_msg: &str,
+ ) -> MlResult<ModelSessionGuard<'a>> {
+ let guard = slot.session_guard_for(path, error_msg)?;
+ let mut used = self.used_providers.get();
+ used.record(guard.execution_provider());
+ self.used_providers.set(used);
+ Ok(guard)
+ }
+
pub(crate) fn face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.face_detection.session_guard_for(
+ self.tracked_session(
+ &self.runtime.face_detection,
&self.model_paths.face_detection,
"missing model path: faceDetectionModelPath is required when runFaces is true",
)
}
pub(crate) fn face_embedding_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.face_embedding.session_guard_for(
+ self.tracked_session(
+ &self.runtime.face_embedding,
&self.model_paths.face_embedding,
"missing model path: faceEmbeddingModelPath is required when runFaces is true",
)
}
pub(crate) fn clip_image_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.clip_image.session_guard_for(
+ self.tracked_session(
+ &self.runtime.clip_image,
&self.model_paths.clip_image,
"missing model path: clipImageModelPath is required when runClip is true",
)
}
pub(crate) fn clip_text_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.clip_text.session_guard_for(
+ self.tracked_session(
+ &self.runtime.clip_text,
&self.model_paths.clip_text,
"missing model path: clipTextModelPath is required when running clip text",
)
}
pub(crate) fn pet_face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.pet_face_detection.session_guard_for(
+ self.tracked_session(
+ &self.runtime.pet_face_detection,
&self.model_paths.pet_face_detection,
"missing model path: petFaceDetectionModelPath is required when runPets is true",
)
}
pub(crate) fn pet_face_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.pet_face_embedding_dog.session_guard_for(
+ self.tracked_session(
+ &self.runtime.pet_face_embedding_dog,
&self.model_paths.pet_face_embedding_dog,
"missing model path: petFaceEmbeddingDogModelPath is required",
)
}
pub(crate) fn pet_face_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.pet_face_embedding_cat.session_guard_for(
+ self.tracked_session(
+ &self.runtime.pet_face_embedding_cat,
&self.model_paths.pet_face_embedding_cat,
"missing model path: petFaceEmbeddingCatModelPath is required",
)
}
pub(crate) fn pet_body_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.pet_body_detection.session_guard_for(
+ self.tracked_session(
+ &self.runtime.pet_body_detection,
&self.model_paths.pet_body_detection,
"missing model path: petBodyDetectionModelPath is required when runPets is true",
)
}
pub(crate) fn pet_body_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.pet_body_embedding_dog.session_guard_for(
+ self.tracked_session(
+ &self.runtime.pet_body_embedding_dog,
&self.model_paths.pet_body_embedding_dog,
"missing model path: petBodyEmbeddingDogModelPath is required",
)
}
pub(crate) fn pet_body_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.runtime.pet_body_embedding_cat.session_guard_for(
+ self.tracked_session(
+ &self.runtime.pet_body_embedding_cat,
&self.model_paths.pet_body_embedding_cat,
"missing model path: petBodyEmbeddingCatModelPath is required",
)
@@ -464,6 +534,9 @@ where
rt_log(&format!(
"execution provider failed, retrying with the next provider fallback: {error}"
));
+ // Drop provider attributions from the failed attempt; only the
+ // retry that produces the returned value should count.
+ runtime_view.reset_used_providers();
result = func(&runtime_view);
}
}