π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust][photos] Scope execution provider fallback per model
laurenspriem1161e333bbe7
diff --git a/rust/crates/photos/src/ml/clip/image.rs b/rust/crates/photos/src/ml/clip/image.rsindex 5750d08d39..8ca06f56db 100644--- a/rust/crates/photos/src/ml/clip/image.rs+++ b/rust/crates/photos/src/ml/clip/image.rs@@ -11,18 +11,19 @@ pub(crate) fn run_clip_image(runtime: &MlRuntimeView<'_>,decoded: &DecodedImage,) -> MlResult<ClipResult> {- let input = preprocess::preprocess_clip(decoded)?;- let mut clip_image = runtime.clip_image_session()?;- let (shape, output) = onnx::run_f32(- &mut clip_image,- input,- [- 1,- 3,- CLIP_IMAGE_INPUT_SIZE as i64,- CLIP_IMAGE_INPUT_SIZE as i64,- ],- )?;+ let input = onnx::PreparedF32Input::new(preprocess::preprocess_clip(decoded)?);+ let (shape, output) = runtime.with_clip_image_session(|session| {+ onnx::run_f32(+ session,+ &input,+ [+ 1,+ 3,+ CLIP_IMAGE_INPUT_SIZE as i64,+ CLIP_IMAGE_INPUT_SIZE as i64,+ ],+ )+ })?;finish_embedding("CLIP", shape, output)}diff --git a/rust/crates/photos/src/ml/clip/text.rs b/rust/crates/photos/src/ml/clip/text.rsindex a42db183d8..6e19d42aab 100644--- a/rust/crates/photos/src/ml/clip/text.rs+++ b/rust/crates/photos/src/ml/clip/text.rs@@ -16,9 +16,9 @@ fn run_clip_text(runtime: &MlRuntimeView<'_>, token_ids: &[i32]) -> MlResult<Cli)));}- let mut clip_text = runtime.clip_text_session()?;- let (shape, output) =- onnx::run_i32_f32(&mut clip_text, token_ids, [1, CLIP_TEXT_TOKEN_COUNT as i64])?;+ let (shape, output) = runtime.with_clip_text_session(|session| {+ onnx::run_i32_f32(session, token_ids, [1, CLIP_TEXT_TOKEN_COUNT as i64])+ })?;finish_embedding("CLIP text", shape, output)}diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rsindex b01fe3aae3..6a4da38d7e 100644--- a/rust/crates/photos/src/ml/face/detect.rs+++ b/rust/crates/photos/src/ml/face/detect.rs@@ -14,13 +14,14 @@ pub(crate) fn run_face_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,) -> MlResult<Vec<FaceDetection>> {- let mut face_detection = runtime.face_detection_session()?;- onnx::with_prepared_float_output(- &mut face_detection,- &input.tensor,- [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],- |_output_shape, output_data| postprocess_face_detections(output_data, input),- )+ runtime.with_face_detection_session(|session| {+ onnx::with_prepared_float_output(+ session,+ &input.tensor,+ [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],+ |_output_shape, output_data| postprocess_face_detections(output_data, input),+ )+ })}fn postprocess_face_detections(diff --git a/rust/crates/photos/src/ml/face/embed.rs b/rust/crates/photos/src/ml/face/embed.rsindex dae858dc3c..4a045b82cb 100644--- a/rust/crates/photos/src/ml/face/embed.rs+++ b/rust/crates/photos/src/ml/face/embed.rs@@ -28,7 +28,6 @@ pub(crate) fn run_face_embedding(let face_input_size = FACE_INPUT_SIZE as i64;let expected_input_len = (face_input_size * face_input_size * FACE_INPUT_CHANNELS) as usize;- let mut face_embedding = runtime.face_embedding_session()?;for (aligned, face_result) in aligned_faces.into_iter().zip(face_results.iter_mut()) {if aligned.len() != expected_input_len {return Err(MlError::Preprocess(format!(@@ -38,11 +37,14 @@ pub(crate) fn run_face_embedding()));}- let (shape, mut embedding) = onnx::run_f32(- &mut face_embedding,- aligned,- [1, face_input_size, face_input_size, FACE_INPUT_CHANNELS],- )?;+ let input = onnx::PreparedF32Input::new(aligned);+ let (shape, mut embedding) = runtime.with_face_embedding_session(|session| {+ onnx::run_f32(+ session,+ &input,+ [1, face_input_size, face_input_size, FACE_INPUT_CHANNELS],+ )+ })?;if shape.first() != Some(&1) || embedding.is_empty() {return Err(MlError::Postprocess(format!("invalid single-face embedding tensor shape {:?} for data length {}",diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rsindex cd5b7c01b6..bd0e1a7c66 100644--- a/rust/crates/photos/src/ml/onnx.rs+++ b/rust/crates/photos/src/ml/onnx.rs@@ -1018,15 +1018,16 @@ fn session_expects_f16(session: &Session) -> bool {pub(crate) fn run_f32<const N: usize>(session: &mut Session,- input: Vec<f32>,+ input: &PreparedF32Input,input_shape: [i64; N],) -> MlResult<(Vec<i64>, Vec<f32>)> {let outputs = if session_expects_f16(session) {- let f16_input = Vec::<half::f16>::from_f32_slice(&input);- let input_tensor = Tensor::<half::f16>::from_array((input_shape, f16_input))?;+ let input_tensor =+ TensorRef::<half::f16>::from_array_view((input_shape, input.f16_data()))?;session.run(ort::inputs![input_tensor])?} else {- let input_tensor = Tensor::<f32>::from_array((input_shape, input))?;+ let input_tensor =+ TensorRef::<f32>::from_array_view((input_shape, input.f32_data.as_slice()))?;session.run(ort::inputs![input_tensor])?};diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rsindex 89f77c479c..0124541f2c 100644--- a/rust/crates/photos/src/ml/pet/detect.rs+++ b/rust/crates/photos/src/ml/pet/detect.rs@@ -27,15 +27,16 @@ pub(crate) fn run_pet_face_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,) -> MlResult<Vec<PetFaceDetection>> {- let mut pet_face_detection = runtime.pet_face_detection_session()?;- onnx::with_prepared_float_output(- &mut pet_face_detection,- &input.tensor,- [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],- |output_shape, output_data| {- postprocess_pet_face_detections(output_shape, output_data, input)- },- )+ runtime.with_pet_face_detection_session(|session| {+ onnx::with_prepared_float_output(+ session,+ &input.tensor,+ [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],+ |output_shape, output_data| {+ postprocess_pet_face_detections(output_shape, output_data, input)+ },+ )+ })}fn postprocess_pet_face_detections(@@ -171,13 +172,14 @@ pub(crate) fn run_pet_body_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,) -> MlResult<Vec<PetBodyDetection>> {- let mut body_detection = runtime.pet_body_detection_session()?;- onnx::with_prepared_float_output(- &mut body_detection,- &input.tensor,- [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],- |_output_shape, output_data| postprocess_pet_body_detections(output_data, input),- )+ runtime.with_pet_body_detection_session(|session| {+ onnx::with_prepared_float_output(+ session,+ &input.tensor,+ [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],+ |_output_shape, output_data| postprocess_pet_body_detections(output_data, input),+ )+ })}fn postprocess_pet_body_detections(diff --git a/rust/crates/photos/src/ml/pet/embed.rs b/rust/crates/photos/src/ml/pet/embed.rsindex 967ec836b4..60f5cda0ad 100644--- a/rust/crates/photos/src/ml/pet/embed.rs+++ b/rust/crates/photos/src/ml/pet/embed.rs@@ -45,23 +45,23 @@ pub(crate) fn run_pet_face_embedding()));}- let mut session = if species == PET_SPECIES_DOG {- runtime.pet_face_embedding_dog_session()?+ let input = onnx::PreparedF32Input::new(batch.input);+ let input_shape = [+ batch.indices.len() as i64,+ PET_EMBEDDING_CHANNELS as i64,+ PET_EMBEDDING_INPUT_SIZE as i64,+ PET_EMBEDDING_INPUT_SIZE as i64,+ ];+ let (shape, output) = if species == PET_SPECIES_DOG {+ runtime.with_pet_face_embedding_dog_session(|session| {+ onnx::run_f32(session, &input, input_shape)+ })?} else {- runtime.pet_face_embedding_cat_session()?+ runtime.with_pet_face_embedding_cat_session(|session| {+ onnx::run_f32(session, &input, input_shape)+ })?};- let (shape, output) = onnx::run_f32(- &mut session,- batch.input,- [- batch.indices.len() as i64,- PET_EMBEDDING_CHANNELS as i64,- PET_EMBEDDING_INPUT_SIZE as i64,- PET_EMBEDDING_INPUT_SIZE as i64,- ],- )?;-let embedding_size =validate_embedding_batch_output("face", &shape, output.len(), batch.indices.len())?;@@ -120,23 +120,23 @@ pub(crate) fn run_pet_body_embedding(continue;}- let mut session = if is_cat {- runtime.pet_body_embedding_cat_session()?+ let input = onnx::PreparedF32Input::new(batch.input);+ let input_shape = [+ batch.indices.len() as i64,+ PET_EMBEDDING_CHANNELS as i64,+ PET_EMBEDDING_INPUT_SIZE as i64,+ PET_EMBEDDING_INPUT_SIZE as i64,+ ];+ let (shape, output) = if is_cat {+ runtime.with_pet_body_embedding_cat_session(|session| {+ onnx::run_f32(session, &input, input_shape)+ })?} else {- runtime.pet_body_embedding_dog_session()?+ runtime.with_pet_body_embedding_dog_session(|session| {+ onnx::run_f32(session, &input, input_shape)+ })?};- let (shape, output) = onnx::run_f32(- &mut session,- batch.input,- [- batch.indices.len() as i64,- PET_EMBEDDING_CHANNELS as i64,- PET_EMBEDDING_INPUT_SIZE as i64,- PET_EMBEDDING_INPUT_SIZE as i64,- ],- )?;-let embedding_size =validate_embedding_batch_output("body", &shape, output.len(), batch.indices.len())?;diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex bef629c6ca..da182aa604 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -1,6 +1,5 @@use std::{cell::Cell,- ops::{Deref, DerefMut},sync::{Mutex, MutexGuard},};@@ -65,14 +64,9 @@ struct ModelSlot {state: Mutex<ModelSlotState>,}-pub(crate) struct ModelSessionGuard<'a> {- state: MutexGuard<'a, ModelSlotState>,-}-/// Which accelerated execution providers were behind the sessions used through-/// a [`MlRuntimeView`]. ORed over every session guard the view hands out, so a-/// result produced through the view can be attributed to the providers that-/// actually computed (any part of) it.+/// a [`MlRuntimeView`]. ORed over successful model executions so a result can be+/// attributed to the providers that actually computed it.#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]pub(crate) struct UsedProviders {pub(crate) coreml: bool,@@ -95,26 +89,6 @@ pub(crate) struct MlRuntimeView<'a> {used_providers: Cell<UsedProviders>,}-impl Deref for ModelSessionGuard<'_> {- type Target = Session;-- fn deref(&self) -> &Self::Target {- self.state- .session- .as_ref()- .expect("session must be loaded before creating ModelSessionGuard")- }-}--impl DerefMut for ModelSessionGuard<'_> {- fn deref_mut(&mut self) -> &mut Self::Target {- self.state- .session- .as_mut()- .expect("session must be loaded before creating ModelSessionGuard")- }-}-impl ModelSlot {fn new(default_execution_mode: onnx::ExecutionMode,@@ -169,15 +143,7 @@ impl ModelSlot {}}- fn advance_provider_fallback_if_configured(&self, path: &str) -> bool {- if path.trim().is_empty() {- return false;- }-- let mut state = self.lock_state();- if state.path != path {- return false;- }+ fn advance_provider_fallback_locked(&self, state: &mut ModelSlotState) -> bool {let current_mode = state.fallback_execution_mode.unwrap_or(self.default_execution_mode);@@ -191,15 +157,66 @@ impl ModelSlot {true}- fn session_guard_for(&self, path: &str, error_msg: &str) -> MlResult<ModelSessionGuard<'_>> {+ fn run<T>(+ &self,+ path: &str,+ error_msg: &str,+ mut operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<(T, onnx::ExecutionProvider)> {if path.trim().is_empty() {return Err(MlError::InvalidRequest(error_msg.to_string()));}- let mut state = self.lock_state();- Self::set_config_locked(&mut state, path);- self.ensure_loaded_locked(&mut state, error_msg)?;- Ok(ModelSessionGuard { state })+ loop {+ let mut state = self.lock_state();+ Self::set_config_locked(&mut state, path);+ if let Err(error) = self.ensure_loaded_locked(&mut state, error_msg) {+ if self.retry_after_provider_failure_locked(&mut state, &error) {+ continue;+ }+ return Err(error);+ }++ let execution_provider = state+ .execution_provider+ .expect("a loaded session must record its execution provider");+ let result = operation(+ state+ .session+ .as_mut()+ .expect("session must be loaded before model execution"),+ );+ match result {+ Ok(value) => return Ok((value, execution_provider)),+ Err(error) => {+ if self.retry_after_provider_failure_locked(&mut state, &error) {+ continue;+ }+ return Err(error);+ }+ }+ }+ }++ fn retry_after_provider_failure_locked(+ &self,+ state: &mut ModelSlotState,+ error: &MlError,+ ) -> bool {+ if !should_retry_execution_provider_runtime(error)+ || !self.advance_provider_fallback_locked(state)+ {+ return false;+ }++ crate::ml::events::record(+ crate::ml::events::Severity::Warning,+ format!(+ "execution provider failed, retrying model with the next provider fallback: \+ {error}"+ ),+ );+ true}fn set_config_locked(state: &mut ModelSlotState, path: &str) {@@ -253,14 +270,6 @@ impl ModelSlot {}}-impl ModelSessionGuard<'_> {- fn execution_provider(&self) -> onnx::ExecutionProvider {- self.state- .execution_provider- .expect("a loaded session must record its execution provider")- }-}-#[derive(Debug)]struct MlRuntime {face_detection: ModelSlot,@@ -356,41 +365,6 @@ impl MlRuntime {self.pet_body_embedding_cat.release_residency();}- fn advance_provider_fallbacks_for_requested_models(&self, model_paths: &ModelPaths) -> bool {- let mut advanced = false;- advanced |= self- .face_detection- .advance_provider_fallback_if_configured(&model_paths.face_detection);- advanced |= self- .face_embedding- .advance_provider_fallback_if_configured(&model_paths.face_embedding);- advanced |= self- .clip_image- .advance_provider_fallback_if_configured(&model_paths.clip_image);- advanced |= self- .clip_text- .advance_provider_fallback_if_configured(&model_paths.clip_text);- advanced |= self- .pet_face_detection- .advance_provider_fallback_if_configured(&model_paths.pet_face_detection);- advanced |= self- .pet_face_embedding_dog- .advance_provider_fallback_if_configured(&model_paths.pet_face_embedding_dog);- advanced |= self- .pet_face_embedding_cat- .advance_provider_fallback_if_configured(&model_paths.pet_face_embedding_cat);- advanced |= self- .pet_body_detection- .advance_provider_fallback_if_configured(&model_paths.pet_body_detection);- advanced |= self- .pet_body_embedding_dog- .advance_provider_fallback_if_configured(&model_paths.pet_body_embedding_dog);- advanced |= self- .pet_body_embedding_cat- .advance_provider_fallback_if_configured(&model_paths.pet_body_embedding_cat);- advanced- }-fn view<'a>(&'a self, model_paths: &'a ModelPaths) -> MlRuntimeView<'a> {MlRuntimeView {runtime: self,@@ -401,106 +375,143 @@ impl MlRuntime {}impl<'a> MlRuntimeView<'a> {- /// The accelerated providers behind every session used through this view- /// so far (since creation or the last [`Self::reset_used_providers`]).+ /// The accelerated providers behind every successful model execution+ /// through this view so far.pub(crate) fn used_providers(&self) -> UsedProviders {self.used_providers.get()}- fn reset_used_providers(&self) {- self.used_providers.set(UsedProviders::default());- }-- fn tracked_session(+ fn run_tracked<T>(&self,- slot: &'a ModelSlot,+ slot: &ModelSlot,path: &str,error_msg: &str,- ) -> MlResult<ModelSessionGuard<'a>> {- let guard = slot.session_guard_for(path, error_msg)?;+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ let (value, execution_provider) = slot.run(path, error_msg, operation)?;let mut used = self.used_providers.get();- used.record(guard.execution_provider());+ used.record(execution_provider);self.used_providers.set(used);- Ok(guard)+ Ok(value)}- pub(crate) fn face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_face_detection_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.face_detection,&self.model_paths.face_detection,"missing model path: faceDetectionModelPath is required when runFaces is true",+ operation,)}- pub(crate) fn face_embedding_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_face_embedding_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.face_embedding,&self.model_paths.face_embedding,"missing model path: faceEmbeddingModelPath is required when runFaces is true",+ operation,)}- pub(crate) fn clip_image_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_clip_image_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.clip_image,&self.model_paths.clip_image,"missing model path: clipImageModelPath is required when runClip is true",+ operation,)}- pub(crate) fn clip_text_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_clip_text_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.clip_text,&self.model_paths.clip_text,"missing model path: clipTextModelPath is required when running clip text",+ operation,)}- pub(crate) fn pet_face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_pet_face_detection_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.pet_face_detection,&self.model_paths.pet_face_detection,"missing model path: petFaceDetectionModelPath is required when runPets is true",+ operation,)}- pub(crate) fn pet_face_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_pet_face_embedding_dog_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.pet_face_embedding_dog,&self.model_paths.pet_face_embedding_dog,"missing model path: petFaceEmbeddingDogModelPath is required",+ operation,)}- pub(crate) fn pet_face_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_pet_face_embedding_cat_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.pet_face_embedding_cat,&self.model_paths.pet_face_embedding_cat,"missing model path: petFaceEmbeddingCatModelPath is required",+ operation,)}- pub(crate) fn pet_body_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_pet_body_detection_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.pet_body_detection,&self.model_paths.pet_body_detection,"missing model path: petBodyDetectionModelPath is required when runPets is true",+ operation,)}- pub(crate) fn pet_body_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_pet_body_embedding_dog_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.pet_body_embedding_dog,&self.model_paths.pet_body_embedding_dog,"missing model path: petBodyEmbeddingDogModelPath is required",+ operation,)}- pub(crate) fn pet_body_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {- self.tracked_session(+ pub(crate) fn with_pet_body_embedding_cat_session<T>(+ &self,+ operation: impl FnMut(&mut Session) -> MlResult<T>,+ ) -> MlResult<T> {+ self.run_tracked(&self.runtime.pet_body_embedding_cat,&self.model_paths.pet_body_embedding_cat,"missing model path: petBodyEmbeddingCatModelPath is required",+ operation,)}}@@ -513,38 +524,14 @@ pub(crate) fn prepare_runtime(model_paths: &ModelPaths) {GLOBAL_RUNTIME.prepare_indexing_models(model_paths);}-pub(crate) fn with_runtime<F, R>(model_paths: &ModelPaths, func: F) -> MlResult<R>-where- F: for<'a> Fn(&MlRuntimeView<'a>) -> MlResult<R>,-{+pub(crate) fn with_runtime<R>(+ model_paths: &ModelPaths,+ func: impl FnOnce(&MlRuntimeView<'_>) -> MlResult<R>,+) -> MlResult<R> {ensure_runtime(model_paths);let runtime_view = GLOBAL_RUNTIME.view(model_paths);- let mut result = func(&runtime_view);- loop {- match result {- Ok(value) => return Ok(value),- Err(error) => {- if !should_retry_execution_provider_runtime(&error)- || !GLOBAL_RUNTIME.advance_provider_fallbacks_for_requested_models(model_paths)- {- return Err(error);- }-- crate::ml::events::record(- crate::ml::events::Severity::Warning,- format!(- "execution provider failed, retrying with the next provider fallback: \- {error}"- ),- );- // Drop provider attributions from the failed attempt; only the- // retry that produces the returned value should count.- runtime_view.reset_used_providers();- result = func(&runtime_view);- }- }- }+ func(&runtime_view)}pub(crate) fn release_runtime() {@@ -616,6 +603,47 @@ mod tests {assert_eq!(clip_text.path, "clip_text.onnx");}+ #[test]+ fn provider_fallback_advances_only_the_selected_model_slot() {+ let runtime = MlRuntime::new();+ let model_paths = ModelPaths {+ face_detection: "face.onnx".to_string(),+ clip_image: "clip.onnx".to_string(),+ ..empty_paths()+ };+ runtime.configure_requested_models(&model_paths);++ {+ let mut face_detection = runtime.face_detection.lock_state();+ assert!(+ runtime+ .face_detection+ .advance_provider_fallback_locked(&mut face_detection)+ );+ }++ let face_detection = runtime.face_detection.lock_state();+ assert_eq!(+ face_detection.fallback_execution_mode,+ onnx::ExecutionMode::PlatformDefault.fallback()+ );+ let clip_image = runtime.clip_image.lock_state();+ assert_eq!(clip_image.fallback_execution_mode, None);+ }++ #[test]+ fn runtime_does_not_replay_the_pipeline_after_provider_failure() {+ let calls = Cell::new(0);++ let result: MlResult<()> = with_runtime(&empty_paths(), |_| {+ calls.set(calls.get() + 1);+ Err(MlError::Ort("ExecutionProvider failed".to_string()))+ });++ assert!(result.is_err());+ assert_eq!(calls.get(), 1);+ }+#[test]fn release_indexing_models_keeps_clip_text_state() {let runtime = MlRuntime::new();@@ -732,6 +760,23 @@ mod tests {assert_eq!(onnx::ExecutionMode::CpuOnly.fallback(), None);}+ #[test]+ fn model_slot_provider_fallback_is_bounded() {+ let slot = ModelSlot::new(onnx::ExecutionMode::PlatformDefault, "test-model");+ let mut state = slot.lock_state();+ let mut advances = 0;++ while slot.advance_provider_fallback_locked(&mut state) {+ advances += 1;+ assert!(advances <= 2);+ }++ #[cfg(target_os = "android")]+ assert_eq!(advances, 2);+ #[cfg(not(target_os = "android"))]+ assert_eq!(advances, 1);+ }+#[test]fn model_execution_modes_match_platform_policy() {let runtime = MlRuntime::new();