Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][photos] Scope execution provider fallback per model

laurenspriem1161e333bbe7
diff --git a/rust/crates/photos/src/ml/clip/image.rs b/rust/crates/photos/src/ml/clip/image.rs
index 5750d08d39..8ca06f56db 100644
--- a/rust/crates/photos/src/ml/clip/image.rs
+++ b/rust/crates/photos/src/ml/clip/image.rs
@@ -11,18 +11,19 @@ pub(crate) fn run_clip_image(
runtime: &MlRuntimeView<'_>,
decoded: &DecodedImage,
) -> MlResult<ClipResult> {
- let input = preprocess::preprocess_clip(decoded)?;
- let mut clip_image = runtime.clip_image_session()?;
- let (shape, output) = onnx::run_f32(
- &mut clip_image,
- input,
- [
- 1,
- 3,
- CLIP_IMAGE_INPUT_SIZE as i64,
- CLIP_IMAGE_INPUT_SIZE as i64,
- ],
- )?;
+ let input = onnx::PreparedF32Input::new(preprocess::preprocess_clip(decoded)?);
+ let (shape, output) = runtime.with_clip_image_session(|session| {
+ onnx::run_f32(
+ session,
+ &input,
+ [
+ 1,
+ 3,
+ CLIP_IMAGE_INPUT_SIZE as i64,
+ CLIP_IMAGE_INPUT_SIZE as i64,
+ ],
+ )
+ })?;
finish_embedding("CLIP", shape, output)
}
diff --git a/rust/crates/photos/src/ml/clip/text.rs b/rust/crates/photos/src/ml/clip/text.rs
index a42db183d8..6e19d42aab 100644
--- a/rust/crates/photos/src/ml/clip/text.rs
+++ b/rust/crates/photos/src/ml/clip/text.rs
@@ -16,9 +16,9 @@ fn run_clip_text(runtime: &MlRuntimeView<'_>, token_ids: &[i32]) -> MlResult<Cli
)));
}
- let mut clip_text = runtime.clip_text_session()?;
- let (shape, output) =
- onnx::run_i32_f32(&mut clip_text, token_ids, [1, CLIP_TEXT_TOKEN_COUNT as i64])?;
+ let (shape, output) = runtime.with_clip_text_session(|session| {
+ onnx::run_i32_f32(session, token_ids, [1, CLIP_TEXT_TOKEN_COUNT as i64])
+ })?;
finish_embedding("CLIP text", shape, output)
}
diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rs
index b01fe3aae3..6a4da38d7e 100644
--- a/rust/crates/photos/src/ml/face/detect.rs
+++ b/rust/crates/photos/src/ml/face/detect.rs
@@ -14,13 +14,14 @@ pub(crate) fn run_face_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<FaceDetection>> {
- let mut face_detection = runtime.face_detection_session()?;
- onnx::with_prepared_float_output(
- &mut face_detection,
- &input.tensor,
- [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
- |_output_shape, output_data| postprocess_face_detections(output_data, input),
- )
+ runtime.with_face_detection_session(|session| {
+ onnx::with_prepared_float_output(
+ session,
+ &input.tensor,
+ [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
+ |_output_shape, output_data| postprocess_face_detections(output_data, input),
+ )
+ })
}
fn postprocess_face_detections(
diff --git a/rust/crates/photos/src/ml/face/embed.rs b/rust/crates/photos/src/ml/face/embed.rs
index dae858dc3c..4a045b82cb 100644
--- a/rust/crates/photos/src/ml/face/embed.rs
+++ b/rust/crates/photos/src/ml/face/embed.rs
@@ -28,7 +28,6 @@ pub(crate) fn run_face_embedding(
let face_input_size = FACE_INPUT_SIZE as i64;
let expected_input_len = (face_input_size * face_input_size * FACE_INPUT_CHANNELS) as usize;
- let mut face_embedding = runtime.face_embedding_session()?;
for (aligned, face_result) in aligned_faces.into_iter().zip(face_results.iter_mut()) {
if aligned.len() != expected_input_len {
return Err(MlError::Preprocess(format!(
@@ -38,11 +37,14 @@ pub(crate) fn run_face_embedding(
)));
}
- let (shape, mut embedding) = onnx::run_f32(
- &mut face_embedding,
- aligned,
- [1, face_input_size, face_input_size, FACE_INPUT_CHANNELS],
- )?;
+ let input = onnx::PreparedF32Input::new(aligned);
+ let (shape, mut embedding) = runtime.with_face_embedding_session(|session| {
+ onnx::run_f32(
+ session,
+ &input,
+ [1, face_input_size, face_input_size, FACE_INPUT_CHANNELS],
+ )
+ })?;
if shape.first() != Some(&1) || embedding.is_empty() {
return Err(MlError::Postprocess(format!(
"invalid single-face embedding tensor shape {:?} for data length {}",
diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rs
index cd5b7c01b6..bd0e1a7c66 100644
--- a/rust/crates/photos/src/ml/onnx.rs
+++ b/rust/crates/photos/src/ml/onnx.rs
@@ -1018,15 +1018,16 @@ fn session_expects_f16(session: &Session) -> bool {
pub(crate) fn run_f32<const N: usize>(
session: &mut Session,
- input: Vec<f32>,
+ input: &PreparedF32Input,
input_shape: [i64; N],
) -> MlResult<(Vec<i64>, Vec<f32>)> {
let outputs = if session_expects_f16(session) {
- let f16_input = Vec::<half::f16>::from_f32_slice(&input);
- let input_tensor = Tensor::<half::f16>::from_array((input_shape, f16_input))?;
+ let input_tensor =
+ TensorRef::<half::f16>::from_array_view((input_shape, input.f16_data()))?;
session.run(ort::inputs![input_tensor])?
} else {
- let input_tensor = Tensor::<f32>::from_array((input_shape, input))?;
+ let input_tensor =
+ TensorRef::<f32>::from_array_view((input_shape, input.f32_data.as_slice()))?;
session.run(ort::inputs![input_tensor])?
};
diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rs
index 89f77c479c..0124541f2c 100644
--- a/rust/crates/photos/src/ml/pet/detect.rs
+++ b/rust/crates/photos/src/ml/pet/detect.rs
@@ -27,15 +27,16 @@ pub(crate) fn run_pet_face_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetFaceDetection>> {
- let mut pet_face_detection = runtime.pet_face_detection_session()?;
- onnx::with_prepared_float_output(
- &mut pet_face_detection,
- &input.tensor,
- [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
- |output_shape, output_data| {
- postprocess_pet_face_detections(output_shape, output_data, input)
- },
- )
+ runtime.with_pet_face_detection_session(|session| {
+ onnx::with_prepared_float_output(
+ session,
+ &input.tensor,
+ [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
+ |output_shape, output_data| {
+ postprocess_pet_face_detections(output_shape, output_data, input)
+ },
+ )
+ })
}
fn postprocess_pet_face_detections(
@@ -171,13 +172,14 @@ pub(crate) fn run_pet_body_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetBodyDetection>> {
- let mut body_detection = runtime.pet_body_detection_session()?;
- onnx::with_prepared_float_output(
- &mut body_detection,
- &input.tensor,
- [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
- |_output_shape, output_data| postprocess_pet_body_detections(output_data, input),
- )
+ runtime.with_pet_body_detection_session(|session| {
+ onnx::with_prepared_float_output(
+ session,
+ &input.tensor,
+ [1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
+ |_output_shape, output_data| postprocess_pet_body_detections(output_data, input),
+ )
+ })
}
fn postprocess_pet_body_detections(
diff --git a/rust/crates/photos/src/ml/pet/embed.rs b/rust/crates/photos/src/ml/pet/embed.rs
index 967ec836b4..60f5cda0ad 100644
--- a/rust/crates/photos/src/ml/pet/embed.rs
+++ b/rust/crates/photos/src/ml/pet/embed.rs
@@ -45,23 +45,23 @@ pub(crate) fn run_pet_face_embedding(
)));
}
- let mut session = if species == PET_SPECIES_DOG {
- runtime.pet_face_embedding_dog_session()?
+ let input = onnx::PreparedF32Input::new(batch.input);
+ let input_shape = [
+ batch.indices.len() as i64,
+ PET_EMBEDDING_CHANNELS as i64,
+ PET_EMBEDDING_INPUT_SIZE as i64,
+ PET_EMBEDDING_INPUT_SIZE as i64,
+ ];
+ let (shape, output) = if species == PET_SPECIES_DOG {
+ runtime.with_pet_face_embedding_dog_session(|session| {
+ onnx::run_f32(session, &input, input_shape)
+ })?
} else {
- runtime.pet_face_embedding_cat_session()?
+ runtime.with_pet_face_embedding_cat_session(|session| {
+ onnx::run_f32(session, &input, input_shape)
+ })?
};
- let (shape, output) = onnx::run_f32(
- &mut session,
- batch.input,
- [
- batch.indices.len() as i64,
- PET_EMBEDDING_CHANNELS as i64,
- PET_EMBEDDING_INPUT_SIZE as i64,
- PET_EMBEDDING_INPUT_SIZE as i64,
- ],
- )?;
-
let embedding_size =
validate_embedding_batch_output("face", &shape, output.len(), batch.indices.len())?;
@@ -120,23 +120,23 @@ pub(crate) fn run_pet_body_embedding(
continue;
}
- let mut session = if is_cat {
- runtime.pet_body_embedding_cat_session()?
+ let input = onnx::PreparedF32Input::new(batch.input);
+ let input_shape = [
+ batch.indices.len() as i64,
+ PET_EMBEDDING_CHANNELS as i64,
+ PET_EMBEDDING_INPUT_SIZE as i64,
+ PET_EMBEDDING_INPUT_SIZE as i64,
+ ];
+ let (shape, output) = if is_cat {
+ runtime.with_pet_body_embedding_cat_session(|session| {
+ onnx::run_f32(session, &input, input_shape)
+ })?
} else {
- runtime.pet_body_embedding_dog_session()?
+ runtime.with_pet_body_embedding_dog_session(|session| {
+ onnx::run_f32(session, &input, input_shape)
+ })?
};
- let (shape, output) = onnx::run_f32(
- &mut session,
- batch.input,
- [
- batch.indices.len() as i64,
- PET_EMBEDDING_CHANNELS as i64,
- PET_EMBEDDING_INPUT_SIZE as i64,
- PET_EMBEDDING_INPUT_SIZE as i64,
- ],
- )?;
-
let embedding_size =
validate_embedding_batch_output("body", &shape, output.len(), batch.indices.len())?;
diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rs
index bef629c6ca..da182aa604 100644
--- a/rust/crates/photos/src/ml/runtime.rs
+++ b/rust/crates/photos/src/ml/runtime.rs
@@ -1,6 +1,5 @@
use std::{
cell::Cell,
- ops::{Deref, DerefMut},
sync::{Mutex, MutexGuard},
};
@@ -65,14 +64,9 @@ struct ModelSlot {
state: Mutex<ModelSlotState>,
}
-pub(crate) struct ModelSessionGuard<'a> {
- state: MutexGuard<'a, ModelSlotState>,
-}
-
/// Which accelerated execution providers were behind the sessions used through
-/// a [`MlRuntimeView`]. ORed over every session guard the view hands out, so a
-/// result produced through the view can be attributed to the providers that
-/// actually computed (any part of) it.
+/// a [`MlRuntimeView`]. ORed over successful model executions so a result can be
+/// attributed to the providers that actually computed it.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct UsedProviders {
pub(crate) coreml: bool,
@@ -95,26 +89,6 @@ pub(crate) struct MlRuntimeView<'a> {
used_providers: Cell<UsedProviders>,
}
-impl Deref for ModelSessionGuard<'_> {
- type Target = Session;
-
- fn deref(&self) -> &Self::Target {
- self.state
- .session
- .as_ref()
- .expect("session must be loaded before creating ModelSessionGuard")
- }
-}
-
-impl DerefMut for ModelSessionGuard<'_> {
- fn deref_mut(&mut self) -> &mut Self::Target {
- self.state
- .session
- .as_mut()
- .expect("session must be loaded before creating ModelSessionGuard")
- }
-}
-
impl ModelSlot {
fn new(
default_execution_mode: onnx::ExecutionMode,
@@ -169,15 +143,7 @@ impl ModelSlot {
}
}
- fn advance_provider_fallback_if_configured(&self, path: &str) -> bool {
- if path.trim().is_empty() {
- return false;
- }
-
- let mut state = self.lock_state();
- if state.path != path {
- return false;
- }
+ fn advance_provider_fallback_locked(&self, state: &mut ModelSlotState) -> bool {
let current_mode = state
.fallback_execution_mode
.unwrap_or(self.default_execution_mode);
@@ -191,15 +157,66 @@ impl ModelSlot {
true
}
- fn session_guard_for(&self, path: &str, error_msg: &str) -> MlResult<ModelSessionGuard<'_>> {
+ fn run<T>(
+ &self,
+ path: &str,
+ error_msg: &str,
+ mut operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<(T, onnx::ExecutionProvider)> {
if path.trim().is_empty() {
return Err(MlError::InvalidRequest(error_msg.to_string()));
}
- let mut state = self.lock_state();
- Self::set_config_locked(&mut state, path);
- self.ensure_loaded_locked(&mut state, error_msg)?;
- Ok(ModelSessionGuard { state })
+ loop {
+ let mut state = self.lock_state();
+ Self::set_config_locked(&mut state, path);
+ if let Err(error) = self.ensure_loaded_locked(&mut state, error_msg) {
+ if self.retry_after_provider_failure_locked(&mut state, &error) {
+ continue;
+ }
+ return Err(error);
+ }
+
+ let execution_provider = state
+ .execution_provider
+ .expect("a loaded session must record its execution provider");
+ let result = operation(
+ state
+ .session
+ .as_mut()
+ .expect("session must be loaded before model execution"),
+ );
+ match result {
+ Ok(value) => return Ok((value, execution_provider)),
+ Err(error) => {
+ if self.retry_after_provider_failure_locked(&mut state, &error) {
+ continue;
+ }
+ return Err(error);
+ }
+ }
+ }
+ }
+
+ fn retry_after_provider_failure_locked(
+ &self,
+ state: &mut ModelSlotState,
+ error: &MlError,
+ ) -> bool {
+ if !should_retry_execution_provider_runtime(error)
+ || !self.advance_provider_fallback_locked(state)
+ {
+ return false;
+ }
+
+ crate::ml::events::record(
+ crate::ml::events::Severity::Warning,
+ format!(
+ "execution provider failed, retrying model with the next provider fallback: \
+ {error}"
+ ),
+ );
+ true
}
fn set_config_locked(state: &mut ModelSlotState, path: &str) {
@@ -253,14 +270,6 @@ impl ModelSlot {
}
}
-impl ModelSessionGuard<'_> {
- fn execution_provider(&self) -> onnx::ExecutionProvider {
- self.state
- .execution_provider
- .expect("a loaded session must record its execution provider")
- }
-}
-
#[derive(Debug)]
struct MlRuntime {
face_detection: ModelSlot,
@@ -356,41 +365,6 @@ impl MlRuntime {
self.pet_body_embedding_cat.release_residency();
}
- fn advance_provider_fallbacks_for_requested_models(&self, model_paths: &ModelPaths) -> bool {
- let mut advanced = false;
- advanced |= self
- .face_detection
- .advance_provider_fallback_if_configured(&model_paths.face_detection);
- advanced |= self
- .face_embedding
- .advance_provider_fallback_if_configured(&model_paths.face_embedding);
- advanced |= self
- .clip_image
- .advance_provider_fallback_if_configured(&model_paths.clip_image);
- advanced |= self
- .clip_text
- .advance_provider_fallback_if_configured(&model_paths.clip_text);
- advanced |= self
- .pet_face_detection
- .advance_provider_fallback_if_configured(&model_paths.pet_face_detection);
- advanced |= self
- .pet_face_embedding_dog
- .advance_provider_fallback_if_configured(&model_paths.pet_face_embedding_dog);
- advanced |= self
- .pet_face_embedding_cat
- .advance_provider_fallback_if_configured(&model_paths.pet_face_embedding_cat);
- advanced |= self
- .pet_body_detection
- .advance_provider_fallback_if_configured(&model_paths.pet_body_detection);
- advanced |= self
- .pet_body_embedding_dog
- .advance_provider_fallback_if_configured(&model_paths.pet_body_embedding_dog);
- advanced |= self
- .pet_body_embedding_cat
- .advance_provider_fallback_if_configured(&model_paths.pet_body_embedding_cat);
- advanced
- }
-
fn view<'a>(&'a self, model_paths: &'a ModelPaths) -> MlRuntimeView<'a> {
MlRuntimeView {
runtime: self,
@@ -401,106 +375,143 @@ impl MlRuntime {
}
impl<'a> MlRuntimeView<'a> {
- /// The accelerated providers behind every session used through this view
- /// so far (since creation or the last [`Self::reset_used_providers`]).
+ /// The accelerated providers behind every successful model execution
+ /// through this view so far.
pub(crate) fn used_providers(&self) -> UsedProviders {
self.used_providers.get()
}
- fn reset_used_providers(&self) {
- self.used_providers.set(UsedProviders::default());
- }
-
- fn tracked_session(
+ fn run_tracked<T>(
&self,
- slot: &'a ModelSlot,
+ slot: &ModelSlot,
path: &str,
error_msg: &str,
- ) -> MlResult<ModelSessionGuard<'a>> {
- let guard = slot.session_guard_for(path, error_msg)?;
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ let (value, execution_provider) = slot.run(path, error_msg, operation)?;
let mut used = self.used_providers.get();
- used.record(guard.execution_provider());
+ used.record(execution_provider);
self.used_providers.set(used);
- Ok(guard)
+ Ok(value)
}
- pub(crate) fn face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_face_detection_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.face_detection,
&self.model_paths.face_detection,
"missing model path: faceDetectionModelPath is required when runFaces is true",
+ operation,
)
}
- pub(crate) fn face_embedding_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_face_embedding_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.face_embedding,
&self.model_paths.face_embedding,
"missing model path: faceEmbeddingModelPath is required when runFaces is true",
+ operation,
)
}
- pub(crate) fn clip_image_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_clip_image_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.clip_image,
&self.model_paths.clip_image,
"missing model path: clipImageModelPath is required when runClip is true",
+ operation,
)
}
- pub(crate) fn clip_text_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_clip_text_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.clip_text,
&self.model_paths.clip_text,
"missing model path: clipTextModelPath is required when running clip text",
+ operation,
)
}
- pub(crate) fn pet_face_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_pet_face_detection_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.pet_face_detection,
&self.model_paths.pet_face_detection,
"missing model path: petFaceDetectionModelPath is required when runPets is true",
+ operation,
)
}
- pub(crate) fn pet_face_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_pet_face_embedding_dog_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.pet_face_embedding_dog,
&self.model_paths.pet_face_embedding_dog,
"missing model path: petFaceEmbeddingDogModelPath is required",
+ operation,
)
}
- pub(crate) fn pet_face_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_pet_face_embedding_cat_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.pet_face_embedding_cat,
&self.model_paths.pet_face_embedding_cat,
"missing model path: petFaceEmbeddingCatModelPath is required",
+ operation,
)
}
- pub(crate) fn pet_body_detection_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_pet_body_detection_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.pet_body_detection,
&self.model_paths.pet_body_detection,
"missing model path: petBodyDetectionModelPath is required when runPets is true",
+ operation,
)
}
- pub(crate) fn pet_body_embedding_dog_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_pet_body_embedding_dog_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.pet_body_embedding_dog,
&self.model_paths.pet_body_embedding_dog,
"missing model path: petBodyEmbeddingDogModelPath is required",
+ operation,
)
}
- pub(crate) fn pet_body_embedding_cat_session(&self) -> MlResult<ModelSessionGuard<'_>> {
- self.tracked_session(
+ pub(crate) fn with_pet_body_embedding_cat_session<T>(
+ &self,
+ operation: impl FnMut(&mut Session) -> MlResult<T>,
+ ) -> MlResult<T> {
+ self.run_tracked(
&self.runtime.pet_body_embedding_cat,
&self.model_paths.pet_body_embedding_cat,
"missing model path: petBodyEmbeddingCatModelPath is required",
+ operation,
)
}
}
@@ -513,38 +524,14 @@ pub(crate) fn prepare_runtime(model_paths: &ModelPaths) {
GLOBAL_RUNTIME.prepare_indexing_models(model_paths);
}
-pub(crate) fn with_runtime<F, R>(model_paths: &ModelPaths, func: F) -> MlResult<R>
-where
- F: for<'a> Fn(&MlRuntimeView<'a>) -> MlResult<R>,
-{
+pub(crate) fn with_runtime<R>(
+ model_paths: &ModelPaths,
+ func: impl FnOnce(&MlRuntimeView<'_>) -> MlResult<R>,
+) -> MlResult<R> {
ensure_runtime(model_paths);
let runtime_view = GLOBAL_RUNTIME.view(model_paths);
- let mut result = func(&runtime_view);
- loop {
- match result {
- Ok(value) => return Ok(value),
- Err(error) => {
- if !should_retry_execution_provider_runtime(&error)
- || !GLOBAL_RUNTIME.advance_provider_fallbacks_for_requested_models(model_paths)
- {
- return Err(error);
- }
-
- crate::ml::events::record(
- crate::ml::events::Severity::Warning,
- format!(
- "execution provider failed, retrying with the next provider fallback: \
- {error}"
- ),
- );
- // Drop provider attributions from the failed attempt; only the
- // retry that produces the returned value should count.
- runtime_view.reset_used_providers();
- result = func(&runtime_view);
- }
- }
- }
+ func(&runtime_view)
}
pub(crate) fn release_runtime() {
@@ -616,6 +603,47 @@ mod tests {
assert_eq!(clip_text.path, "clip_text.onnx");
}
+ #[test]
+ fn provider_fallback_advances_only_the_selected_model_slot() {
+ let runtime = MlRuntime::new();
+ let model_paths = ModelPaths {
+ face_detection: "face.onnx".to_string(),
+ clip_image: "clip.onnx".to_string(),
+ ..empty_paths()
+ };
+ runtime.configure_requested_models(&model_paths);
+
+ {
+ let mut face_detection = runtime.face_detection.lock_state();
+ assert!(
+ runtime
+ .face_detection
+ .advance_provider_fallback_locked(&mut face_detection)
+ );
+ }
+
+ let face_detection = runtime.face_detection.lock_state();
+ assert_eq!(
+ face_detection.fallback_execution_mode,
+ onnx::ExecutionMode::PlatformDefault.fallback()
+ );
+ let clip_image = runtime.clip_image.lock_state();
+ assert_eq!(clip_image.fallback_execution_mode, None);
+ }
+
+ #[test]
+ fn runtime_does_not_replay_the_pipeline_after_provider_failure() {
+ let calls = Cell::new(0);
+
+ let result: MlResult<()> = with_runtime(&empty_paths(), |_| {
+ calls.set(calls.get() + 1);
+ Err(MlError::Ort("ExecutionProvider failed".to_string()))
+ });
+
+ assert!(result.is_err());
+ assert_eq!(calls.get(), 1);
+ }
+
#[test]
fn release_indexing_models_keeps_clip_text_state() {
let runtime = MlRuntime::new();
@@ -732,6 +760,23 @@ mod tests {
assert_eq!(onnx::ExecutionMode::CpuOnly.fallback(), None);
}
+ #[test]
+ fn model_slot_provider_fallback_is_bounded() {
+ let slot = ModelSlot::new(onnx::ExecutionMode::PlatformDefault, "test-model");
+ let mut state = slot.lock_state();
+ let mut advances = 0;
+
+ while slot.advance_provider_fallback_locked(&mut state) {
+ advances += 1;
+ assert!(advances <= 2);
+ }
+
+ #[cfg(target_os = "android")]
+ assert_eq!(advances, 2);
+ #[cfg(not(target_os = "android"))]
+ assert_eq!(advances, 1);
+ }
+
#[test]
fn model_execution_modes_match_platform_policy() {
let runtime = MlRuntime::new();