Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Use typed ONNX Runtime errors

Manav Rathi1356af9fefe9
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index 006c056297..ac4e799bba 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -230,6 +230,15 @@ impl SessionRunError {
Self::Retryable(error)
}
+ fn from_inference_error(error: ort::Error) -> Self {
+ match error.code() {
+ ort::ErrorCode::GenericFailure
+ | ort::ErrorCode::RuntimeException
+ | ort::ErrorCode::ExecutionProviderFailure => Self::Retryable(error.into()),
+ _ => Self::Terminal(error.into()),
+ }
+ }
+
fn is_retryable(&self) -> bool {
matches!(self, Self::Retryable(_))
}
@@ -263,12 +272,14 @@ impl From<SessionRunError> for MlError {
impl<R> From<ort::Error<R>> for SessionRunError {
fn from(error: ort::Error<R>) -> Self {
- let error = MlError::Ort(error.to_string());
- if is_execution_provider_run_failure(&error) {
- Self::Retryable(error)
- } else {
- Self::Terminal(error)
- }
+ Self::Terminal(error.into())
+ }
+}
+
+fn session_load_error<R>(model_path: &str, error: ort::Error<R>) -> MlError {
+ match error.code() {
+ ort::ErrorCode::InvalidProtobuf => MlError::CorruptModel(model_path.to_string()),
+ _ => error.into(),
}
}
@@ -321,7 +332,10 @@ fn build_next_session(
Err(errors) => errors,
};
- if has_protobuf_parse_failure(&errors) {
+ if errors
+ .iter()
+ .any(|error| matches!(error, MlError::CorruptModel(_)))
+ {
return Err(MlError::CorruptModel(model_path.to_string()));
}
@@ -350,23 +364,6 @@ fn build_cpu_session(model_path: &str) -> MlResult<Session> {
.map(|loaded| loaded.session)
}
-fn is_execution_provider_run_failure(error: &MlError) -> bool {
- let MlError::Ort(message) = error else {
- return false;
- };
- let normalized = message.to_ascii_lowercase();
- normalized.contains("executionprovider")
- || normalized.contains("unknown allocation device")
- || normalized.contains("xnnpackexecutionprovider")
- || normalized.contains("coremlexecutionprovider")
- || normalized.contains("webgpu")
- || normalized.contains("wgpu")
- || normalized.contains("dawn")
- || normalized.contains("vulkan")
- || normalized.contains("vk_error")
- || normalized.contains("ep error")
-}
-
// A CoreML self-test failure is treated as construction failure so the caller
// invalidates the possibly corrupt persistent cache. WebGPU validation stays
// inside its crash-canary window instead.
@@ -620,24 +617,9 @@ fn model_file_label(model_path: &str) -> &str {
.unwrap_or(model_path)
}
-fn has_protobuf_parse_failure(errors: &[MlError]) -> bool {
- errors.iter().any(|error| {
- matches!(
- error,
- MlError::Ort(message)
- if message
- .to_ascii_lowercase()
- .contains("protobuf parsing failed")
- )
- })
-}
-
#[cfg(test)]
mod tests {
- use super::{
- ExecutionMode, ExecutionProvider, OnnxSession, SessionRunError, has_protobuf_parse_failure,
- is_execution_provider_run_failure, provider_attempt_failure_message,
- };
+ use super::{ExecutionMode, ExecutionProvider, OnnxSession, provider_attempt_failure_message};
fn first_run_canary(temp: &tempfile::TempDir) -> super::webgpu::ArmedCanary {
let model = temp.path().join("model.onnx");
@@ -690,45 +672,6 @@ mod tests {
assert!(has_canary(&temp));
}
- #[test]
- fn provider_failure_detection_is_limited_to_ort_execution_errors() {
- assert!(is_execution_provider_run_failure(&super::MlError::Ort(
- "WebGPU EP error: VK_ERROR_DEVICE_LOST".to_string()
- )));
- assert!(!is_execution_provider_run_failure(&super::MlError::Ort(
- "invalid tensor shape".to_string()
- )));
- assert!(!is_execution_provider_run_failure(
- &super::MlError::Postprocess("WebGPU".to_string())
- ));
- }
-
- #[test]
- fn typed_run_errors_control_retryability() {
- let retryable =
- SessionRunError::retryable(super::MlError::Ort("non-finite output".to_string()));
- assert!(retryable.is_retryable());
-
- let terminal = SessionRunError::from(super::MlError::Ort(
- "WebGPU text from a typed terminal error".to_string(),
- ));
- assert!(!terminal.is_retryable());
- }
-
- #[test]
- fn detects_protobuf_parse_failure() {
- assert!(has_protobuf_parse_failure(&[super::MlError::Ort(
- "Load model failed:Protobuf parsing failed.".to_string(),
- )]));
- }
-
- #[test]
- fn ignores_other_onnx_errors() {
- assert!(!has_protobuf_parse_failure(&[super::MlError::Ort(
- "Load model failed: missing initializer".to_string(),
- )]));
- }
-
#[test]
fn reports_accelerated_provider_attempt_failures_with_model_context() {
let error = super::MlError::Ort("provider registration failed".to_string());
diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rs
index 98ba26b5f2..96a4536afd 100644
--- a/rust/crates/ml/src/onnx/providers.rs
+++ b/rust/crates/ml/src/onnx/providers.rs
@@ -19,7 +19,6 @@ use ort::ep::{
#[cfg(target_os = "android")]
use std::num::NonZeroUsize;
-use super::AccelerationValidation;
#[cfg(any(target_os = "ios", target_os = "macos"))]
use super::coreml_cache;
#[cfg(any(
@@ -40,6 +39,7 @@ use super::golden_test;
use super::model_file_label;
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
use super::webgpu;
+use super::{AccelerationValidation, session_load_error};
use crate::error::MlResult;
#[cfg(any(target_os = "ios", target_os = "macos"))]
@@ -207,7 +207,9 @@ pub(super) fn build_session(model_path: &str, attempt: ProviderAttempt) -> MlRes
}
builder = builder.with_execution_providers(attempt.providers)?;
- let session = builder.commit_from_file(model_path)?;
+ let session = builder
+ .commit_from_file(model_path)
+ .map_err(|error| session_load_error(model_path, error))?;
Ok(session)
}
diff --git a/rust/crates/ml/src/onnx/tensor.rs b/rust/crates/ml/src/onnx/tensor.rs
index 129ef19b6b..a3b39248ef 100644
--- a/rust/crates/ml/src/onnx/tensor.rs
+++ b/rust/crates/ml/src/onnx/tensor.rs
@@ -2,7 +2,7 @@ use std::cell::OnceCell;
use half::prelude::{HalfFloatSliceExt, HalfFloatVecExt};
use ort::{
- session::Session,
+ session::{Session, SessionInputs, SessionOutputs},
value::{Tensor, TensorElementType, TensorRef, ValueType},
};
@@ -74,16 +74,16 @@ pub(super) fn run_golden_tensor(
input_shape,
Vec::<half::f16>::from_f32_slice(data),
))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
} else {
let input_tensor =
TensorRef::<f32>::from_array_view((input_shape, data.as_slice()))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
}
}
golden_test::PreparedGoldenInput::I32(data) => {
let input_tensor = TensorRef::<i32>::from_array_view((input_shape, data.as_slice()))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
}
};
@@ -101,6 +101,15 @@ pub(super) fn run_golden_tensor(
}
}
+fn run_inference<'s, 'i, 'v: 'i, const N: usize>(
+ session: &'s mut Session,
+ inputs: impl Into<SessionInputs<'i, 'v, N>>,
+) -> SessionRunResult<SessionOutputs<'s>> {
+ session
+ .run(inputs)
+ .map_err(SessionRunError::from_inference_error)
+}
+
fn ensure_finite_f32(data: &[f32]) -> SessionRunResult<()> {
if data.iter().copied().all(f32::is_finite) {
return Ok(());
@@ -138,11 +147,11 @@ pub(crate) fn run_f32<const N: usize>(
let outputs = if session_expects_f16(session) {
let input_tensor =
TensorRef::<half::f16>::from_array_view((input_shape, input.f16_data()))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
} else {
let input_tensor =
TensorRef::<f32>::from_array_view((input_shape, input.f32_data.as_slice()))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
};
if outputs.len() == 0 {
@@ -174,11 +183,11 @@ pub(crate) fn with_prepared_float_output<const N: usize, T>(
let outputs = if session_expects_f16(session) {
let input_tensor =
TensorRef::<half::f16>::from_array_view((input_shape, input.f16_data()))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
} else {
let input_tensor =
TensorRef::<f32>::from_array_view((input_shape, input.f32_data.as_slice()))?;
- session.run(ort::inputs![input_tensor])?
+ run_inference(session, ort::inputs![input_tensor])?
};
if outputs.len() == 0 {
@@ -201,7 +210,7 @@ pub(crate) fn run_i32_f32<const N: usize>(
input_shape: [i64; N],
) -> SessionRunResult<(Vec<i64>, Vec<f32>)> {
let input_tensor = TensorRef::<i32>::from_array_view((input_shape, input))?;
- let outputs = session.run(ort::inputs![input_tensor])?;
+ let outputs = run_inference(session, ort::inputs![input_tensor])?;
if outputs.len() == 0 {
return Err(MlError::Ort("missing first output tensor".to_string()).into());
}
diff --git a/rust/crates/ml/tests/ml_indexing.rs b/rust/crates/ml/tests/ml_indexing.rs
index aefa062f87..d14b1419a6 100644
--- a/rust/crates/ml/tests/ml_indexing.rs
+++ b/rust/crates/ml/tests/ml_indexing.rs
@@ -26,5 +26,6 @@ fn run_ml_indexing_test(context: MlIndexingTestContext) -> Result<()> {
stats.print_if_requested();
}
- fail_if_any(failures, &stats)
+ fail_if_any(failures, &stats)?;
+ context.verify_corrupt_model()
}
diff --git a/rust/crates/ml/tests/support/ml_indexing.rs b/rust/crates/ml/tests/support/ml_indexing.rs
index b6ce3f78b8..08bb9e6350 100644
--- a/rust/crates/ml/tests/support/ml_indexing.rs
+++ b/rust/crates/ml/tests/support/ml_indexing.rs
@@ -1,6 +1,7 @@
use std::{
collections::{BTreeSet, HashMap, HashSet},
fs,
+ io::Write,
path::{Component, Path, PathBuf},
};
@@ -336,6 +337,41 @@ impl MlIndexingTestContext {
Ok(())
}
+ pub(crate) fn verify_corrupt_model(&self) -> Result<()> {
+ let mut model = tempfile::NamedTempFile::new()?;
+ model.write_all(b"not an ONNX protobuf")?;
+ let model_path = model.path().to_string_lossy().into_owned();
+ let unsupported = self.unsupported_decode_file_ids();
+ let image_path = self
+ .manifest
+ .files
+ .iter()
+ .zip(&self.fixture_paths)
+ .find_map(|(fixture, path)| {
+ let file_id = file_id_for_manifest_path(&fixture.path).ok()?;
+ (!unsupported.contains(&file_id)).then_some(path)
+ })
+ .context("find supported ML fixture")?;
+ let mut model_paths = self.model_paths.clone();
+ model_paths.face_detection = model_path.clone();
+
+ let error = analyze_image(AnalyzeImageRequest {
+ file_id: -1,
+ source: ImageSource::Path(image_path.to_string_lossy().into_owned()),
+ run_faces: true,
+ run_clip: false,
+ run_pets: false,
+ generate_face_crops: false,
+ model_paths,
+ })
+ .expect_err("invalid ONNX protobuf should fail");
+
+ match error {
+ MlError::CorruptModel(path) if path == model_path => Ok(()),
+ error => bail!("invalid ONNX protobuf returned {error:?}"),
+ }
+ }
+
fn manifest_file_ids(&self) -> Result<BTreeSet<String>> {
let mut ids = BTreeSet::new();
for file in &self.manifest.files {