Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Simplify ONNX provider configuration plumbing

laurenspriemc34af22e135f
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index fc57a26d1e..1ea0418ef5 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -312,7 +312,6 @@ fn build_next_session(
model_namespace,
attempt,
validation,
- gpu_options,
);
}
#[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]
@@ -320,7 +319,7 @@ fn build_next_session(
}
let coreml_cache_dir = attempt.coreml_cache_dir().map(Path::to_path_buf);
- match build_and_validate_session(model_path, attempt, validation, gpu_options) {
+ match build_and_validate_session(model_path, attempt, validation) {
Ok(session) => {
if let Some(cache_dir) = coreml_cache_dir {
coreml_cache::finalize(&cache_dir, model_path);
@@ -386,7 +385,6 @@ fn build_and_validate_session(
model_path: &str,
attempt: providers::ProviderAttempt,
_validation: AccelerationValidation,
- gpu_options: Option<&GpuOptions>,
) -> MlResult<Session> {
#[cfg(any(
target_os = "android",
@@ -397,7 +395,7 @@ fn build_and_validate_session(
))]
let execution_provider = attempt.execution_provider();
- let session = match providers::build_session(model_path, attempt, gpu_options) {
+ let session = match providers::build_session(model_path, attempt) {
Ok(session) => session,
Err(error) => {
#[cfg(any(
@@ -438,7 +436,6 @@ fn build_webgpu_session_with_canary(
model_namespace: &str,
attempt: providers::ProviderAttempt,
validation: AccelerationValidation,
- gpu_options: Option<&GpuOptions>,
) -> MlResult<LoadedSession> {
// Fail closed: without a durable failure record, a crash during the
// attempt would go unnoticed and the crash loop protection would be lost.
@@ -477,7 +474,7 @@ fn build_webgpu_session_with_canary(
}
}
}
- let mut session = match providers::build_session(model_path, attempt, gpu_options) {
+ let mut session = match providers::build_session(model_path, attempt) {
Ok(session) => session,
Err(error) => {
record_provider_attempt_failure(
@@ -640,6 +637,24 @@ fn model_file_label(model_path: &str) -> &str {
mod tests {
use super::{ExecutionMode, ExecutionProvider, OnnxSession, provider_attempt_failure_message};
+ #[test]
+ fn unvalidated_acceleration_does_not_enable_ocr_gpu_options() {
+ let indexing = OnnxSession::new("model.onnx", "indexing", ExecutionMode::PlatformDefault);
+ assert!(indexing.gpu_options.is_none());
+ assert_eq!(
+ indexing.validation,
+ super::AccelerationValidation::GoldenRequired
+ );
+
+ let scanner = indexing.with_unvalidated_acceleration();
+ assert!(scanner.gpu_options.is_none());
+ assert_eq!(
+ scanner.validation,
+ super::AccelerationValidation::Unvalidated
+ );
+ assert_eq!(scanner.mode, ExecutionMode::PlatformDefault);
+ }
+
fn first_run_canary(temp: &tempfile::TempDir) -> super::webgpu::ArmedCanary {
let model = temp.path().join("model.onnx");
super::webgpu::arm_canary(&model.to_string_lossy(), "scanner").unwrap()
diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rs
index 9187c029e3..d1eee6f0cd 100644
--- a/rust/crates/ml/src/onnx/providers.rs
+++ b/rust/crates/ml/src/onnx/providers.rs
@@ -123,6 +123,7 @@ pub(super) fn run_provider_plan<T, E>(
pub(super) struct ProviderAttempt {
providers: Vec<ExecutionProviderDispatch>,
disable_intra_op_spinning: bool,
+ disable_cpu_fallback: bool,
coreml_cache_dir: Option<PathBuf>,
execution_provider: ExecutionProvider,
}
@@ -132,6 +133,7 @@ impl ProviderAttempt {
Self {
providers: vec![CPU::default().with_arena_allocator(true).build()],
disable_intra_op_spinning: false,
+ disable_cpu_fallback: false,
coreml_cache_dir: None,
execution_provider: ExecutionProvider::Cpu,
}
@@ -152,6 +154,10 @@ pub(super) fn provider_attempt(
_model_namespace: &str,
_gpu_options: Option<&GpuOptions>,
) -> ProviderAttempt {
+ let _disable_cpu_fallback = _gpu_options.is_some_and(|options| {
+ provider == ExecutionProvider::WebGpu
+ || (provider == ExecutionProvider::CoreMl && !options.subgraphs)
+ });
match provider {
ExecutionProvider::Cpu => ProviderAttempt::cpu_only(),
#[cfg(any(target_os = "ios", target_os = "macos"))]
@@ -168,6 +174,7 @@ pub(super) fn provider_attempt(
]
},
disable_intra_op_spinning: false,
+ disable_cpu_fallback: _disable_cpu_fallback,
coreml_cache_dir,
execution_provider: ExecutionProvider::CoreMl,
}
@@ -180,6 +187,7 @@ pub(super) fn provider_attempt(
webgpu_attempt_providers()
},
disable_intra_op_spinning: true,
+ disable_cpu_fallback: _disable_cpu_fallback,
coreml_cache_dir: None,
execution_provider: ExecutionProvider::WebGpu,
},
@@ -189,20 +197,13 @@ pub(super) fn provider_attempt(
}
}
-pub(super) fn build_session(
- model_path: &str,
- attempt: ProviderAttempt,
- gpu_options: Option<&GpuOptions>,
-) -> MlResult<Session> {
+pub(super) fn build_session(model_path: &str, attempt: ProviderAttempt) -> MlResult<Session> {
let mut builder = Session::builder()?
.with_optimization_level(GraphOptimizationLevel::All)?
.with_intra_threads(1)?
.with_inter_threads(1)?;
- if let Some(options) = gpu_options
- && (attempt.execution_provider == ExecutionProvider::WebGpu
- || (attempt.execution_provider == ExecutionProvider::CoreMl && !options.subgraphs))
- {
+ if attempt.disable_cpu_fallback {
builder = builder.with_disable_cpu_fallback()?;
}
@@ -342,13 +343,8 @@ fn coreml_provider(
}
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
-fn webgpu_provider(prefer_nhwc: bool) -> ExecutionProviderDispatch {
- let layout = if prefer_nhwc {
- PreferredLayout::NHWC
- } else {
- PreferredLayout::NCHW
- };
- let provider = WebGPU::default().with_preferred_layout(layout);
+fn webgpu_provider() -> ExecutionProviderDispatch {
+ let provider = WebGPU::default().with_preferred_layout(PreferredLayout::NCHW);
#[cfg(any(target_os = "android", target_os = "linux"))]
let provider = provider.with_dawn_backend_type(DawnBackendType::Vulkan);
#[cfg(target_os = "windows")]
@@ -376,7 +372,7 @@ fn ocr_webgpu_provider(prefer_nhwc: bool) -> ExecutionProviderDispatch {
#[cfg(target_os = "android")]
fn webgpu_attempt_providers() -> Vec<ExecutionProviderDispatch> {
vec![
- webgpu_provider(false),
+ webgpu_provider(),
xnnpack_provider().fail_silently(),
CPU::default().with_arena_allocator(true).build(),
]
@@ -385,7 +381,7 @@ fn webgpu_attempt_providers() -> Vec<ExecutionProviderDispatch> {
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn webgpu_attempt_providers() -> Vec<ExecutionProviderDispatch> {
vec![
- webgpu_provider(false),
+ webgpu_provider(),
CPU::default().with_arena_allocator(true).build(),
]
}
@@ -407,6 +403,7 @@ fn xnnpack_attempt() -> ProviderAttempt {
CPU::default().with_arena_allocator(true).build(),
],
disable_intra_op_spinning: true,
+ disable_cpu_fallback: false,
coreml_cache_dir: None,
execution_provider: ExecutionProvider::Xnnpack,
}
@@ -415,9 +412,53 @@ fn xnnpack_attempt() -> ProviderAttempt {
#[cfg(test)]
mod tests {
use super::{
- AccelerationValidation, ExecutionMode, ExecutionProvider, ProviderPlan, run_provider_plan,
+ AccelerationValidation, ExecutionMode, ExecutionProvider, GpuOptions, ProviderPlan,
+ provider_attempt, run_provider_plan,
};
+ #[test]
+ fn ocr_options_preserve_default_and_cpu_fallback_configuration() {
+ let providers = [
+ ExecutionProvider::Cpu,
+ #[cfg(any(target_os = "ios", target_os = "macos"))]
+ ExecutionProvider::CoreMl,
+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
+ ExecutionProvider::WebGpu,
+ #[cfg(target_os = "android")]
+ ExecutionProvider::Xnnpack,
+ ];
+ for provider in providers {
+ let default = provider_attempt(provider, "missing-model.onnx", "test", None);
+ assert!(!default.disable_cpu_fallback);
+ assert!(
+ default
+ .providers
+ .iter()
+ .any(|dispatch| dispatch.downcast_ref::<ort::ep::CPU>().is_some())
+ );
+ for subgraphs in [false, true] {
+ let options = GpuOptions {
+ subgraphs,
+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
+ prefer_nhwc: false,
+ };
+ let ocr = provider_attempt(provider, "missing-model.onnx", "test", Some(&options));
+ let (strict, provider_count) = match provider {
+ ExecutionProvider::CoreMl => (!subgraphs, 1),
+ ExecutionProvider::WebGpu => (true, 1),
+ ExecutionProvider::Cpu => (false, 1),
+ ExecutionProvider::Xnnpack => (false, 2),
+ };
+ assert_eq!(ocr.disable_cpu_fallback, strict);
+ assert_eq!(ocr.providers.len(), provider_count);
+ assert_eq!(
+ ocr.disable_intra_op_spinning,
+ default.disable_intra_op_spinning
+ );
+ }
+ }
+ }
+
#[test]
fn gpu_preferred_falls_back_directly_to_cpu() {
let mut plan = ProviderPlan::new(