π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Clean up ONNX acceleration validation
laurenspriemb733d4ae983c
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rsindex 4b115050de..1204669e53 100644--- a/rust/crates/ml/src/onnx.rs+++ b/rust/crates/ml/src/onnx.rs@@ -22,7 +22,13 @@ pub(crate) use tensor::{with_prepared_float_output,};-use providers::{ExecutionProvider, GoldenSelfTest, ProviderPlan};+use providers::{ExecutionProvider, ProviderPlan};++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum AccelerationValidation {+ GoldenRequired,+ Unvalidated,+}#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]pub(crate) struct ProviderUsage {@@ -49,7 +55,7 @@ impl ProviderUsage {#[derive(Debug)]pub(crate) struct OnnxSession {mode: ExecutionMode,- golden: GoldenSelfTest,+ validation: AccelerationValidation,provider_plan: Option<ProviderPlan>,session: Option<Session>,first_run_canary: Option<webgpu::ArmedCanary>,@@ -59,18 +65,17 @@ impl OnnxSession {pub(crate) fn new(mode: ExecutionMode) -> Self {Self {mode,- golden: GoldenSelfTest::Required,+ validation: AccelerationValidation::GoldenRequired,provider_plan: None,session: None,first_run_canary: None,}}- /// Opt out of the golden self-test that otherwise gates accelerated- /// providers. Only for models whose output is not indexed and so cannot+ /// Only for models whose output is not indexed and so cannot/// silently poison stored data.- pub(crate) fn without_golden_self_test(mut self) -> Self {- self.golden = GoldenSelfTest::Skipped;+ pub(crate) fn with_unvalidated_acceleration(mut self) -> Self {+ self.validation = AccelerationValidation::Unvalidated;self}@@ -129,11 +134,12 @@ impl OnnxSession {let provider_plan = self.provider_plan- .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path, self.golden));+ .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path, self.validation));let model_name = model_file_label(model_path);log::info!("loading {model_name} with {:?} execution", self.mode);let started_at = std::time::Instant::now();- let loaded = build_next_session(model_path, provider_plan, model_namespace, self.golden)?;+ let loaded =+ build_next_session(model_path, provider_plan, model_namespace, self.validation)?;let execution_provider = provider_plan.selected_provider().expect("successful session build must select an execution provider");@@ -247,27 +253,26 @@ fn build_next_session(model_path: &str,plan: &mut ProviderPlan,model_namespace: &str,- golden: GoldenSelfTest,+ validation: AccelerationValidation,) -> MlResult<LoadedSession> {let result = providers::run_provider_plan(plan, |execution_provider| {let attempt = providers::provider_attempt(execution_provider, model_path, model_namespace);- if attempt.uses_webgpu() {+ if attempt.execution_provider() == ExecutionProvider::WebGpu {#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]{return build_webgpu_session_with_canary(model_path,model_namespace,attempt,- golden,- )- .map_err(|error| format!("{error}"));+ validation,+ );}#[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]unreachable!("WebGPU provider attempts are not constructed on this platform");}let coreml_cache_dir = attempt.coreml_cache_dir().map(Path::to_path_buf);- match build_and_validate_session(model_path, attempt, golden) {+ match build_and_validate_session(model_path, attempt, validation) {Ok(session) => {if let Some(cache_dir) = coreml_cache_dir {coreml_cache::finalize(&cache_dir, model_path);@@ -283,7 +288,7 @@ fn build_next_session(model_file_label(model_path));}- Err(format!("{error}"))+ Err(error)}}});@@ -299,17 +304,25 @@ fn build_next_session(Err(MlError::Ort(format!("failed to create ONNX session for model '{model_path}' across EP fallbacks: {}",- errors.join(" | ")+ errors+ .iter()+ .map(ToString::to_string)+ .collect::<Vec<_>>()+ .join(" | "))))}fn build_cpu_session(model_path: &str) -> MlResult<Session> {- let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, model_path, GoldenSelfTest::Required);+ let mut plan = ProviderPlan::new(+ ExecutionMode::CpuOnly,+ model_path,+ AccelerationValidation::GoldenRequired,+ );build_next_session(model_path,&mut plan,"golden-tooling",- GoldenSelfTest::Required,+ AccelerationValidation::GoldenRequired,).map(|loaded| loaded.session)}@@ -334,14 +347,10 @@ fn is_execution_provider_run_failure(error: &MlError) -> bool {// A CoreML self-test failure is treated as construction failure so the caller// invalidates the possibly corrupt persistent cache. WebGPU validation stays// inside its crash-canary window instead.-#[cfg_attr(- not(any(target_os = "ios", target_os = "macos")),- allow(unused_variables)-)]fn build_and_validate_session(model_path: &str,attempt: providers::ProviderAttempt,- golden: GoldenSelfTest,+ _validation: AccelerationValidation,) -> MlResult<Session> {#[cfg(any(target_os = "android",@@ -374,7 +383,9 @@ fn build_and_validate_session(};#[cfg(any(target_os = "ios", target_os = "macos"))]- if execution_provider == ExecutionProvider::CoreMl && golden == GoldenSelfTest::Required {+ if execution_provider == ExecutionProvider::CoreMl+ && _validation == AccelerationValidation::GoldenRequired+ {run_session_self_test(model_path, &mut session, "CoreML")?;}@@ -388,7 +399,7 @@ fn build_webgpu_session_with_canary(model_path: &str,model_namespace: &str,attempt: providers::ProviderAttempt,- golden: GoldenSelfTest,+ validation: AccelerationValidation,) -> MlResult<LoadedSession> {// Fail closed: without a durable failure record, a crash during the// attempt would go unnoticed and the crash loop protection would be lost.@@ -439,7 +450,7 @@ fn build_webgpu_session_with_canary(return Err(error);}};- let first_run_canary = if golden == GoldenSelfTest::Required {+ let first_run_canary = if validation == AccelerationValidation::GoldenRequired {run_session_self_test(model_path, &mut session, "WebGPU")?;canary.disarm();None@@ -586,11 +597,15 @@ fn model_file_label(model_path: &str) -> &str {.unwrap_or(model_path)}-fn has_protobuf_parse_failure(errors: &[String]) -> bool {+fn has_protobuf_parse_failure(errors: &[MlError]) -> bool {errors.iter().any(|error| {- error- .to_ascii_lowercase()- .contains("protobuf parsing failed")+ matches!(+ error,+ MlError::Ort(message)+ if message+ .to_ascii_lowercase()+ .contains("protobuf parsing failed")+ )})}@@ -659,15 +674,15 @@ mod tests {#[test]fn detects_protobuf_parse_failure() {- assert!(has_protobuf_parse_failure(&[String::from(- "Load model failed:Protobuf parsing failed.",+ assert!(has_protobuf_parse_failure(&[super::MlError::Ort(+ "Load model failed:Protobuf parsing failed.".to_string(),)]));}#[test]fn ignores_other_onnx_errors() {- assert!(!has_protobuf_parse_failure(&[String::from(- "Load model failed: missing initializer",+ assert!(!has_protobuf_parse_failure(&[super::MlError::Ort(+ "Load model failed: missing initializer".to_string(),)]));}diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rsindex 971e8c3843..98ba26b5f2 100644--- a/rust/crates/ml/src/onnx/providers.rs+++ b/rust/crates/ml/src/onnx/providers.rs@@ -19,6 +19,7 @@ use ort::ep::{#[cfg(target_os = "android")]use std::num::NonZeroUsize;+use super::AccelerationValidation;#[cfg(any(target_os = "ios", target_os = "macos"))]use super::coreml_cache;#[cfg(any(@@ -44,12 +45,6 @@ use crate::error::MlResult;#[cfg(any(target_os = "ios", target_os = "macos"))]const ENABLE_PERSISTENT_COREML_CACHE: bool = true;-#[derive(Clone, Copy, Debug, PartialEq, Eq)]-pub(super) enum GoldenSelfTest {- Required,- Skipped,-}-#[derive(Clone, Copy, Debug, PartialEq, Eq)]pub(crate) enum ExecutionMode {PlatformDefault,@@ -75,9 +70,13 @@ pub(super) struct ProviderPlan {}impl ProviderPlan {- pub(super) fn new(mode: ExecutionMode, model_path: &str, golden: GoldenSelfTest) -> Self {+ pub(super) fn new(+ mode: ExecutionMode,+ model_path: &str,+ validation: AccelerationValidation,+ ) -> Self {let providers = match mode {- ExecutionMode::PlatformDefault => platform_default_providers(model_path, golden),+ ExecutionMode::PlatformDefault => platform_default_providers(model_path, validation),ExecutionMode::CpuOnly => vec![ExecutionProvider::Cpu],};Self::from_providers(providers)@@ -140,7 +139,6 @@ pub(super) struct ProviderAttempt {providers: Vec<ExecutionProviderDispatch>,disable_intra_op_spinning: bool,coreml_cache_dir: Option<PathBuf>,- uses_webgpu: bool,execution_provider: ExecutionProvider,}@@ -150,7 +148,6 @@ impl ProviderAttempt {providers: vec![CPU::default().with_arena_allocator(true).build()],disable_intra_op_spinning: false,coreml_cache_dir: None,- uses_webgpu: false,execution_provider: ExecutionProvider::Cpu,}}@@ -162,10 +159,6 @@ impl ProviderAttempt {pub(super) fn execution_provider(&self) -> ExecutionProvider {self.execution_provider}-- pub(super) fn uses_webgpu(&self) -> bool {- self.uses_webgpu- }}pub(super) fn provider_attempt(@@ -186,7 +179,6 @@ pub(super) fn provider_attempt(],disable_intra_op_spinning: false,coreml_cache_dir,- uses_webgpu: false,execution_provider: ExecutionProvider::CoreMl,}}@@ -195,7 +187,6 @@ pub(super) fn provider_attempt(providers: webgpu_attempt_providers(),disable_intra_op_spinning: true,coreml_cache_dir: None,- uses_webgpu: true,execution_provider: ExecutionProvider::WebGpu,},#[cfg(target_os = "android")]@@ -221,9 +212,12 @@ pub(super) fn build_session(model_path: &str, attempt: ProviderAttempt) -> MlRes}#[cfg(any(target_os = "ios", target_os = "macos"))]-fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {+fn platform_default_providers(+ model_path: &str,+ validation: AccelerationValidation,+) -> Vec<ExecutionProvider> {let mut providers = Vec::new();- if accelerated_provider_allowed(model_path, "CoreML", golden) {+ if accelerated_provider_allowed(model_path, "CoreML", validation) {providers.push(ExecutionProvider::CoreMl);}providers.push(ExecutionProvider::Cpu);@@ -231,10 +225,13 @@ fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<E}#[cfg(target_os = "android")]-fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {+fn platform_default_providers(+ model_path: &str,+ validation: AccelerationValidation,+) -> Vec<ExecutionProvider> {let mut providers = Vec::new();if webgpu::attempt_permitted(model_path)- && accelerated_provider_allowed(model_path, "WebGPU", golden)+ && accelerated_provider_allowed(model_path, "WebGPU", validation){providers.push(ExecutionProvider::WebGpu);}@@ -244,10 +241,13 @@ fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<E}#[cfg(any(target_os = "linux", target_os = "windows"))]-fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {+fn platform_default_providers(+ model_path: &str,+ validation: AccelerationValidation,+) -> Vec<ExecutionProvider> {let mut providers = Vec::new();if webgpu::attempt_permitted(model_path)- && accelerated_provider_allowed(model_path, "WebGPU", golden)+ && accelerated_provider_allowed(model_path, "WebGPU", validation){providers.push(ExecutionProvider::WebGpu);}@@ -264,7 +264,7 @@ fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<E)))]fn platform_default_providers(_model_path: &str,- _golden: GoldenSelfTest,+ _validation: AccelerationValidation,) -> Vec<ExecutionProvider> {vec![ExecutionProvider::Cpu]}@@ -281,9 +281,11 @@ fn platform_default_providers(fn accelerated_provider_allowed(model_path: &str,provider_label: &str,- golden: GoldenSelfTest,+ validation: AccelerationValidation,) -> bool {- if golden == GoldenSelfTest::Skipped || golden_test::lookup(model_path).is_some() {+ if validation == AccelerationValidation::Unvalidated+ || golden_test::lookup(model_path).is_some()+ {return true;}log::error!(@@ -368,7 +370,6 @@ fn xnnpack_attempt() -> ProviderAttempt {],disable_intra_op_spinning: true,coreml_cache_dir: None,- uses_webgpu: false,execution_provider: ExecutionProvider::Xnnpack,}}@@ -376,7 +377,7 @@ fn xnnpack_attempt() -> ProviderAttempt {#[cfg(test)]mod tests {use super::{- ExecutionMode, ExecutionProvider, GoldenSelfTest, ProviderPlan, run_provider_plan,+ AccelerationValidation, ExecutionMode, ExecutionProvider, ProviderPlan, run_provider_plan,};#[test]@@ -450,7 +451,7 @@ mod tests {let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly,"model.onnx",- GoldenSelfTest::Required,+ AccelerationValidation::GoldenRequired,);let mut attempts = 0;diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rsindex ad84804560..7e628a1786 100644--- a/rust/crates/ml/src/scan/segmentation.rs+++ b/rust/crates/ml/src/scan/segmentation.rs@@ -1,13 +1,11 @@use std::sync::Mutex;-use ort::value::TensorRef;-use super::OpResult;use super::scanner::ScanError;use crate::cv;use crate::cv::image::ImageU8;use crate::error::MlError;-use crate::onnx::{ExecutionMode, OnnxSession, SessionRunError};+use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};pub const MASK_SIDE: i32 = 256;@@ -22,7 +20,7 @@ impl Segmenter {pub(crate) fn new(model_path: &str) -> Result<Self, ScanError> {let segmenter = Self {session: Mutex::new(- OnnxSession::new(ExecutionMode::PlatformDefault).without_golden_self_test(),+ OnnxSession::new(ExecutionMode::PlatformDefault).with_unvalidated_acceleration(),),model_path: model_path.to_string(),};@@ -36,27 +34,20 @@ impl Segmenter {fn infer(&self, input: Vec<f32>) -> OpResult<Vec<f32>> {let side = MASK_SIDE as i64;let expected = (MASK_SIDE * MASK_SIDE) as usize;+ let input = PreparedF32Input::new(input);let mut guard = match self.session.lock() {Ok(guard) => guard,Err(poisoned) => poisoned.into_inner(),};let (data, _usage) = guard.run(&self.model_path, MODEL_NAMESPACE, |session| {- let tensor =- TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))?;- let outputs = session.run(ort::inputs![tensor])?;- if outputs.len() == 0 {- return Err(SessionRunError::from(MlError::Ort(- "model produced no outputs".to_string(),- )));- }- let (shape, values) = outputs[0].try_extract_tensor::<f32>()?;+ let (shape, values) = run_f32(session, &input, [1i64, side, side, 3])?;if values.len() != expected {return Err(SessionRunError::from(MlError::Ort(format!("unexpected model output shape {shape:?}"))));}- Ok(values.to_vec())+ Ok(values)}).map_err(|error| error.to_string())?;Ok(data)