π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Run the segmentation model on the GPU
laurenspriemd27f266ddb56
diff --git a/rust/bindings/frb/locker/src/api/document_scanner_api.rs b/rust/bindings/frb/locker/src/api/document_scanner_api.rsindex 4354013b8e..9bdae52253 100644--- a/rust/bindings/frb/locker/src/api/document_scanner_api.rs+++ b/rust/bindings/frb/locker/src/api/document_scanner_api.rs@@ -2,6 +2,7 @@ use std::any::Any;use std::panic::{AssertUnwindSafe, catch_unwind};use ente_assets::AssetStore;+use ente_ml::indexing::set_ml_execution_config;use ente_ml::scan;use flutter_rust_bridge::frb;@@ -34,14 +35,6 @@ pub struct RustPlaneLayout {pub uv_pixel_stride: i32,}-#[derive(Clone, Debug)]-pub struct RustScanOptions {- pub color_mode_override: Option<RustColorMode>,- pub max_pixels: Option<u32>,- /// Must be a multiple of 90.- pub rotation_degrees: i32,-}-#[derive(Clone, Debug)]pub struct RustReprocessOptions {/// Same space as `RustScanResult.quad`: the decoded source image.@@ -50,7 +43,6 @@ pub struct RustReprocessOptions {pub rotation_degrees: i32,pub color_mode: RustColorMode,pub max_pixels: Option<u32>,- pub jpeg_quality: Option<u8>,}#[derive(Clone, Debug)]@@ -92,6 +84,7 @@ impl ScannerSession {let store = AssetStore::new(assets_dir);let model_path = scan::ensure_segmentation_model(&store).await?;catch_panic(|| {+ set_ml_execution_config(true);let inner = scan::ScannerSession::new(&model_path.to_string_lossy())?;Ok(ScannerSession { inner })})@@ -142,12 +135,10 @@ impl ScannerSession {pub fn process_capture(&self,image_bytes: Vec<u8>,- options: RustScanOptions,+ max_pixels: Option<u32>,) -> Result<RustScanResult, RustScanError> {catch_panic(|| {- let result = self- .inner- .process_capture(&image_bytes, &to_scan_options(&options))?;+ let result = self.inner.process_capture(&image_bytes, max_pixels)?;Ok(to_api_scan_result(result))})}@@ -207,21 +198,12 @@ fn to_plane_layout(layout: RustPlaneLayout) -> scan::PlaneLayout {}}-fn to_scan_options(options: &RustScanOptions) -> scan::ScanOptions {- scan::ScanOptions {- color_mode_override: options.color_mode_override.map(to_color_mode),- max_pixels: options.max_pixels,- rotation_degrees: options.rotation_degrees,- }-}-fn to_reprocess_options(options: &RustReprocessOptions) -> scan::ReprocessOptions {scan::ReprocessOptions {quad: to_quad(options.quad),rotation_degrees: options.rotation_degrees,color_mode: to_color_mode(options.color_mode),max_pixels: options.max_pixels,- jpeg_quality: options.jpeg_quality,}}diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rsindex 33562fb114..a74901f10b 100644--- a/rust/crates/ml/src/onnx.rs+++ b/rust/crates/ml/src/onnx.rs@@ -22,7 +22,7 @@ pub(crate) use tensor::{with_prepared_float_output,};-use providers::{ExecutionProvider, ProviderPlan};+use providers::{ExecutionProvider, GoldenSelfTest, ProviderPlan};#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]pub(crate) struct ProviderUsage {@@ -49,6 +49,7 @@ impl ProviderUsage {#[derive(Debug)]pub(crate) struct OnnxSession {mode: ExecutionMode,+ golden: GoldenSelfTest,provider_plan: Option<ProviderPlan>,session: Option<Session>,}@@ -57,11 +58,20 @@ impl OnnxSession {pub(crate) fn new(mode: ExecutionMode) -> Self {Self {mode,+ golden: GoldenSelfTest::Required,provider_plan: None,session: None,}}+ /// Opt out of the golden self-test that otherwise gates accelerated+ /// providers. Only for models whose output is not indexed and so cannot+ /// silently poison stored data.+ pub(crate) fn without_golden_self_test(mut self) -> Self {+ self.golden = GoldenSelfTest::Skipped;+ self+ }+pub(crate) fn clear(&mut self) {self.provider_plan = None;self.session = None;@@ -111,11 +121,11 @@ impl OnnxSession {let provider_plan = self.provider_plan- .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path));+ .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path, self.golden));let model_name = model_file_label(model_path);log::info!("loading {model_name} with {:?} execution", self.mode);let started_at = std::time::Instant::now();- let session = build_next_session(model_path, provider_plan, model_namespace)?;+ let session = build_next_session(model_path, provider_plan, model_namespace, self.golden)?;let execution_provider = provider_plan.selected_provider().expect("successful session build must select an execution provider");@@ -204,21 +214,27 @@ fn build_next_session(model_path: &str,plan: &mut ProviderPlan,model_namespace: &str,+ golden: GoldenSelfTest,) -> MlResult<Session> {let result = providers::run_provider_plan(plan, |execution_provider| {let attempt = providers::provider_attempt(execution_provider, model_path, model_namespace);if attempt.uses_webgpu() {#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]{- return build_webgpu_session_with_canary(model_path, model_namespace, attempt)- .map_err(|error| format!("{error}"));+ return build_webgpu_session_with_canary(+ model_path,+ model_namespace,+ attempt,+ golden,+ )+ .map_err(|error| format!("{error}"));}#[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]unreachable!("WebGPU provider attempts are not constructed on this platform");}let coreml_cache_dir = attempt.coreml_cache_dir().map(Path::to_path_buf);- match build_and_validate_session(model_path, attempt) {+ match build_and_validate_session(model_path, attempt, golden) {Ok(session) => {if let Some(cache_dir) = coreml_cache_dir {coreml_cache::finalize(&cache_dir, model_path);@@ -255,8 +271,13 @@ fn build_next_session(}fn build_cpu_session(model_path: &str) -> MlResult<Session> {- let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, model_path);- build_next_session(model_path, &mut plan, "golden-tooling")+ let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, model_path, GoldenSelfTest::Required);+ build_next_session(+ model_path,+ &mut plan,+ "golden-tooling",+ GoldenSelfTest::Required,+ )}fn is_execution_provider_run_failure(error: &MlError) -> bool {@@ -282,6 +303,7 @@ fn is_execution_provider_run_failure(error: &MlError) -> bool {fn build_and_validate_session(model_path: &str,attempt: providers::ProviderAttempt,+ golden: GoldenSelfTest,) -> MlResult<Session> {#[cfg(any(target_os = "android",@@ -314,7 +336,7 @@ fn build_and_validate_session(};#[cfg(any(target_os = "ios", target_os = "macos"))]- if execution_provider == ExecutionProvider::CoreMl {+ if execution_provider == ExecutionProvider::CoreMl && golden == GoldenSelfTest::Required {run_session_self_test(model_path, &mut session, "CoreML")?;}@@ -328,6 +350,7 @@ fn build_webgpu_session_with_canary(model_path: &str,model_namespace: &str,attempt: providers::ProviderAttempt,+ golden: GoldenSelfTest,) -> MlResult<Session> {// Fail closed: without a durable failure record, a crash during the// attempt would go unnoticed and the crash loop protection would be lost.@@ -378,7 +401,9 @@ fn build_webgpu_session_with_canary(return Err(error);}};- run_session_self_test(model_path, &mut session, "WebGPU")?;+ if golden == GoldenSelfTest::Required {+ run_session_self_test(model_path, &mut session, "WebGPU")?;+ }canary.disarm();Ok(session)}diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rsindex 228e4752f5..971e8c3843 100644--- a/rust/crates/ml/src/onnx/providers.rs+++ b/rust/crates/ml/src/onnx/providers.rs@@ -44,6 +44,12 @@ use crate::error::MlResult;#[cfg(any(target_os = "ios", target_os = "macos"))]const ENABLE_PERSISTENT_COREML_CACHE: bool = true;+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(super) enum GoldenSelfTest {+ Required,+ Skipped,+}+#[derive(Clone, Copy, Debug, PartialEq, Eq)]pub(crate) enum ExecutionMode {PlatformDefault,@@ -69,9 +75,9 @@ pub(super) struct ProviderPlan {}impl ProviderPlan {- pub(super) fn new(mode: ExecutionMode, model_path: &str) -> Self {+ pub(super) fn new(mode: ExecutionMode, model_path: &str, golden: GoldenSelfTest) -> Self {let providers = match mode {- ExecutionMode::PlatformDefault => platform_default_providers(model_path),+ ExecutionMode::PlatformDefault => platform_default_providers(model_path, golden),ExecutionMode::CpuOnly => vec![ExecutionProvider::Cpu],};Self::from_providers(providers)@@ -215,9 +221,9 @@ pub(super) fn build_session(model_path: &str, attempt: ProviderAttempt) -> MlRes}#[cfg(any(target_os = "ios", target_os = "macos"))]-fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {+fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {let mut providers = Vec::new();- if golden_entry_required(model_path, "CoreML") {+ if accelerated_provider_allowed(model_path, "CoreML", golden) {providers.push(ExecutionProvider::CoreMl);}providers.push(ExecutionProvider::Cpu);@@ -225,9 +231,11 @@ fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {}#[cfg(target_os = "android")]-fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {+fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {let mut providers = Vec::new();- if webgpu::attempt_permitted(model_path) && golden_entry_required(model_path, "WebGPU") {+ if webgpu::attempt_permitted(model_path)+ && accelerated_provider_allowed(model_path, "WebGPU", golden)+ {providers.push(ExecutionProvider::WebGpu);}providers.push(ExecutionProvider::Xnnpack);@@ -236,9 +244,11 @@ fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {}#[cfg(any(target_os = "linux", target_os = "windows"))]-fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {+fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {let mut providers = Vec::new();- if webgpu::attempt_permitted(model_path) && golden_entry_required(model_path, "WebGPU") {+ if webgpu::attempt_permitted(model_path)+ && accelerated_provider_allowed(model_path, "WebGPU", golden)+ {providers.push(ExecutionProvider::WebGpu);}providers.push(ExecutionProvider::Cpu);@@ -252,7 +262,10 @@ fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {target_os = "macos",target_os = "windows")))]-fn platform_default_providers(_model_path: &str) -> Vec<ExecutionProvider> {+fn platform_default_providers(+ _model_path: &str,+ _golden: GoldenSelfTest,+) -> Vec<ExecutionProvider> {vec![ExecutionProvider::Cpu]}@@ -265,8 +278,12 @@ fn platform_default_providers(_model_path: &str) -> Vec<ExecutionProvider> {target_os = "macos",target_os = "windows"))]-fn golden_entry_required(model_path: &str, provider_label: &str) -> bool {- if golden_test::lookup(model_path).is_some() {+fn accelerated_provider_allowed(+ model_path: &str,+ provider_label: &str,+ golden: GoldenSelfTest,+) -> bool {+ if golden == GoldenSelfTest::Skipped || golden_test::lookup(model_path).is_some() {return true;}log::error!(@@ -358,7 +375,9 @@ fn xnnpack_attempt() -> ProviderAttempt {#[cfg(test)]mod tests {- use super::{ExecutionMode, ExecutionProvider, ProviderPlan, run_provider_plan};+ use super::{+ ExecutionMode, ExecutionProvider, GoldenSelfTest, ProviderPlan, run_provider_plan,+ };#[test]fn construction_falls_through_and_selects_the_successful_provider() {@@ -428,7 +447,11 @@ mod tests {#[test]fn exhausted_cpu_is_retained_as_the_only_construction_retry() {- let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, "model.onnx");+ let mut plan = ProviderPlan::new(+ ExecutionMode::CpuOnly,+ "model.onnx",+ GoldenSelfTest::Required,+ );let mut attempts = 0;let first = run_provider_plan::<(), _>(&mut plan, |provider| {diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rsindex 7182e9b8af..ad84804560 100644--- a/rust/crates/ml/src/scan/segmentation.rs+++ b/rust/crates/ml/src/scan/segmentation.rs@@ -1,42 +1,65 @@use std::sync::Mutex;-use ort::ep::CPU;-use ort::session::Session;-use ort::session::builder::GraphOptimizationLevel;use ort::value::TensorRef;use super::OpResult;use super::scanner::ScanError;use crate::cv;use crate::cv::image::ImageU8;+use crate::error::MlError;+use crate::onnx::{ExecutionMode, OnnxSession, SessionRunError};pub const MASK_SIDE: i32 = 256;+const MODEL_NAMESPACE: &str = "document-segmentation";+pub(crate) struct Segmenter {- session: Mutex<Session>,+ session: Mutex<OnnxSession>,+ model_path: String,}impl Segmenter {- /// CPU-only on purpose: downstream thresholds need deterministic output across devices.pub(crate) fn new(model_path: &str) -> Result<Self, ScanError> {- let build = || -> Result<Session, ort::Error> {- Session::builder()?- .with_optimization_level(GraphOptimizationLevel::All)?- .with_intra_threads(1)?- .with_inter_threads(1)?- .with_execution_providers([CPU::default().with_arena_allocator(true).build()])?- .commit_from_file(model_path)+ let segmenter = Self {+ session: Mutex::new(+ OnnxSession::new(ExecutionMode::PlatformDefault).without_golden_self_test(),+ ),+ model_path: model_path.to_string(),};- let session = build().map_err(|err| ScanError::ModelLoad(err.to_string()))?;- if session.inputs().len() != 1 {- return Err(ScanError::ModelLoad(format!(- "expected exactly 1 model input, got {}",- session.inputs().len()- )));- }- Ok(Self {- session: Mutex::new(session),- })+ let samples = (MASK_SIDE * MASK_SIDE * 3) as usize;+ segmenter+ .infer(vec![0.0f32; samples])+ .map_err(ScanError::ModelLoad)?;+ Ok(segmenter)+ }++ fn infer(&self, input: Vec<f32>) -> OpResult<Vec<f32>> {+ let side = MASK_SIDE as i64;+ let expected = (MASK_SIDE * MASK_SIDE) as usize;+ let mut guard = match self.session.lock() {+ Ok(guard) => guard,+ Err(poisoned) => poisoned.into_inner(),+ };+ let (data, _usage) = guard+ .run(&self.model_path, MODEL_NAMESPACE, |session| {+ let tensor =+ TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))?;+ let outputs = session.run(ort::inputs![tensor])?;+ if outputs.len() == 0 {+ return Err(SessionRunError::from(MlError::Ort(+ "model produced no outputs".to_string(),+ )));+ }+ let (shape, values) = outputs[0].try_extract_tensor::<f32>()?;+ if values.len() != expected {+ return Err(SessionRunError::from(MlError::Ort(format!(+ "unexpected model output shape {shape:?}"+ ))));+ }+ Ok(values.to_vec())+ })+ .map_err(|error| error.to_string())?;+ Ok(data)}pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {@@ -55,30 +78,7 @@ impl Segmenter {out[2] = (px[0] as f32 - 127.5) / 127.5;}- let side = MASK_SIDE as i64;- let mut session = match self.session.lock() {- Ok(guard) => guard,- Err(poisoned) => poisoned.into_inner(),- };- let mut run = || -> Result<Vec<f32>, String> {- let tensor = TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))- .map_err(|err| err.to_string())?;- let outputs = session- .run(ort::inputs![tensor])- .map_err(|err| err.to_string())?;- if outputs.len() == 0 {- return Err("model produced no outputs".to_string());- }- let (shape, data) = outputs[0]- .try_extract_tensor::<f32>()- .map_err(|err| err.to_string())?;- let expected = (MASK_SIDE * MASK_SIDE) as usize;- if data.len() != expected {- return Err(format!("unexpected model output shape {shape:?}"));- }- Ok(data.to_vec())- };- let data = run()?;+ let data = self.infer(input)?;Ok(data.iter()