Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Run the segmentation model on the GPU

laurenspriemd27f266ddb56
diff --git a/rust/bindings/frb/locker/src/api/document_scanner_api.rs b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
index 4354013b8e..9bdae52253 100644
--- a/rust/bindings/frb/locker/src/api/document_scanner_api.rs
+++ b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
@@ -2,6 +2,7 @@ use std::any::Any;
use std::panic::{AssertUnwindSafe, catch_unwind};
use ente_assets::AssetStore;
+use ente_ml::indexing::set_ml_execution_config;
use ente_ml::scan;
use flutter_rust_bridge::frb;
@@ -34,14 +35,6 @@ pub struct RustPlaneLayout {
pub uv_pixel_stride: i32,
}
-#[derive(Clone, Debug)]
-pub struct RustScanOptions {
- pub color_mode_override: Option<RustColorMode>,
- pub max_pixels: Option<u32>,
- /// Must be a multiple of 90.
- pub rotation_degrees: i32,
-}
-
#[derive(Clone, Debug)]
pub struct RustReprocessOptions {
/// Same space as `RustScanResult.quad`: the decoded source image.
@@ -50,7 +43,6 @@ pub struct RustReprocessOptions {
pub rotation_degrees: i32,
pub color_mode: RustColorMode,
pub max_pixels: Option<u32>,
- pub jpeg_quality: Option<u8>,
}
#[derive(Clone, Debug)]
@@ -92,6 +84,7 @@ impl ScannerSession {
let store = AssetStore::new(assets_dir);
let model_path = scan::ensure_segmentation_model(&store).await?;
catch_panic(|| {
+ set_ml_execution_config(true);
let inner = scan::ScannerSession::new(&model_path.to_string_lossy())?;
Ok(ScannerSession { inner })
})
@@ -142,12 +135,10 @@ impl ScannerSession {
pub fn process_capture(
&self,
image_bytes: Vec<u8>,
- options: RustScanOptions,
+ max_pixels: Option<u32>,
) -> Result<RustScanResult, RustScanError> {
catch_panic(|| {
- let result = self
- .inner
- .process_capture(&image_bytes, &to_scan_options(&options))?;
+ let result = self.inner.process_capture(&image_bytes, max_pixels)?;
Ok(to_api_scan_result(result))
})
}
@@ -207,21 +198,12 @@ fn to_plane_layout(layout: RustPlaneLayout) -> scan::PlaneLayout {
}
}
-fn to_scan_options(options: &RustScanOptions) -> scan::ScanOptions {
- scan::ScanOptions {
- color_mode_override: options.color_mode_override.map(to_color_mode),
- max_pixels: options.max_pixels,
- rotation_degrees: options.rotation_degrees,
- }
-}
-
fn to_reprocess_options(options: &RustReprocessOptions) -> scan::ReprocessOptions {
scan::ReprocessOptions {
quad: to_quad(options.quad),
rotation_degrees: options.rotation_degrees,
color_mode: to_color_mode(options.color_mode),
max_pixels: options.max_pixels,
- jpeg_quality: options.jpeg_quality,
}
}
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index 33562fb114..a74901f10b 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -22,7 +22,7 @@ pub(crate) use tensor::{
with_prepared_float_output,
};
-use providers::{ExecutionProvider, ProviderPlan};
+use providers::{ExecutionProvider, GoldenSelfTest, ProviderPlan};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct ProviderUsage {
@@ -49,6 +49,7 @@ impl ProviderUsage {
#[derive(Debug)]
pub(crate) struct OnnxSession {
mode: ExecutionMode,
+ golden: GoldenSelfTest,
provider_plan: Option<ProviderPlan>,
session: Option<Session>,
}
@@ -57,11 +58,20 @@ impl OnnxSession {
pub(crate) fn new(mode: ExecutionMode) -> Self {
Self {
mode,
+ golden: GoldenSelfTest::Required,
provider_plan: None,
session: None,
}
}
+ /// Opt out of the golden self-test that otherwise gates accelerated
+ /// providers. Only for models whose output is not indexed and so cannot
+ /// silently poison stored data.
+ pub(crate) fn without_golden_self_test(mut self) -> Self {
+ self.golden = GoldenSelfTest::Skipped;
+ self
+ }
+
pub(crate) fn clear(&mut self) {
self.provider_plan = None;
self.session = None;
@@ -111,11 +121,11 @@ impl OnnxSession {
let provider_plan = self
.provider_plan
- .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path));
+ .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path, self.golden));
let model_name = model_file_label(model_path);
log::info!("loading {model_name} with {:?} execution", self.mode);
let started_at = std::time::Instant::now();
- let session = build_next_session(model_path, provider_plan, model_namespace)?;
+ let session = build_next_session(model_path, provider_plan, model_namespace, self.golden)?;
let execution_provider = provider_plan
.selected_provider()
.expect("successful session build must select an execution provider");
@@ -204,21 +214,27 @@ fn build_next_session(
model_path: &str,
plan: &mut ProviderPlan,
model_namespace: &str,
+ golden: GoldenSelfTest,
) -> MlResult<Session> {
let result = providers::run_provider_plan(plan, |execution_provider| {
let attempt = providers::provider_attempt(execution_provider, model_path, model_namespace);
if attempt.uses_webgpu() {
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
{
- return build_webgpu_session_with_canary(model_path, model_namespace, attempt)
- .map_err(|error| format!("{error}"));
+ return build_webgpu_session_with_canary(
+ model_path,
+ model_namespace,
+ attempt,
+ golden,
+ )
+ .map_err(|error| format!("{error}"));
}
#[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]
unreachable!("WebGPU provider attempts are not constructed on this platform");
}
let coreml_cache_dir = attempt.coreml_cache_dir().map(Path::to_path_buf);
- match build_and_validate_session(model_path, attempt) {
+ match build_and_validate_session(model_path, attempt, golden) {
Ok(session) => {
if let Some(cache_dir) = coreml_cache_dir {
coreml_cache::finalize(&cache_dir, model_path);
@@ -255,8 +271,13 @@ fn build_next_session(
}
fn build_cpu_session(model_path: &str) -> MlResult<Session> {
- let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, model_path);
- build_next_session(model_path, &mut plan, "golden-tooling")
+ let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, model_path, GoldenSelfTest::Required);
+ build_next_session(
+ model_path,
+ &mut plan,
+ "golden-tooling",
+ GoldenSelfTest::Required,
+ )
}
fn is_execution_provider_run_failure(error: &MlError) -> bool {
@@ -282,6 +303,7 @@ fn is_execution_provider_run_failure(error: &MlError) -> bool {
fn build_and_validate_session(
model_path: &str,
attempt: providers::ProviderAttempt,
+ golden: GoldenSelfTest,
) -> MlResult<Session> {
#[cfg(any(
target_os = "android",
@@ -314,7 +336,7 @@ fn build_and_validate_session(
};
#[cfg(any(target_os = "ios", target_os = "macos"))]
- if execution_provider == ExecutionProvider::CoreMl {
+ if execution_provider == ExecutionProvider::CoreMl && golden == GoldenSelfTest::Required {
run_session_self_test(model_path, &mut session, "CoreML")?;
}
@@ -328,6 +350,7 @@ fn build_webgpu_session_with_canary(
model_path: &str,
model_namespace: &str,
attempt: providers::ProviderAttempt,
+ golden: GoldenSelfTest,
) -> MlResult<Session> {
// Fail closed: without a durable failure record, a crash during the
// attempt would go unnoticed and the crash loop protection would be lost.
@@ -378,7 +401,9 @@ fn build_webgpu_session_with_canary(
return Err(error);
}
};
- run_session_self_test(model_path, &mut session, "WebGPU")?;
+ if golden == GoldenSelfTest::Required {
+ run_session_self_test(model_path, &mut session, "WebGPU")?;
+ }
canary.disarm();
Ok(session)
}
diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rs
index 228e4752f5..971e8c3843 100644
--- a/rust/crates/ml/src/onnx/providers.rs
+++ b/rust/crates/ml/src/onnx/providers.rs
@@ -44,6 +44,12 @@ use crate::error::MlResult;
#[cfg(any(target_os = "ios", target_os = "macos"))]
const ENABLE_PERSISTENT_COREML_CACHE: bool = true;
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum GoldenSelfTest {
+ Required,
+ Skipped,
+}
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ExecutionMode {
PlatformDefault,
@@ -69,9 +75,9 @@ pub(super) struct ProviderPlan {
}
impl ProviderPlan {
- pub(super) fn new(mode: ExecutionMode, model_path: &str) -> Self {
+ pub(super) fn new(mode: ExecutionMode, model_path: &str, golden: GoldenSelfTest) -> Self {
let providers = match mode {
- ExecutionMode::PlatformDefault => platform_default_providers(model_path),
+ ExecutionMode::PlatformDefault => platform_default_providers(model_path, golden),
ExecutionMode::CpuOnly => vec![ExecutionProvider::Cpu],
};
Self::from_providers(providers)
@@ -215,9 +221,9 @@ pub(super) fn build_session(model_path: &str, attempt: ProviderAttempt) -> MlRes
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
-fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {
+fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {
let mut providers = Vec::new();
- if golden_entry_required(model_path, "CoreML") {
+ if accelerated_provider_allowed(model_path, "CoreML", golden) {
providers.push(ExecutionProvider::CoreMl);
}
providers.push(ExecutionProvider::Cpu);
@@ -225,9 +231,11 @@ fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {
}
#[cfg(target_os = "android")]
-fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {
+fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {
let mut providers = Vec::new();
- if webgpu::attempt_permitted(model_path) && golden_entry_required(model_path, "WebGPU") {
+ if webgpu::attempt_permitted(model_path)
+ && accelerated_provider_allowed(model_path, "WebGPU", golden)
+ {
providers.push(ExecutionProvider::WebGpu);
}
providers.push(ExecutionProvider::Xnnpack);
@@ -236,9 +244,11 @@ fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
-fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {
+fn platform_default_providers(model_path: &str, golden: GoldenSelfTest) -> Vec<ExecutionProvider> {
let mut providers = Vec::new();
- if webgpu::attempt_permitted(model_path) && golden_entry_required(model_path, "WebGPU") {
+ if webgpu::attempt_permitted(model_path)
+ && accelerated_provider_allowed(model_path, "WebGPU", golden)
+ {
providers.push(ExecutionProvider::WebGpu);
}
providers.push(ExecutionProvider::Cpu);
@@ -252,7 +262,10 @@ fn platform_default_providers(model_path: &str) -> Vec<ExecutionProvider> {
target_os = "macos",
target_os = "windows"
)))]
-fn platform_default_providers(_model_path: &str) -> Vec<ExecutionProvider> {
+fn platform_default_providers(
+ _model_path: &str,
+ _golden: GoldenSelfTest,
+) -> Vec<ExecutionProvider> {
vec![ExecutionProvider::Cpu]
}
@@ -265,8 +278,12 @@ fn platform_default_providers(_model_path: &str) -> Vec<ExecutionProvider> {
target_os = "macos",
target_os = "windows"
))]
-fn golden_entry_required(model_path: &str, provider_label: &str) -> bool {
- if golden_test::lookup(model_path).is_some() {
+fn accelerated_provider_allowed(
+ model_path: &str,
+ provider_label: &str,
+ golden: GoldenSelfTest,
+) -> bool {
+ if golden == GoldenSelfTest::Skipped || golden_test::lookup(model_path).is_some() {
return true;
}
log::error!(
@@ -358,7 +375,9 @@ fn xnnpack_attempt() -> ProviderAttempt {
#[cfg(test)]
mod tests {
- use super::{ExecutionMode, ExecutionProvider, ProviderPlan, run_provider_plan};
+ use super::{
+ ExecutionMode, ExecutionProvider, GoldenSelfTest, ProviderPlan, run_provider_plan,
+ };
#[test]
fn construction_falls_through_and_selects_the_successful_provider() {
@@ -428,7 +447,11 @@ mod tests {
#[test]
fn exhausted_cpu_is_retained_as_the_only_construction_retry() {
- let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, "model.onnx");
+ let mut plan = ProviderPlan::new(
+ ExecutionMode::CpuOnly,
+ "model.onnx",
+ GoldenSelfTest::Required,
+ );
let mut attempts = 0;
let first = run_provider_plan::<(), _>(&mut plan, |provider| {
diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rs
index 7182e9b8af..ad84804560 100644
--- a/rust/crates/ml/src/scan/segmentation.rs
+++ b/rust/crates/ml/src/scan/segmentation.rs
@@ -1,42 +1,65 @@
use std::sync::Mutex;
-use ort::ep::CPU;
-use ort::session::Session;
-use ort::session::builder::GraphOptimizationLevel;
use ort::value::TensorRef;
use super::OpResult;
use super::scanner::ScanError;
use crate::cv;
use crate::cv::image::ImageU8;
+use crate::error::MlError;
+use crate::onnx::{ExecutionMode, OnnxSession, SessionRunError};
pub const MASK_SIDE: i32 = 256;
+const MODEL_NAMESPACE: &str = "document-segmentation";
+
pub(crate) struct Segmenter {
- session: Mutex<Session>,
+ session: Mutex<OnnxSession>,
+ model_path: String,
}
impl Segmenter {
- /// CPU-only on purpose: downstream thresholds need deterministic output across devices.
pub(crate) fn new(model_path: &str) -> Result<Self, ScanError> {
- let build = || -> Result<Session, ort::Error> {
- Session::builder()?
- .with_optimization_level(GraphOptimizationLevel::All)?
- .with_intra_threads(1)?
- .with_inter_threads(1)?
- .with_execution_providers([CPU::default().with_arena_allocator(true).build()])?
- .commit_from_file(model_path)
+ let segmenter = Self {
+ session: Mutex::new(
+ OnnxSession::new(ExecutionMode::PlatformDefault).without_golden_self_test(),
+ ),
+ model_path: model_path.to_string(),
};
- let session = build().map_err(|err| ScanError::ModelLoad(err.to_string()))?;
- if session.inputs().len() != 1 {
- return Err(ScanError::ModelLoad(format!(
- "expected exactly 1 model input, got {}",
- session.inputs().len()
- )));
- }
- Ok(Self {
- session: Mutex::new(session),
- })
+ let samples = (MASK_SIDE * MASK_SIDE * 3) as usize;
+ segmenter
+ .infer(vec![0.0f32; samples])
+ .map_err(ScanError::ModelLoad)?;
+ Ok(segmenter)
+ }
+
+ fn infer(&self, input: Vec<f32>) -> OpResult<Vec<f32>> {
+ let side = MASK_SIDE as i64;
+ let expected = (MASK_SIDE * MASK_SIDE) as usize;
+ let mut guard = match self.session.lock() {
+ Ok(guard) => guard,
+ Err(poisoned) => poisoned.into_inner(),
+ };
+ let (data, _usage) = guard
+ .run(&self.model_path, MODEL_NAMESPACE, |session| {
+ let tensor =
+ TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))?;
+ let outputs = session.run(ort::inputs![tensor])?;
+ if outputs.len() == 0 {
+ return Err(SessionRunError::from(MlError::Ort(
+ "model produced no outputs".to_string(),
+ )));
+ }
+ let (shape, values) = outputs[0].try_extract_tensor::<f32>()?;
+ if values.len() != expected {
+ return Err(SessionRunError::from(MlError::Ort(format!(
+ "unexpected model output shape {shape:?}"
+ ))));
+ }
+ Ok(values.to_vec())
+ })
+ .map_err(|error| error.to_string())?;
+ Ok(data)
}
pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {
@@ -55,30 +78,7 @@ impl Segmenter {
out[2] = (px[0] as f32 - 127.5) / 127.5;
}
- let side = MASK_SIDE as i64;
- let mut session = match self.session.lock() {
- Ok(guard) => guard,
- Err(poisoned) => poisoned.into_inner(),
- };
- let mut run = || -> Result<Vec<f32>, String> {
- let tensor = TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))
- .map_err(|err| err.to_string())?;
- let outputs = session
- .run(ort::inputs![tensor])
- .map_err(|err| err.to_string())?;
- if outputs.len() == 0 {
- return Err("model produced no outputs".to_string());
- }
- let (shape, data) = outputs[0]
- .try_extract_tensor::<f32>()
- .map_err(|err| err.to_string())?;
- let expected = (MASK_SIDE * MASK_SIDE) as usize;
- if data.len() != expected {
- return Err(format!("unexpected model output shape {shape:?}"));
- }
- Ok(data.to_vec())
- };
- let data = run()?;
+ let data = self.infer(input)?;
Ok(data
.iter()