π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Add OCR text region detection
laurenspriemb203329093d7
diff --git a/rust/crates/ml/examples/ocr.rs b/rust/crates/ml/examples/ocr.rsnew file mode 100644index 0000000000..f368d39a35--- /dev/null+++ b/rust/crates/ml/examples/ocr.rs@@ -0,0 +1,179 @@+#[allow(dead_code)]+#[path = "../tests/support/mod.rs"]+mod support;++use std::path::{Path, PathBuf};+use std::time::Instant;++use anyhow::{Context, Result, bail};+use ente_assets::AssetStore;+use ente_image::decode::decode_image_from_path;+use ente_ml::ocr::{+ DetectRegionsRequest, OcrEngine, OcrModelPaths, Point, ProbabilityMap, RegionDetectionDebug,+ TextRegion, assets,+};+use image::{GrayImage, Rgb, RgbImage};+use imageproc::drawing::draw_line_segment_mut;+use support::ml_indexing::{asset_cache_dir, load_onnx_runtime, run_with_large_stack};++const OVERLAY_COLOR: Rgb<u8> = Rgb([255, 0, 0]);++struct Options {+ dump_dir: Option<PathBuf>,+ images: Vec<String>,+}++#[tokio::main(flavor = "current_thread")]+async fn main() -> Result<()> {+ let options = parse_args(std::env::args().skip(1))?;+ load_onnx_runtime().await?;+ let store = AssetStore::new(asset_cache_dir()?);+ let paths = assets::ensure_models(&store).await?;+ run_with_large_stack("ocr", move || run(options, paths))+}++fn usage() -> &'static str {+ "usage: cargo run -p ente-ml --example ocr -- [--regions] [--all-confidences] \+ [--dump-dir DIR] <image>..."+}++fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Options> {+ let mut options = Options {+ dump_dir: None,+ images: Vec::new(),+ };+ while let Some(arg) = args.next() {+ match arg.as_str() {+ "--regions" | "--all-confidences" => {}+ "--dump-dir" => {+ let dir = args.next().context("--dump-dir needs a directory")?;+ options.dump_dir = Some(PathBuf::from(dir));+ }+ other if other.starts_with("--") => bail!("unknown option {other}\n{}", usage()),+ image => options.images.push(image.to_string()),+ }+ }+ if options.images.is_empty() {+ bail!(usage());+ }+ Ok(options)+}++fn run(options: Options, paths: OcrModelPaths) -> Result<()> {+ let engine = OcrEngine::new(paths)?;+ let mut failures = 0usize;+ for image_path in &options.images {+ match detect_regions(&engine, image_path, options.dump_dir.as_deref()) {+ Ok(()) => {}+ Err(error) => {+ failures += 1;+ eprintln!("{image_path}: {error:#}");+ }+ }+ }+ if failures > 0 {+ bail!("{failures} image(s) failed");+ }+ Ok(())+}++fn detect_regions(engine: &OcrEngine, image_path: &str, dump_dir: Option<&Path>) -> Result<()> {+ let started = Instant::now();+ let debug = engine.detect_text_regions_debug(&DetectRegionsRequest {+ image_path: image_path.to_string(),+ request_id: None,+ })?;+ let elapsed_ms = started.elapsed().as_millis();+ let result = &debug.result;+ println!(+ "{image_path}: decoded {}x{}, working {}x{}, probmap {}x{}, {} regions in {elapsed_ms}ms",+ result.image_width,+ result.image_height,+ debug.working_width,+ debug.working_height,+ debug.probability_map.width,+ debug.probability_map.height,+ result.regions.len()+ );+ for region in &result.regions {+ println!(" {}", format_region(region));+ }+ if let Some(dir) = dump_dir {+ dump(dir, image_path, &debug)?;+ }+ Ok(())+}++fn format_region(region: &TextRegion) -> String {+ let corners = region+ .points+ .iter()+ .map(|point| format!("({:.0},{:.0})", point.x, point.y))+ .collect::<Vec<_>>()+ .join(" ");+ format!("[{:.3}] {corners}", region.confidence)+}++fn dump(dir: &Path, image_path: &str, debug: &RegionDetectionDebug) -> Result<()> {+ let stem = Path::new(image_path)+ .file_stem()+ .and_then(|stem| stem.to_str())+ .with_context(|| format!("no file stem in {image_path}"))?;+ let target = dir.join(stem);+ std::fs::create_dir_all(&target).with_context(|| format!("create {}", target.display()))?;+ probability_map_image(&debug.probability_map)?+ .save(target.join("probmap.png"))+ .context("write probmap.png")?;+ overlay_image(image_path, &debug.result.regions)?+ .save(target.join("overlay.png"))+ .context("write overlay.png")?;+ println!(" wrote {}", target.display());+ Ok(())+}++fn probability_map_image(map: &ProbabilityMap) -> Result<GrayImage> {+ let pixels = map+ .values+ .iter()+ .map(|value| (value.clamp(0.0, 1.0) * 255.0).round() as u8)+ .collect();+ GrayImage::from_raw(map.width as u32, map.height as u32, pixels)+ .context("probability map size does not match its values")+}++fn overlay_image(image_path: &str, regions: &[TextRegion]) -> Result<RgbImage> {+ let decoded = decode_image_from_path(image_path)?;+ let mut canvas = RgbImage::from_raw(+ decoded.dimensions.width,+ decoded.dimensions.height,+ decoded.rgb,+ )+ .context("decoded image size does not match its buffer")?;+ let thickness = (canvas.width().max(canvas.height()) / 800).clamp(1, 6);+ for region in regions {+ draw_quad(&mut canvas, ®ion.points, thickness);+ }+ Ok(canvas)+}++fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32) {+ for index in 0..4 {+ let start = points[index];+ let end = points[(index + 1) % 4];+ for offset in 0..thickness {+ let shift = offset as f32;+ draw_line_segment_mut(+ canvas,+ (start.x + shift, start.y),+ (end.x + shift, end.y),+ OVERLAY_COLOR,+ );+ draw_line_segment_mut(+ canvas,+ (start.x, start.y + shift),+ (end.x, end.y + shift),+ OVERLAY_COLOR,+ );+ }+ }+}diff --git a/rust/crates/ml/src/ocr/assets.rs b/rust/crates/ml/src/ocr/assets.rsnew file mode 100644index 0000000000..c5b3f81f00--- /dev/null+++ b/rust/crates/ml/src/ocr/assets.rs@@ -0,0 +1,186 @@+use std::time::Instant;++use ente_assets::download::CancellationToken;+use ente_assets::{Asset, AssetFile, AssetStore};++use super::OcrModelPaths;+use crate::error::{MlError, MlResult};++const MODEL_BASE_URL: &str = "https://models.ente.com/PP-OCRv5";+const MODELS: &str = "models";++struct OcrModelFile {+ key: &'static str,+ name: &'static str,+ size: u64,+ sha256: &'static str,+}++const DETECTION: OcrModelFile = OcrModelFile {+ key: "ppocrv5_det",+ name: "det.onnx",+ size: 4_748_769,+ sha256: "d7fe3ea74652890722c0f4d02458b7261d9f5ae6c92904d05707c9eb155c7924",+};++const CLASSIFICATION: OcrModelFile = OcrModelFile {+ key: "ppocrv5_cls",+ name: "cls.onnx",+ size: 582_663,+ sha256: "f4bb53707100c5f3d59ba834eb05bb400369f20aed35d4b26807b1bfadd2a70e",+};++const RECOGNITION: OcrModelFile = OcrModelFile {+ key: "ppocrv5_rec",+ name: "rec.onnx",+ size: 16_517_247,+ sha256: "bf66820f48fa99f779974c4df78e5274a9d8e0458c4137e8c5357e40e2c3faf2",+};++const DICTIONARY: OcrModelFile = OcrModelFile {+ key: "ppocrv5_dict",+ name: "ppocrv5_dict.txt",+ size: 74_012,+ sha256: "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b",+};++const CATALOG: [&OcrModelFile; 4] = [&DETECTION, &CLASSIFICATION, &RECOGNITION, &DICTIONARY];++impl OcrModelFile {+ fn url(&self) -> String {+ format!("{MODEL_BASE_URL}/{}", self.name)+ }++ fn asset(&self) -> Asset {+ Asset::file(+ vec![MODELS.to_string(), self.key.to_string()],+ AssetFile {+ name: self.name.to_string(),+ url: self.url(),+ size: self.size,+ sha256: self.sha256.to_string(),+ },+ )+ .expect("valid OCR model catalog")+ }++ fn path(&self, store: &AssetStore) -> String {+ store+ .file_path(&self.asset(), self.name)+ .expect("OCR model file")+ .to_string_lossy()+ .into_owned()+ }+}++pub fn model_assets() -> Vec<Asset> {+ CATALOG.iter().map(|file| file.asset()).collect()+}++pub async fn ensure_models(store: &AssetStore) -> MlResult<OcrModelPaths> {+ let missing: Vec<(&OcrModelFile, Asset)> = CATALOG+ .iter()+ .map(|file| (*file, file.asset()))+ .filter(|(_, asset)| !store.is_downloaded(asset))+ .collect();+ if missing.is_empty() {+ log::info!("ocr models: using cached copies");+ } else {+ for (file, _) in &missing {+ log::info!(+ "ocr models: downloading {} ({} bytes) from {}",+ file.name,+ file.size,+ file.url()+ );+ }+ let start = Instant::now();+ let assets: Vec<Asset> = missing.iter().map(|(_, asset)| asset.clone()).collect();+ store+ .download(&assets, |_| {}, CancellationToken::default())+ .await+ .map_err(|error| MlError::Runtime(format!("ocr model download failed: {error}")))?;+ log::info!(+ "ocr models: downloaded {} file(s) in {}ms",+ assets.len(),+ start.elapsed().as_millis()+ );+ }+ Ok(OcrModelPaths {+ detection: DETECTION.path(store),+ classification: CLASSIFICATION.path(store),+ recognition: RECOGNITION.path(store),+ dictionary: DICTIONARY.path(store),+ })+}++#[cfg(test)]+mod tests {+ use std::collections::HashSet;+ use std::path::Path;++ use super::*;++ #[test]+ fn catalog_keys_and_names_are_unique() {+ let keys: HashSet<&str> = CATALOG.iter().map(|file| file.key).collect();+ let names: HashSet<&str> = CATALOG.iter().map(|file| file.name).collect();+ assert_eq!(keys.len(), CATALOG.len());+ assert_eq!(names.len(), CATALOG.len());+ }++ #[test]+ fn catalog_directories_do_not_collide_with_the_indexing_catalog() {+ let store = AssetStore::new(Path::new("cache"));+ let clip_text = crate::assets::clip_text_asset();+ let others: Vec<_> = crate::assets::indexing_assets(true, true, true)+ .iter()+ .chain(std::iter::once(&clip_text))+ .map(|asset| store.asset_dir(asset))+ .collect();+ assert!(!others.is_empty());+ for asset in model_assets() {+ let dir = store.asset_dir(&asset);+ for other in &others {+ assert_ne!(&dir, other);+ assert!(!dir.starts_with(other), "{}", dir.display());+ assert!(!other.starts_with(&dir), "{}", other.display());+ }+ }+ }++ #[test]+ fn catalog_points_at_the_pp_ocrv5_cdn_directory() {+ assert_eq!(DETECTION.url(), "https://models.ente.com/PP-OCRv5/det.onnx");+ assert_eq!(+ DICTIONARY.url(),+ "https://models.ente.com/PP-OCRv5/ppocrv5_dict.txt"+ );+ assert_eq!(model_assets().len(), 4);+ }++ #[test]+ fn model_paths_live_under_the_store_models_directory() {+ let store = AssetStore::new(Path::new("cache"));+ for file in CATALOG {+ let path = file.path(&store);+ assert!(path.starts_with("cache/models/"), "{path}");+ assert!(path.ends_with(file.name), "{path}");+ assert!(!store.is_downloaded(&file.asset()));+ }+ }++ #[test]+ fn local_copies_match_the_catalog_sizes() {+ let local = Path::new(+ "/private/tmp/claude-501/-Users-laurens-dev-ente-repo-ente8/a307b632-3b09-4f43-884c-2830ff54fe8f/scratchpad/models",+ );+ if !local.is_dir() {+ return;+ }+ for file in CATALOG {+ let metadata = std::fs::metadata(local.join(file.name)).unwrap();+ assert_eq!(metadata.len(), file.size, "{}", file.name);+ }+ }+}diff --git a/rust/crates/ml/src/ocr/cancel.rs b/rust/crates/ml/src/ocr/cancel.rsnew file mode 100644index 0000000000..37cbb9100e--- /dev/null+++ b/rust/crates/ml/src/ocr/cancel.rs@@ -0,0 +1,129 @@+use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};++use super::OcrError;++type Flags = HashMap<String, Arc<AtomicBool>>;++#[derive(Default)]+pub(crate) struct RequestRegistry {+ flags: Mutex<Flags>,+}++impl RequestRegistry {+ pub(crate) fn begin(&self, request_id: Option<&str>) -> RequestGuard<'_> {+ let flag = Arc::new(AtomicBool::new(false));+ if let Some(id) = request_id+ && let Some(replaced) = self.flags().insert(id.to_string(), Arc::clone(&flag))+ {+ replaced.store(true, Ordering::SeqCst);+ }+ RequestGuard {+ registry: self,+ request_id: request_id.map(str::to_string),+ flag,+ }+ }++ pub(crate) fn cancel(&self, request_id: &str) {+ if let Some(flag) = self.flags().get(request_id) {+ flag.store(true, Ordering::SeqCst);+ }+ }++ fn finish(&self, request_id: &str, flag: &Arc<AtomicBool>) {+ let mut flags = self.flags();+ if flags+ .get(request_id)+ .is_some_and(|current| Arc::ptr_eq(current, flag))+ {+ flags.remove(request_id);+ }+ }++ fn flags(&self) -> MutexGuard<'_, Flags> {+ self.flags.lock().unwrap_or_else(PoisonError::into_inner)+ }+}++pub(crate) struct RequestGuard<'a> {+ registry: &'a RequestRegistry,+ request_id: Option<String>,+ flag: Arc<AtomicBool>,+}++impl RequestGuard<'_> {+ pub(crate) fn check(&self) -> Result<(), OcrError> {+ if self.flag.load(Ordering::SeqCst) {+ Err(OcrError::Cancelled)+ } else {+ Ok(())+ }+ }+}++impl Drop for RequestGuard<'_> {+ fn drop(&mut self) {+ if let Some(request_id) = &self.request_id {+ self.registry.finish(request_id, &self.flag);+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn is_cancelled(guard: &RequestGuard<'_>) -> bool {+ matches!(guard.check(), Err(OcrError::Cancelled))+ }++ #[test]+ fn cancel_marks_the_registered_request() {+ let registry = RequestRegistry::default();+ let guard = registry.begin(Some("req-1"));+ assert!(guard.check().is_ok());++ registry.cancel("req-1");++ assert!(is_cancelled(&guard));+ }++ #[test]+ fn cancelling_an_unknown_or_anonymous_request_is_a_no_op() {+ let registry = RequestRegistry::default();+ let anonymous = registry.begin(None);++ registry.cancel("missing");++ assert!(anonymous.check().is_ok());+ assert!(registry.flags().is_empty());+ }++ #[test]+ fn a_new_request_with_the_same_id_cancels_and_replaces_the_older_one() {+ let registry = RequestRegistry::default();+ let first = registry.begin(Some("shared"));+ let second = registry.begin(Some("shared"));++ assert!(is_cancelled(&first));+ assert!(second.check().is_ok());++ drop(first);+ assert_eq!(registry.flags().len(), 1);+ registry.cancel("shared");+ assert!(is_cancelled(&second));+ }++ #[test]+ fn finishing_a_request_removes_its_flag() {+ let registry = RequestRegistry::default();+ let guard = registry.begin(Some("done"));+ assert_eq!(registry.flags().len(), 1);++ drop(guard);++ assert!(registry.flags().is_empty());+ }+}diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rsindex 588161bcc8..896cfae879 100644--- a/rust/crates/ml/src/ocr/detect.rs+++ b/rust/crates/ml/src/ocr/detect.rs@@ -1,3 +1,5 @@+use std::sync::{Mutex, PoisonError};+use super::Point;use super::geometry::{clip_to_bounds, mean_inside_quad, min_area_rect, min_edge, order_corners, scale_points, unclip,@@ -5,7 +7,13 @@ use super::geometry::{use crate::cv;use crate::cv::image::{Contour, ImageU8};use crate::error::{MlError, MlResult};+use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};+const MODEL_NAMESPACE: &str = "ocr-detection";+const MAX_INPUT_SIDE: i32 = 960;+const INPUT_STRIDE: i32 = 32;+const CHANNEL_MEAN_BGR: [f32; 3] = [0.485, 0.456, 0.406];+const CHANNEL_STD_BGR: [f32; 3] = [0.229, 0.224, 0.225];const BITMAP_THRESHOLD: f32 = 0.3;const BOX_SCORE_THRESHOLD: f32 = 0.6;const UNCLIP_RATIO: f32 = 1.5;@@ -15,11 +23,113 @@ const MAX_CANDIDATES: usize = 1000;const READING_LINE_TOLERANCE: f32 = 10.0;#[derive(Clone, Debug, PartialEq)]-pub struct DetectionCandidate {+pub(crate) struct DetectionCandidate {pub points: [Point; 4],pub score: f32,}+#[derive(Clone, Debug, PartialEq)]+pub struct ProbabilityMap {+ pub width: usize,+ pub height: usize,+ pub values: Vec<f32>,+}++pub(crate) struct TextDetection {+ pub(crate) candidates: Vec<DetectionCandidate>,+ pub(crate) probability_map: ProbabilityMap,+}++pub(crate) struct TextDetector {+ session: Mutex<OnnxSession>,+}++impl TextDetector {+ pub(crate) fn new(model_path: &str) -> Self {+ Self {+ session: Mutex::new(+ OnnxSession::new(model_path, MODEL_NAMESPACE, ExecutionMode::CpuAccelerated)+ .with_unvalidated_acceleration(),+ ),+ }+ }++ pub(crate) fn detect(&self, working: &ImageU8) -> MlResult<TextDetection> {+ let (input_width, input_height) = detector_input_size(working.width, working.height);+ let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)+ .map_err(MlError::Preprocess)?;+ let input = PreparedF32Input::new(normalized_bgr_planes(&resized));+ let values = self.infer(&input, input_width, input_height)?;+ let probability_map = ProbabilityMap {+ width: input_width as usize,+ height: input_height as usize,+ values,+ };+ let mut candidates = candidates_from_probability_map(+ &probability_map.values,+ probability_map.width,+ probability_map.height,+ working.width as u32,+ working.height as u32,+ )?;+ sort_reading_order(&mut candidates);+ Ok(TextDetection {+ candidates,+ probability_map,+ })+ }++ fn infer(&self, input: &PreparedF32Input, width: i32, height: i32) -> MlResult<Vec<f32>> {+ let expected_shape = [1i64, 1, i64::from(height), i64::from(width)];+ let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);+ let (values, _usage) = session.run(|session| {+ let (shape, values) =+ run_f32(session, input, [1, 3, i64::from(height), i64::from(width)])?;+ if shape != expected_shape {+ return Err(SessionRunError::from(MlError::CorruptModel(format!(+ "text detector produced output shape {shape:?}, expected {expected_shape:?}"+ ))));+ }+ Ok(values)+ })?;+ Ok(values)+ }+}++pub(crate) fn detector_input_size(width: i32, height: i32) -> (i32, i32) {+ let longest = width.max(height);+ let ratio = if longest > MAX_INPUT_SIDE {+ f64::from(MAX_INPUT_SIDE) / f64::from(longest)+ } else {+ 1.0+ };+ (+ rounded_to_stride(f64::from(width) * ratio),+ rounded_to_stride(f64::from(height) * ratio),+ )+}++fn rounded_to_stride(side: f64) -> i32 {+ let stride = f64::from(INPUT_STRIDE);+ ((side.trunc() / stride).round_ties_even() as i32 * INPUT_STRIDE).max(INPUT_STRIDE)+}++fn normalized_bgr_planes(rgb: &ImageU8) -> Vec<f32> {+ let pixels = rgb.width as usize * rgb.height as usize;+ let mut planes = vec![0.0f32; pixels * 3];+ let (blue, rest) = planes.split_at_mut(pixels);+ let (green, red) = rest.split_at_mut(pixels);+ let normalize = |channel: usize, value: u8| {+ (value as f32 / 255.0 - CHANNEL_MEAN_BGR[channel]) / CHANNEL_STD_BGR[channel]+ };+ for (index, px) in rgb.data.as_chunks::<3>().0.iter().enumerate() {+ blue[index] = normalize(0, px[2]);+ green[index] = normalize(1, px[1]);+ red[index] = normalize(2, px[0]);+ }+ planes+}+impl DetectionCandidate {fn scaled(self, scale_x: f32, scale_y: f32) -> Self {Self {@@ -33,13 +143,13 @@ impl DetectionCandidate {}}-struct ProbabilityMap<'a> {+struct MapView<'a> {values: &'a [f32],width: usize,height: usize,}-pub fn candidates_from_probability_map(+pub(crate) fn candidates_from_probability_map(values: &[f32],map_width: usize,map_height: usize,@@ -52,7 +162,7 @@ pub fn candidates_from_probability_map(values.len())));}- let map = ProbabilityMap {+ let map = MapView {values,width: map_width,height: map_height,@@ -67,7 +177,7 @@ pub fn candidates_from_probability_map(Ok(candidates)}-fn threshold_bitmap(map: &ProbabilityMap<'_>) -> MlResult<ImageU8> {+fn threshold_bitmap(map: &MapView<'_>) -> MlResult<ImageU8> {let data = map.values.iter()@@ -76,7 +186,7 @@ fn threshold_bitmap(map: &ProbabilityMap<'_>) -> MlResult<ImageU8> {ImageU8::new(map.width as i32, map.height as i32, 1, data).map_err(MlError::Postprocess)}-fn largest_outer_contours(map: &ProbabilityMap<'_>) -> MlResult<Vec<Contour>> {+fn largest_outer_contours(map: &MapView<'_>) -> MlResult<Vec<Contour>> {let bitmap = threshold_bitmap(map)?;let mut contours: Vec<Contour> = cv::find_contours(&bitmap).map_err(MlError::Postprocess)?@@ -88,10 +198,7 @@ fn largest_outer_contours(map: &ProbabilityMap<'_>) -> MlResult<Vec<Contour>> {Ok(contours)}-fn candidate_from_contour(- contour: &Contour,- map: &ProbabilityMap<'_>,-) -> Option<DetectionCandidate> {+fn candidate_from_contour(contour: &Contour, map: &MapView<'_>) -> Option<DetectionCandidate> {let rect = order_corners(min_area_rect(&contour.points)?);if min_edge(&rect) < MIN_BOX_SIDE {return None;@@ -108,7 +215,7 @@ fn candidate_from_contour(Some(DetectionCandidate { points, score })}-pub fn sort_reading_order(candidates: &mut [DetectionCandidate]) {+pub(crate) fn sort_reading_order(candidates: &mut [DetectionCandidate]) {candidates.sort_by(|a, b| a.top_left().y.total_cmp(&b.top_left().y));let mut start = 0;while start < candidates.len() {@@ -342,4 +449,44 @@ mod tests {let error = candidates_from_probability_map(&[0.0; 10], 4, 4, 4, 4).unwrap_err();assert!(matches!(error, MlError::Postprocess(_)), "{error}");}++ #[test]+ fn detector_input_truncates_then_rounds_half_to_even_to_a_multiple_of_32() {+ assert_eq!(detector_input_size(1000, 700), (960, 672));+ assert_eq!(detector_input_size(500, 300), (512, 288));+ assert_eq!(detector_input_size(33, 33), (32, 32));+ assert_eq!(detector_input_size(4000, 3000), (960, 704));+ assert_eq!(detector_input_size(1280, 960), (960, 704));+ assert_eq!(detector_input_size(752, 720), (768, 704));+ assert_eq!(detector_input_size(1920, 1441), (960, 704));+ assert_eq!(detector_input_size(960, 960), (960, 960));+ assert_eq!(detector_input_size(1, 1), (32, 32));+ }++ #[test]+ fn detector_planes_are_bgr_and_normalised_per_channel() {+ let pixel = ImageU8::new(1, 1, 3, vec![10, 20, 30]).unwrap();+ let planes = normalized_bgr_planes(&pixel);+ let expected = [+ (30.0 / 255.0 - 0.485) / 0.229,+ (20.0 / 255.0 - 0.456) / 0.224,+ (10.0 / 255.0 - 0.406) / 0.225,+ ];+ assert_eq!(planes.len(), 3);+ for (actual, expected) in planes.iter().zip(expected) {+ assert!((actual - expected).abs() <= 1e-6, "{planes:?}");+ }+ }++ #[test]+ fn detector_planes_are_laid_out_plane_by_plane() {+ let two_pixels = ImageU8::new(2, 1, 3, vec![255, 0, 0, 0, 0, 255]).unwrap();+ let planes = normalized_bgr_planes(&two_pixels);+ let blue_of = |value: f32| (value / 255.0 - 0.485) / 0.229;+ let red_of = |value: f32| (value / 255.0 - 0.406) / 0.225;+ assert!((planes[0] - blue_of(0.0)).abs() <= 1e-6);+ assert!((planes[1] - blue_of(255.0)).abs() <= 1e-6);+ assert!((planes[4] - red_of(255.0)).abs() <= 1e-6);+ assert!((planes[5] - red_of(0.0)).abs() <= 1e-6);+ }}diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex bcd6519e65..5c8c69cc26 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -1,14 +1,27 @@+pub mod assets;+mod cancel;mod detect;mod dictionary;mod geometry;+mod source;-pub use detect::{DetectionCandidate, candidates_from_probability_map, sort_reading_order};+pub use detect::ProbabilityMap;pub use dictionary::load_dictionary;pub use geometry::Point;+use std::time::Instant;+use thiserror::Error;-use crate::error::MlError;+use cancel::RequestRegistry;+use detect::{DetectionCandidate, TextDetector};+use geometry::scale_points;+use source::{REGIONS_CAP, SourceImage, load_source};++use crate::error::{MlError, MlResult};++const REGION_MIN_SCORE: f32 = 0.5;+const MAX_REGIONS: usize = 1000;#[derive(Clone, Debug, PartialEq, Eq)]pub struct OcrModelPaths {@@ -66,6 +79,14 @@ pub struct TextRegionDetectionResult {pub image_height: u32,}+#[derive(Clone, Debug, PartialEq)]+pub struct RegionDetectionDebug {+ pub result: TextRegionDetectionResult,+ pub working_width: u32,+ pub working_height: u32,+ pub probability_map: ProbabilityMap,+}+#[derive(Debug, Error)]pub enum OcrError {#[error("image not found: {0}")]@@ -75,3 +96,167 @@ pub enum OcrError {#[error(transparent)]Ml(#[from] MlError),}++pub struct OcrEngine {+ detector: TextDetector,+ requests: RequestRegistry,+}++impl OcrEngine {+ pub fn new(paths: OcrModelPaths) -> MlResult<Self> {+ if paths.detection.is_empty() {+ return Err(MlError::InvalidRequest(+ "OCR detection model path is empty".to_string(),+ ));+ }+ Ok(Self {+ detector: TextDetector::new(&paths.detection),+ requests: RequestRegistry::default(),+ })+ }++ pub fn detect_text_regions(+ &self,+ req: &DetectRegionsRequest,+ ) -> Result<TextRegionDetectionResult, OcrError> {+ self.detect_text_regions_debug(req)+ .map(|debug| debug.result)+ }++ pub fn detect_text_regions_debug(+ &self,+ req: &DetectRegionsRequest,+ ) -> Result<RegionDetectionDebug, OcrError> {+ let started = Instant::now();+ let request = self.requests.begin(req.request_id.as_deref());+ let source = load_source(&req.image_path, REGIONS_CAP)?;+ request.check()?;+ let detection_started = Instant::now();+ let detection = self.detector.detect(&source.working)?;+ let detection_ms = detection_started.elapsed().as_millis();+ request.check()?;+ let regions = regions_in_decoded_pixels(&detection.candidates, &source);+ log::info!(+ "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",+ source.decoded_width,+ source.decoded_height,+ source.working.width,+ source.working.height,+ detection.candidates.len(),+ regions.len(),+ started.elapsed().as_millis()+ );+ Ok(RegionDetectionDebug {+ result: TextRegionDetectionResult {+ regions,+ image_width: source.decoded_width,+ image_height: source.decoded_height,+ },+ working_width: source.working.width as u32,+ working_height: source.working.height as u32,+ probability_map: detection.probability_map,+ })+ }++ pub fn cancel(&self, request_id: &str) {+ self.requests.cancel(request_id);+ }+}++fn regions_in_decoded_pixels(+ candidates: &[DetectionCandidate],+ source: &SourceImage,+) -> Vec<TextRegion> {+ let scale_x = source.decoded_width as f32 / source.working.width as f32;+ let scale_y = source.decoded_height as f32 / source.working.height as f32;+ candidates+ .iter()+ .filter(|candidate| candidate.score >= REGION_MIN_SCORE)+ .take(MAX_REGIONS)+ .map(|candidate| TextRegion {+ confidence: candidate.score,+ points: scale_points(&candidate.points, scale_x, scale_y),+ })+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn engine_requires_a_detection_model_path() {+ let error = OcrEngine::new(OcrModelPaths {+ detection: String::new(),+ classification: "cls.onnx".to_string(),+ recognition: "rec.onnx".to_string(),+ dictionary: "dict.txt".to_string(),+ })+ .err()+ .expect("empty detection path is rejected");+ assert!(matches!(error, MlError::InvalidRequest(_)), "{error}");+ }++ #[test]+ fn engine_accepts_a_detector_only_path_set_without_loading_it() {+ let engine = OcrEngine::new(OcrModelPaths {+ detection: "missing/det.onnx".to_string(),+ classification: String::new(),+ recognition: String::new(),+ dictionary: String::new(),+ });+ assert!(engine.is_ok());+ }++ #[test]+ fn missing_image_is_reported_before_the_detector_is_touched() {+ let engine = OcrEngine::new(OcrModelPaths {+ detection: "missing/det.onnx".to_string(),+ classification: String::new(),+ recognition: String::new(),+ dictionary: String::new(),+ })+ .unwrap();+ let error = engine+ .detect_text_regions(&DetectRegionsRequest {+ image_path: "missing/image.jpg".to_string(),+ request_id: Some("r1".to_string()),+ })+ .unwrap_err();+ assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");+ }++ #[test]+ fn regions_scale_from_working_to_decoded_pixels_and_drop_low_scores() {+ let source = SourceImage {+ decoded_width: 2000,+ decoded_height: 1000,+ working: crate::cv::image::ImageU8::zeros(1000, 500, 3).unwrap(),+ };+ let quad = |x: f32, y: f32| {+ [+ Point::new(x, y),+ Point::new(x + 10.0, y),+ Point::new(x + 10.0, y + 5.0),+ Point::new(x, y + 5.0),+ ]+ };+ let candidates = vec![+ DetectionCandidate {+ points: quad(100.0, 50.0),+ score: 0.9,+ },+ DetectionCandidate {+ points: quad(300.0, 50.0),+ score: 0.4,+ },+ ];++ let regions = regions_in_decoded_pixels(&candidates, &source);++ assert_eq!(regions.len(), 1);+ assert_eq!(regions[0].confidence, 0.9);+ assert_eq!(regions[0].points[0], Point::new(200.0, 100.0));+ assert_eq!(regions[0].points[2], Point::new(220.0, 110.0));+ }+}diff --git a/rust/crates/ml/src/ocr/source.rs b/rust/crates/ml/src/ocr/source.rsnew file mode 100644index 0000000000..240ca72bd0--- /dev/null+++ b/rust/crates/ml/src/ocr/source.rs@@ -0,0 +1,170 @@+use std::path::Path;++use ente_image::decode::decode_image_from_path;++use super::OcrError;+use crate::cv;+use crate::cv::image::ImageU8;+use crate::error::{MlError, MlResult};++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) struct WorkingSizeCap {+ max_side: u32,+ max_pixels: u32,+}++pub(crate) const REGIONS_CAP: WorkingSizeCap = WorkingSizeCap {+ max_side: 1280,+ max_pixels: 2_000_000,+};++impl WorkingSizeCap {+ fn scale(self, width: u32, height: u32) -> f64 {+ let longest = f64::from(width.max(height));+ let pixels = f64::from(width) * f64::from(height);+ (f64::from(self.max_side) / longest)+ .min((f64::from(self.max_pixels) / pixels).sqrt())+ .min(1.0)+ }++ pub(crate) fn working_size(self, width: u32, height: u32) -> (u32, u32) {+ let scale = self.scale(width, height);+ if scale >= 1.0 {+ return (width, height);+ }+ let scaled = |side: u32| ((f64::from(side) * scale).round() as u32).max(1);+ (scaled(width), scaled(height))+ }+}++#[derive(Debug)]+pub(crate) struct SourceImage {+ pub(crate) decoded_width: u32,+ pub(crate) decoded_height: u32,+ pub(crate) working: ImageU8,+}++pub(crate) fn load_source(image_path: &str, cap: WorkingSizeCap) -> Result<SourceImage, OcrError> {+ if !Path::new(image_path).is_file() {+ return Err(OcrError::ImageNotFound(image_path.to_string()));+ }+ let decoded = decode_image_from_path(image_path).map_err(MlError::from)?;+ let (width, height) = (decoded.dimensions.width, decoded.dimensions.height);+ let full =+ ImageU8::new(to_side(width)?, to_side(height)?, 3, decoded.rgb).map_err(MlError::Image)?;+ let (working_width, working_height) = cap.working_size(width, height);+ let working = if (working_width, working_height) == (width, height) {+ full+ } else {+ cv::resize_u8(+ &full,+ to_side(working_width)?,+ to_side(working_height)?,+ cv::Interp::Area,+ )+ .map_err(MlError::Image)?+ };+ Ok(SourceImage {+ decoded_width: width,+ decoded_height: height,+ working,+ })+}++fn to_side(value: u32) -> MlResult<i32> {+ i32::try_from(value).map_err(|_| MlError::Image(format!("image side {value} is too large")))+}++#[cfg(test)]+mod tests {+ use std::fs::File;+ use std::io::BufWriter;+ use std::path::Path;++ use image::ExtendedColorType;+ use image::ImageEncoder;+ use image::codecs::png::{CompressionType, FilterType, PngEncoder};++ use super::*;++ const FULL_TEXT_CAP: WorkingSizeCap = WorkingSizeCap {+ max_side: 4096,+ max_pixels: 12_000_000,+ };++ fn write_png(dir: &Path, name: &str, width: u32, height: u32) -> String {+ let path = dir.join(name);+ let writer = BufWriter::new(File::create(&path).unwrap());+ let pixels = vec![128u8; (width * height * 3) as usize];+ PngEncoder::new_with_quality(writer, CompressionType::Fast, FilterType::NoFilter)+ .write_image(&pixels, width, height, ExtendedColorType::Rgb8)+ .unwrap();+ path.to_string_lossy().into_owned()+ }++ #[test]+ fn wide_image_is_capped_at_the_longest_side() {+ let dir = tempfile::tempdir().unwrap();+ let path = write_png(dir.path(), "wide.png", 5000, 1000);++ let source = load_source(&path, FULL_TEXT_CAP).unwrap();++ assert_eq!((source.decoded_width, source.decoded_height), (5000, 1000));+ assert_eq!(source.working.width, 4096);+ assert_eq!(source.working.height, 819);+ assert_eq!(source.working.channels, 3);+ }++ #[test]+ fn square_image_is_capped_at_the_pixel_budget() {+ let dir = tempfile::tempdir().unwrap();+ let path = write_png(dir.path(), "square.png", 4000, 4000);++ let source = load_source(&path, FULL_TEXT_CAP).unwrap();++ let pixels = source.working.width as u64 * source.working.height as u64;+ assert!(pixels <= 12_000_000, "{pixels}");+ assert!(pixels > 11_900_000, "{pixels}");+ assert_eq!(source.working.width, source.working.height);+ }++ #[test]+ fn small_image_is_kept_as_is() {+ let dir = tempfile::tempdir().unwrap();+ let path = write_png(dir.path(), "small.png", 640, 480);++ let source = load_source(&path, REGIONS_CAP).unwrap();++ assert_eq!((source.decoded_width, source.decoded_height), (640, 480));+ assert_eq!((source.working.width, source.working.height), (640, 480));+ assert_eq!(source.working.data.len(), 640 * 480 * 3);+ }++ #[test]+ fn missing_file_is_reported_before_decoding() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir+ .path()+ .join("missing.jpg")+ .to_string_lossy()+ .into_owned();++ let error = load_source(&path, REGIONS_CAP).unwrap_err();++ assert!(+ matches!(&error, OcrError::ImageNotFound(reported) if reported == &path),+ "{error}"+ );+ }++ #[test]+ fn regions_cap_applies_the_smaller_of_both_limits() {+ assert_eq!(REGIONS_CAP.working_size(5000, 1000), (1280, 256));+ assert_eq!(REGIONS_CAP.working_size(2000, 2000), (1280, 1280));+ assert_eq!(REGIONS_CAP.working_size(1280, 960), (1280, 960));+ assert_eq!(FULL_TEXT_CAP.working_size(4000, 3000), (4000, 3000));+ assert_eq!(FULL_TEXT_CAP.working_size(5000, 3000), (4096, 2458));+ assert_eq!(FULL_TEXT_CAP.working_size(4000, 4000), (3464, 3464));+ assert_eq!(FULL_TEXT_CAP.working_size(1, 9000), (1, 4096));+ }+}diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rsindex 96a4536afd..c6434c6429 100644--- a/rust/crates/ml/src/onnx/providers.rs+++ b/rust/crates/ml/src/onnx/providers.rs@@ -48,6 +48,7 @@ const ENABLE_PERSISTENT_COREML_CACHE: bool = true;#[derive(Clone, Copy, Debug, PartialEq, Eq)]pub(crate) enum ExecutionMode {PlatformDefault,+ CpuAccelerated,CpuOnly,}@@ -77,6 +78,7 @@ impl ProviderPlan {) -> Self {let providers = match mode {ExecutionMode::PlatformDefault => platform_default_providers(model_path, validation),+ ExecutionMode::CpuAccelerated => cpu_accelerated_providers(),ExecutionMode::CpuOnly => vec![ExecutionProvider::Cpu],};Self::from_providers(providers)@@ -271,6 +273,16 @@ fn platform_default_providers(vec![ExecutionProvider::Cpu]}+#[cfg(target_os = "android")]+fn cpu_accelerated_providers() -> Vec<ExecutionProvider> {+ vec![ExecutionProvider::Xnnpack, ExecutionProvider::Cpu]+}++#[cfg(not(target_os = "android"))]+fn cpu_accelerated_providers() -> Vec<ExecutionProvider> {+ vec![ExecutionProvider::Cpu]+}+// Missing goldens fail closed, usually indicating they were not regenerated// after an update.#[cfg(any(@@ -478,6 +490,21 @@ mod tests {assert!(!plan.has_fallback());}+ #[test]+ fn cpu_accelerated_plan_ends_with_cpu_and_never_uses_gpu_providers() {+ let plan = ProviderPlan::new(+ ExecutionMode::CpuAccelerated,+ "model.onnx",+ AccelerationValidation::Unvalidated,+ );++ assert_eq!(plan.providers.last(), Some(&ExecutionProvider::Cpu));+ assert!(!plan.providers.iter().any(|provider| matches!(+ provider,+ ExecutionProvider::WebGpu | ExecutionProvider::CoreMl+ )));+ }+fn accelerated_provider_plan() -> ProviderPlan {ProviderPlan::from_providers(vec![ExecutionProvider::WebGpu,diff --git a/rust/crates/ml/tests/support/ml_indexing.rs b/rust/crates/ml/tests/support/ml_indexing.rsindex e5f80fbd0f..1c77338c0e 100644--- a/rust/crates/ml/tests/support/ml_indexing.rs+++ b/rust/crates/ml/tests/support/ml_indexing.rs@@ -188,11 +188,7 @@ impl MlIndexingTestContext {let golden_results = load_golden_results(&golden_path)?;let fixture_paths = fetch_fixtures(&store, &asset_lock.fixture_base_url, &manifest).await?;- let onnx_runtime_library =- resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;- let _ = ort::init_from(&onnx_runtime_library)- .context("load ONNX Runtime dynamic library")?- .commit();+ init_onnx_runtime(&store, &asset_lock).await?;let model_paths = resolve_model_paths(&store, &asset_lock.models).await?;@@ -415,11 +411,7 @@ impl GoldenTestAssets {let cache_dir = cache_dir(&repo_root);let store = AssetStore::new(&cache_dir);- let onnx_runtime_library =- resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;- let _ = ort::init_from(&onnx_runtime_library)- .context("load ONNX Runtime dynamic library")?- .commit();+ init_onnx_runtime(&store, &asset_lock).await?;let models = &asset_lock.models;Ok(Self {@@ -555,6 +547,28 @@ struct ComparableFace {embedding: Vec<f64>,}+#[allow(dead_code)]+pub(crate) async fn load_onnx_runtime() -> Result<()> {+ let repo_root = repo_root()?;+ let asset_lock = load_asset_lock(&repo_root)?;+ let store = AssetStore::new(cache_dir(&repo_root));+ init_onnx_runtime(&store, &asset_lock).await+}++#[allow(dead_code)]+pub(crate) fn asset_cache_dir() -> Result<PathBuf> {+ Ok(cache_dir(&repo_root()?))+}++async fn init_onnx_runtime(store: &AssetStore, asset_lock: &AssetLock) -> Result<()> {+ let onnx_runtime_library =+ resolve_onnx_runtime_library(store, &asset_lock.onnx_runtime).await?;+ let _ = ort::init_from(&onnx_runtime_library)+ .context("load ONNX Runtime dynamic library")?+ .commit();+ Ok(())+}+fn repo_root() -> Result<PathBuf> {let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));manifest_dir