π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][photos] Optimize Rust OCR preprocessing and decoding
laurenspriemdbd249014514
diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rsindex 81882e6b4b..de0adc365e 100644--- a/rust/crates/ml/src/cv/mod.rs+++ b/rust/crates/ml/src/cv/mod.rs@@ -36,7 +36,7 @@ pub(crate) use stats::{};pub(crate) use structuring::ellipse_kernel;pub(crate) use transform::rotate_u8;-pub(crate) use warp::warp_perspective;+pub(crate) use warp::{warp_perspective, warp_rgb_perspective};pub(crate) const PARALLEL_MIN_ELEMS: usize = 200_000;const PIXELS_PER_CHUNK: usize = 65_536;diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rsindex 122d8a3f7f..73cfd778c2 100644--- a/rust/crates/ml/src/cv/warp.rs+++ b/rust/crates/ml/src/cv/warp.rs@@ -22,15 +22,29 @@ pub(crate) fn warp_perspective(src.channels));}+ let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())+ .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;+ warp_rgb_perspective(&source, src_corners, dst_corners, width, height)+}++pub(crate) fn warp_rgb_perspective(+ source: &RgbImage,+ src_corners: [(f64, f64); 4],+ dst_corners: [(f64, f64); 4],+ width: i32,+ height: i32,+) -> OpResult<ImageU8> {+ if width <= 0 || height <= 0 {+ return Err(format!(+ "warp_perspective: invalid destination size {width}x{height}"+ ));+ }let narrow = |c: [(f64, f64); 4]| c.map(|(x, y)| (x as f32, y as f32));let projection = Projection::from_control_points(narrow(src_corners), narrow(dst_corners)).ok_or_else(|| "warp_perspective: the corner pairs are degenerate".to_string())?;-- let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())- .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;let mut out = RgbImage::new(width as u32, height as u32);warp_into(- &source,+ source,&projection,Interpolation::Bilinear,Rgb([0, 0, 0]),diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rsindex de07e72444..1ef9e2c91f 100644--- a/rust/crates/ml/src/ocr/classify.rs+++ b/rust/crates/ml/src/ocr/classify.rs@@ -1,6 +1,6 @@use std::sync::{Mutex, PoisonError};-use super::tensor::{BgrNormalization, write_bgr_planes};+use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};@@ -51,7 +51,7 @@ impl AngleClassifier {}fn score_batch(&self, batch: &[ImageU8]) -> MlResult<Vec<AngleScores>> {- let input = PreparedF32Input::new(batch_tensor(batch)?);+ let input = PreparedF32Input::new(prepare_crop_tensor(|| batch_tensor(batch))?);let count = batch.len() as i64;let expected_shape = [count, CLASS_COUNT as i64];let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);@@ -180,6 +180,26 @@ mod tests {assert!(!scores(0.15, 0.85).needs_rotation());}+ #[test]+ fn crop_preparation_preserves_resized_classifier_values() {+ let batch: Vec<_> = [(7, 3), (390, 25), (3, 71)]+ .into_iter()+ .map(|(width, height)| {+ let pixels = (0..width * height * 3)+ .map(|index| (index * 29 % 256) as u8)+ .collect();+ ImageU8::new(width, height, 3, pixels).unwrap()+ })+ .collect();+ let parallel = rayon::ThreadPoolBuilder::new()+ .num_threads(4)+ .build()+ .unwrap();+ let expected = parallel.install(|| batch_tensor(&batch)).unwrap();+ let prepared = prepare_crop_tensor(|| batch_tensor(&batch)).unwrap();+ assert_eq!(prepared, expected);+ }+#[test]fn decisions_rotate_only_flagged_crops_in_place() {let mut crop = ImageU8::new(2, 1, 3, vec![1, 2, 3, 4, 5, 6]).unwrap();diff --git a/rust/crates/ml/src/ocr/crop.rs b/rust/crates/ml/src/ocr/crop.rsindex 2bb7926eaa..2442c17b82 100644--- a/rust/crates/ml/src/ocr/crop.rs+++ b/rust/crates/ml/src/ocr/crop.rs@@ -1,3 +1,5 @@+use image::RgbImage;+use super::Point;use crate::cv;use crate::cv::image::ImageU8;@@ -18,7 +20,7 @@ pub(crate) struct TextCrop {pub(crate) orientation: Orientation,}-pub(crate) fn crop_text(working: &ImageU8, quad: &[Point; 4]) -> MlResult<TextCrop> {+pub(crate) fn crop_text(working: &RgbImage, quad: &[Point; 4]) -> MlResult<TextCrop> {let (width, height) = crop_size(quad);let warped = warp_to_rectangle(working, quad, width, height)?;if is_vertical(width, height) {@@ -50,7 +52,7 @@ fn is_vertical(width: i32, height: i32) -> bool {}fn warp_to_rectangle(- working: &ImageU8,+ working: &RgbImage,quad: &[Point; 4],width: i32,height: i32,@@ -58,7 +60,7 @@ fn warp_to_rectangle(let source_corners = quad.map(|p| (f64::from(p.x), f64::from(p.y)));let (w, h) = (f64::from(width), f64::from(height));let target_corners = [(0.0, 0.0), (w, 0.0), (w, h), (0.0, h)];- cv::warp_perspective(working, source_corners, target_corners, width, height)+ cv::warp_rgb_perspective(working, source_corners, target_corners, width, height).map_err(MlError::Preprocess)}@@ -74,11 +76,11 @@ mod tests {points.map(|(x, y)| Point::new(x, y))}- fn gradient_image(width: i32, height: i32) -> ImageU8 {+ fn gradient_image(width: i32, height: i32) -> RgbImage {let data = (0..height).flat_map(|y| (0..width).flat_map(move |x| [x as u8, y as u8, (x + y) as u8])).collect();- ImageU8::new(width, height, 3, data).unwrap()+ RgbImage::from_raw(width as u32, height as u32, data).unwrap()}fn pixel(image: &ImageU8, x: i32, y: i32) -> [u8; 3] {@@ -115,7 +117,7 @@ mod tests {for x in 0..20 {assert_eq!(pixel(&crop.image, x, y),- pixel(&source, x + 8, y + 5),+ source.get_pixel((x + 8) as u32, (y + 5) as u32).0,"pixel ({x}, {y})");}@@ -132,9 +134,9 @@ mod tests {.unwrap();assert_eq!(crop.orientation, Orientation::Vertical);assert_eq!((crop.image.width, crop.image.height), (3, 2));- assert_eq!(pixel(&crop.image, 0, 0), pixel(&source, 5, 2));- assert_eq!(pixel(&crop.image, 2, 0), pixel(&source, 5, 4));- assert_eq!(pixel(&crop.image, 0, 1), pixel(&source, 4, 2));- assert_eq!(pixel(&crop.image, 2, 1), pixel(&source, 4, 4));+ assert_eq!(pixel(&crop.image, 0, 0), source.get_pixel(5, 2).0);+ assert_eq!(pixel(&crop.image, 2, 0), source.get_pixel(5, 4).0);+ assert_eq!(pixel(&crop.image, 0, 1), source.get_pixel(4, 2).0);+ assert_eq!(pixel(&crop.image, 2, 1), source.get_pixel(4, 4).0);}}diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex e2b12c1f4c..2d8295e72d 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -382,9 +382,18 @@ fn crop_candidates(working: &ImageU8,candidates: &[DetectionCandidate],) -> MlResult<Vec<TextCrop>> {+ if candidates.is_empty() {+ return Ok(Vec::new());+ }+ let source = image::RgbImage::from_raw(+ working.width as u32,+ working.height as u32,+ working.data.clone(),+ )+ .ok_or_else(|| MlError::Preprocess("OCR crop source buffer mismatch".to_string()))?;candidates.iter()- .map(|candidate| crop_text(working, &candidate.points))+ .map(|candidate| crop_text(&source, &candidate.points)).collect()}@@ -564,6 +573,12 @@ mod tests {}}+ #[test]+ fn no_detected_regions_produce_no_crops() {+ let image = ImageU8::zeros(32, 32, 3).unwrap();+ assert!(crop_candidates(&image, &[]).unwrap().is_empty());+ }+#[test]fn engine_accepts_a_detector_only_path_set_without_loading_it() {let engine = detector_only_engine();diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex 33ff2ae47d..8443ae3cc7 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -4,11 +4,14 @@ use std::sync::{Mutex, OnceLock, PoisonError};use super::OcrError;use super::cancel::RequestGuard;use super::dictionary::load_dictionary;-use super::tensor::{BgrNormalization, write_bgr_planes};+use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};-use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};+use crate::onnx::{+ BorrowedFloatTensor, ExecutionMode, FloatTensorData, OnnxSession, PreparedF32Input,+ with_prepared_float_output,+};const MODEL_NAMESPACE: &str = "ocr-recognition";const REC_VOCABULARY_SIZE: usize = 18385;@@ -86,7 +89,7 @@ impl TextRecognizer {fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {let layout = BatchLayout::new(batch);- let input = PreparedF32Input::new(layout.tensor(batch)?);+ let input = PreparedF32Input::new(prepare_crop_tensor(|| layout.tensor(batch))?);let count = batch.len();let input_shape = [count as i64,@@ -95,27 +98,45 @@ impl TextRecognizer {i64::from(layout.target_width),];let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);- let (output, _usage) = session.run(|session| {- let (shape, values) = run_f32(session, &input, input_shape)?;- SequenceOutput::new(&shape, values, count, dictionary.len())- .map_err(SessionRunError::from)- })?;- Ok(layout- .content_widths- .iter()- .zip(output.sequences())- .map(|(&content_width, logits)| {- ctc_decode(- logits,- output.vocabulary,- dictionary,- layout.padding_scale(content_width),- )+ let (recognized, _usage) = session.run(|session| {+ with_prepared_float_output(session, &input, input_shape, |shape, values| match values {+ BorrowedFloatTensor::F32(values) => {+ decode_output(shape, values, &layout, dictionary)+ }+ BorrowedFloatTensor::F16(values) => {+ decode_output(shape, values, &layout, dictionary)+ }})- .collect())+ })?;+ Ok(recognized)}}+fn decode_output<'a, T>(+ shape: &[i64],+ values: &'a [T],+ layout: &BatchLayout,+ dictionary: &[String],+) -> MlResult<Vec<Recognition>>+where+ &'a [T]: FloatTensorData,+{+ let output = SequenceOutput::new(shape, values, layout.content_widths.len(), dictionary.len())?;+ Ok(layout+ .content_widths+ .iter()+ .zip(output.sequences())+ .map(|(&content_width, logits)| {+ ctc_decode(+ logits,+ output.vocabulary,+ dictionary,+ layout.padding_scale(content_width),+ )+ })+ .collect())+}+fn recognize_in_batches(crops: &[&ImageU8],request: &RequestGuard<'_>,@@ -209,14 +230,14 @@ fn content_width(crop: &ImageU8, target_width: i32) -> i32 {}#[derive(Debug)]-struct SequenceOutput {+struct SequenceOutput<'a, T> {steps: usize,vocabulary: usize,- values: Vec<f32>,+ values: &'a [T],}-impl SequenceOutput {- fn new(shape: &[i64], values: Vec<f32>, count: usize, vocabulary: usize) -> MlResult<Self> {+impl<'a, T> SequenceOutput<'a, T> {+ fn new(shape: &[i64], values: &'a [T], count: usize, vocabulary: usize) -> MlResult<Self> {match *shape {[n, steps, v]if n == count as i64@@ -237,17 +258,20 @@ impl SequenceOutput {}}- fn sequences(&self) -> impl Iterator<Item = &[f32]> {+ fn sequences(&self) -> impl Iterator<Item = &'a [T]> {self.values.chunks_exact(self.steps * self.vocabulary)}}-fn ctc_decode(- logits: &[f32],+fn ctc_decode<'a, T>(+ logits: &'a [T],vocabulary: usize,dictionary: &[String],padding_scale: f32,-) -> Recognition {+) -> Recognition+where+ &'a [T]: FloatTensorData,+{let best = best_per_step(logits, vocabulary);let spans: Vec<CharacterSpan> = character_runs(&best).iter()@@ -266,19 +290,23 @@ struct StepBest {probability: f32,}-fn best_per_step(logits: &[f32], vocabulary: usize) -> Vec<StepBest> {+fn best_per_step<'a, T>(logits: &'a [T], vocabulary: usize) -> Vec<StepBest>+where+ &'a [T]: FloatTensorData,+{if vocabulary == 0 {return Vec::new();}logits.chunks_exact(vocabulary).map(argmax).collect()}-fn argmax(step: &[f32]) -> StepBest {+fn argmax(step: impl FloatTensorData) -> StepBest {let mut best = StepBest {index: 0,probability: f32::NEG_INFINITY,};- for (index, &probability) in step.iter().enumerate() {+ for index in 0..step.len() {+ let probability = step.value(index);if probability > best.probability {best = StepBest { index, probability };}@@ -497,17 +525,58 @@ mod tests {#[test]fn sequence_output_requires_the_dictionary_vocabulary() {- let ok = SequenceOutput::new(&[1, 2, 4], vec![0.0; 8], 1, 4).unwrap();+ let ok = SequenceOutput::new(&[1, 2, 4], &[0.0f32; 8], 1, 4).unwrap();assert_eq!((ok.steps, ok.vocabulary), (2, 4));assert_eq!(ok.sequences().count(), 1);for shape in [[1, 2, 5], [2, 2, 4], [1, 0, 4]] {- let error = SequenceOutput::new(&shape, vec![0.0; 8], 1, 4).unwrap_err();+ let error = SequenceOutput::new(&shape, &[0.0f32; 8], 1, 4).unwrap_err();assert!(matches!(error, MlError::CorruptModel(_)), "{error}");}- let short = SequenceOutput::new(&[1, 2, 4], vec![0.0; 7], 1, 4).unwrap_err();+ let short = SequenceOutput::new(&[1, 2, 4], &[0.0f32; 7], 1, 4).unwrap_err();assert!(matches!(short, MlError::CorruptModel(_)), "{short}");}+ #[test]+ fn crop_preparation_preserves_resized_batch_values() {+ let crops: Vec<_> = [(79, 11), (12, 160), (611, 53)]+ .into_iter()+ .map(|(width, height)| {+ let pixels = (0..width * height * 3)+ .map(|index| (index * 37 % 256) as u8)+ .collect();+ ImageU8::new(width, height, 3, pixels).unwrap()+ })+ .collect();+ let batch = crops.iter().collect::<Vec<_>>();+ let layout = BatchLayout::new(&batch);+ let parallel = rayon::ThreadPoolBuilder::new()+ .num_threads(4)+ .build()+ .unwrap();+ let expected = parallel.install(|| layout.tensor(&batch)).unwrap();+ let prepared = prepare_crop_tensor(|| layout.tensor(&batch)).unwrap();+ assert_eq!(prepared, expected);+ }++ #[test]+ fn borrowed_outputs_preserve_ties_blanks_and_half_precision_confidences() {+ let values = [+ 0.0, 0.75, 0.75, 0.0, 0.0, 0.5, 0.0, 0.0, 0.875, 0.0, 0.0, 0.0, 0.0, 0.625, 0.0, 0.0,+ ];+ let half_values = values.map(half::f16::from_f32);+ let layout = BatchLayout {+ target_width: 320,+ content_widths: vec![320],+ };+ let full = decode_output(&[1, 4, 4], &values, &layout, &dictionary()).unwrap();+ let half = decode_output(&[1, 4, 4], &half_values, &layout, &dictionary()).unwrap();+ assert_eq!(full, half);+ assert_eq!(full[0].text, "aa");+ assert_span(&full[0].spans[0], "a", 0.625, 0.0, 0.5);+ assert_span(&full[0].spans[1], "a", 0.625, 0.75, 1.0);+ assert_eq!(full[0].confidence, 0.625);+ }+#[test]fn crops_are_batched_by_ascending_aspect_and_restored_to_input_order() {let widths = [80, 10, 60, 30, 70, 20, 50, 40];diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rsindex 82cffa9ac8..62fad88983 100644--- a/rust/crates/ml/src/ocr/tensor.rs+++ b/rust/crates/ml/src/ocr/tensor.rs@@ -1,6 +1,27 @@+use std::sync::OnceLock;++use rayon::{ThreadPool, ThreadPoolBuilder};+use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};+pub(crate) fn prepare_crop_tensor(+ prepare: impl FnOnce() -> MlResult<Vec<f32>> + Send,+) -> MlResult<Vec<f32>> {+ static POOL: OnceLock<Option<ThreadPool>> = OnceLock::new();+ let pool = POOL.get_or_init(|| {+ ThreadPoolBuilder::new()+ .num_threads(1)+ .thread_name(|_| "ocr-crop-preparation".to_string())+ .build()+ .ok()+ });+ match pool {+ Some(pool) => pool.install(prepare),+ None => prepare(),+ }+}+#[derive(Clone, Copy, Debug, PartialEq)]pub(crate) struct BgrNormalization {mean: [f32; 3],@@ -64,6 +85,17 @@ mod tests {(value as f32 / 255.0 - 0.5) / 0.5}+ #[test]+ fn preparation_keeps_the_callers_worker_pool_unchanged() {+ let caller = ThreadPoolBuilder::new().num_threads(4).build().unwrap();+ caller.install(|| {+ let values =+ prepare_crop_tensor(|| Ok(vec![rayon::current_num_threads() as f32])).unwrap();+ assert_eq!(values, [1.0]);+ assert_eq!(rayon::current_num_threads(), 4);+ });+ }+#[test]fn planes_are_bgr_left_aligned_per_row_and_zero_padded() {let rgb = ImageU8::new(