π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][photos] Reduce Rust OCR crop and decoding overhead
laurenspriem575dff8a6a82
diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rsindex 1ef9e2c91f..d6e1fd06d3 100644--- a/rust/crates/ml/src/ocr/classify.rs+++ b/rust/crates/ml/src/ocr/classify.rs@@ -1,6 +1,6 @@use std::sync::{Mutex, PoisonError};-use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};+use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};@@ -51,7 +51,7 @@ impl AngleClassifier {}fn score_batch(&self, batch: &[ImageU8]) -> MlResult<Vec<AngleScores>> {- let input = PreparedF32Input::new(prepare_crop_tensor(|| batch_tensor(batch))?);+ let input = PreparedF32Input::new(prepare_crops(|| batch_tensor(batch))?);let count = batch.len() as i64;let expected_shape = [count, CLASS_COUNT as i64];let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);@@ -196,7 +196,7 @@ mod tests {.build().unwrap();let expected = parallel.install(|| batch_tensor(&batch)).unwrap();- let prepared = prepare_crop_tensor(|| batch_tensor(&batch)).unwrap();+ let prepared = prepare_crops(|| batch_tensor(&batch)).unwrap();assert_eq!(prepared, expected);}diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex 2d8295e72d..04e7534416 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -186,13 +186,13 @@ impl OcrEngine {let (classifier, recognizer) = self.text_models()?;let started = Instant::now();let request = self.requests.begin(req.request_id.as_deref());- let source = load_source(&req.image_path, FULL_TEXT_CAP)?;+ let mut source = load_source(&req.image_path, FULL_TEXT_CAP)?;request.check()?;let detection_started = Instant::now();let detection = self.detector.detect(&source.working)?;let detection_ms = detection_started.elapsed().as_millis();request.check()?;- let crops = crop_candidates(&source.working, &detection.candidates)?;+ let crops = crop_candidates(&mut source.working, &detection.candidates)?;request.check()?;let recognition_started = Instant::now();let mut recognized = recognize_crops(recognizer, crops, &request)?;@@ -379,22 +379,43 @@ impl RecognizedCrop {}fn crop_candidates(- working: &ImageU8,+ working: &mut ImageU8,candidates: &[DetectionCandidate],) -> MlResult<Vec<TextCrop>> {if candidates.is_empty() {return Ok(Vec::new());}+ let expected_len = (working.width as usize)+ .checked_mul(working.height as usize)+ .and_then(|pixels| pixels.checked_mul(3));+ if working.width <= 0+ || working.height <= 0+ || working.channels != 3+ || expected_len != Some(working.data.len())+ {+ return Err(MlError::Preprocess(+ "OCR crop source buffer mismatch".to_string(),+ ));+ }let source = image::RgbImage::from_raw(working.width as u32,working.height as u32,- working.data.clone(),+ std::mem::take(&mut working.data),)- .ok_or_else(|| MlError::Preprocess("OCR crop source buffer mismatch".to_string()))?;- candidates- .iter()- .map(|candidate| crop_text(&source, &candidate.points))- .collect()+ .expect("validated RGB source dimensions and length");+ let prepare = || {+ candidates+ .iter()+ .map(|candidate| crop_text(&source, &candidate.points))+ .collect()+ };+ let crops = if cfg!(target_os = "android") {+ tensor::prepare_crops(prepare)+ } else {+ prepare()+ };+ working.data = source.into_raw();+ crops}fn recognize_crops(@@ -575,8 +596,51 @@ mod tests {#[test]fn no_detected_regions_produce_no_crops() {- let image = ImageU8::zeros(32, 32, 3).unwrap();- assert!(crop_candidates(&image, &[]).unwrap().is_empty());+ let mut image = ImageU8::zeros(32, 32, 3).unwrap();+ assert!(crop_candidates(&mut image, &[]).unwrap().is_empty());+ }++ #[test]+ fn cropping_restores_the_source_buffer_on_success_and_error() {+ let data = (0..32 * 32 * 3).map(|i| (i * 37 % 256) as u8).collect();+ let original = ImageU8::new(32, 32, 3, data).unwrap();+ let mut image = original.clone();+ let pointer = image.data.as_ptr();+ let candidate = DetectionCandidate {+ points: quad(3.0, 4.0),+ score: 0.9,+ };+ let rgb = image::RgbImage::from_raw(32, 32, original.data.clone()).unwrap();+ let expected = crop_text(&rgb, &candidate.points).unwrap();+ let crops = crop_candidates(&mut image, std::slice::from_ref(&candidate)).unwrap();+ assert_eq!(crops[0].image, expected.image);+ assert_eq!(image, original);+ assert_eq!(image.data.as_ptr(), pointer);++ let invalid = DetectionCandidate {+ points: [Point::new(0.0, 0.0); 4],+ score: 0.9,+ };+ assert!(crop_candidates(&mut image, &[candidate, invalid]).is_err());+ assert_eq!(image, original);+ assert_eq!(image.data.as_ptr(), pointer);+ }++ #[test]+ fn cropping_rejects_invalid_source_geometry_without_taking_the_buffer() {+ let candidate = DetectionCandidate {+ points: quad(3.0, 4.0),+ score: 0.9,+ };+ for (width, height, channels) in [(33, 32, 3), (-1, 32, 3), (32, 32, 4)] {+ let mut image = ImageU8::zeros(32, 32, 3).unwrap();+ image.width = width;+ image.height = height;+ image.channels = channels;+ let original = image.clone();+ assert!(crop_candidates(&mut image, std::slice::from_ref(&candidate)).is_err());+ assert_eq!(image, original);+ }}#[test]diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex 8443ae3cc7..e1f6897eb5 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -4,7 +4,7 @@ use std::sync::{Mutex, OnceLock, PoisonError};use super::OcrError;use super::cancel::RequestGuard;use super::dictionary::load_dictionary;-use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};+use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};@@ -89,7 +89,7 @@ impl TextRecognizer {fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {let layout = BatchLayout::new(batch);- let input = PreparedF32Input::new(prepare_crop_tensor(|| layout.tensor(batch))?);+ let input = PreparedF32Input::new(prepare_crops(|| layout.tensor(batch))?);let count = batch.len();let input_shape = [count as i64,@@ -300,18 +300,33 @@ wherelogits.chunks_exact(vocabulary).map(argmax).collect()}-fn argmax(step: impl FloatTensorData) -> StepBest {- let mut best = StepBest {+fn argmax<'a, T>(step: &'a [T]) -> StepBest+where+ &'a [T]: FloatTensorData,+{+ let mut maxima = [f32::NEG_INFINITY; 8];+ let (chunks, remainder) = step.as_chunks::<8>();+ for chunk in chunks {+ for (lane, maximum) in maxima.iter_mut().enumerate() {+ *maximum = maximum.max(chunk.as_slice().value(lane));+ }+ }+ let mut maximum = maxima.into_iter().fold(f32::NEG_INFINITY, f32::max);+ for index in 0..remainder.len() {+ maximum = maximum.max(remainder.value(index));+ }+ if maximum > f32::NEG_INFINITY {+ for index in 0..step.len() {+ let probability = step.value(index);+ if probability == maximum {+ return StepBest { index, probability };+ }+ }+ }+ StepBest {index: 0,probability: f32::NEG_INFINITY,- };- for index in 0..step.len() {- let probability = step.value(index);- if probability > best.probability {- best = StepBest { index, probability };- }}- best}struct CharacterRun {@@ -554,10 +569,71 @@ mod tests {.build().unwrap();let expected = parallel.install(|| layout.tensor(&batch)).unwrap();- let prepared = prepare_crop_tensor(|| layout.tensor(&batch)).unwrap();+ let prepared = prepare_crops(|| layout.tensor(&batch)).unwrap();assert_eq!(prepared, expected);}+ #[test]+ fn argmax_matches_scalar_for_lane_boundaries_ties_and_both_precisions() {+ fn reference(values: impl FloatTensorData) -> StepBest {+ let mut best = StepBest {+ index: 0,+ probability: f32::NEG_INFINITY,+ };+ for index in 0..values.len() {+ let probability = values.value(index);+ if probability > best.probability {+ best = StepBest { index, probability };+ }+ }+ best+ }++ fn check<'a, T>(values: &'a [T])+ where+ &'a [T]: FloatTensorData,+ {+ let expected = reference(values);+ let actual = argmax(values);+ assert_eq!(actual.index, expected.index);+ assert_eq!(actual.probability.to_bits(), expected.probability.to_bits());+ }++ let mut state = 37u32;+ for len in (0..40).chain([191, 320, REC_VOCABULARY_SIZE]) {+ let mut values: Vec<f32> = (0..len)+ .map(|_| {+ state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);+ (state >> 16) as f32 / 65536.0 - 0.5+ })+ .collect();+ for first in 0..len.min(17) {+ for last in [first, len / 2, len - 1] {+ values[first] = 1.0;+ values[last] = 1.0;+ check(values.as_slice());+ let half_values: Vec<_> =+ values.iter().map(|&v| half::f16::from_f32(v)).collect();+ check(half_values.as_slice());+ values[first] = -0.25;+ values[last] = -0.25;+ }+ }+ check(values.as_slice());+ }+ for values in [+ vec![],+ vec![-0.0, 0.0, -0.0, 0.0, -0.0, 0.0, -0.0, 0.0, 0.0],+ vec![f32::NEG_INFINITY; 19],+ vec![f32::NAN; 19],+ vec![f32::NAN, 0.25, f32::INFINITY, f32::INFINITY],+ ] {+ check(values.as_slice());+ let half_values: Vec<_> = values.iter().map(|&v| half::f16::from_f32(v)).collect();+ check(half_values.as_slice());+ }+ }+#[test]fn borrowed_outputs_preserve_ties_blanks_and_half_precision_confidences() {let values = [diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rsindex 62fad88983..53986e025e 100644--- a/rust/crates/ml/src/ocr/tensor.rs+++ b/rust/crates/ml/src/ocr/tensor.rs@@ -5,9 +5,7 @@ use rayon::{ThreadPool, ThreadPoolBuilder};use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};-pub(crate) fn prepare_crop_tensor(- prepare: impl FnOnce() -> MlResult<Vec<f32>> + Send,-) -> MlResult<Vec<f32>> {+pub(crate) fn prepare_crops<T: Send>(prepare: impl FnOnce() -> MlResult<T> + Send) -> MlResult<T> {static POOL: OnceLock<Option<ThreadPool>> = OnceLock::new();let pool = POOL.get_or_init(|| {ThreadPoolBuilder::new()@@ -89,13 +87,19 @@ mod tests {fn preparation_keeps_the_callers_worker_pool_unchanged() {let caller = ThreadPoolBuilder::new().num_threads(4).build().unwrap();caller.install(|| {- let values =- prepare_crop_tensor(|| Ok(vec![rayon::current_num_threads() as f32])).unwrap();+ let values = prepare_crops(|| Ok(vec![rayon::current_num_threads() as f32])).unwrap();assert_eq!(values, [1.0]);assert_eq!(rayon::current_num_threads(), 4);});}+ #[test]+ fn preparation_reuses_the_worker_across_return_types() {+ let first = prepare_crops(|| Ok(std::thread::current().id())).unwrap();+ let second = prepare_crops(|| Ok(vec![std::thread::current().id()])).unwrap();+ assert_eq!(second, [first]);+ }+#[test]fn planes_are_bgr_left_aligned_per_row_and_zero_padded() {let rgb = ImageU8::new(