Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Reduce Rust OCR crop and decoding overhead

laurenspriem575dff8a6a82
diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rs
index 1ef9e2c91f..d6e1fd06d3 100644
--- a/rust/crates/ml/src/ocr/classify.rs
+++ b/rust/crates/ml/src/ocr/classify.rs
@@ -1,6 +1,6 @@
use std::sync::{Mutex, PoisonError};
-use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};
+use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};
use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
@@ -51,7 +51,7 @@ impl AngleClassifier {
}
fn score_batch(&self, batch: &[ImageU8]) -> MlResult<Vec<AngleScores>> {
- let input = PreparedF32Input::new(prepare_crop_tensor(|| batch_tensor(batch))?);
+ let input = PreparedF32Input::new(prepare_crops(|| batch_tensor(batch))?);
let count = batch.len() as i64;
let expected_shape = [count, CLASS_COUNT as i64];
let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
@@ -196,7 +196,7 @@ mod tests {
.build()
.unwrap();
let expected = parallel.install(|| batch_tensor(&batch)).unwrap();
- let prepared = prepare_crop_tensor(|| batch_tensor(&batch)).unwrap();
+ let prepared = prepare_crops(|| batch_tensor(&batch)).unwrap();
assert_eq!(prepared, expected);
}
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index 2d8295e72d..04e7534416 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -186,13 +186,13 @@ impl OcrEngine {
let (classifier, recognizer) = self.text_models()?;
let started = Instant::now();
let request = self.requests.begin(req.request_id.as_deref());
- let source = load_source(&req.image_path, FULL_TEXT_CAP)?;
+ let mut source = load_source(&req.image_path, FULL_TEXT_CAP)?;
request.check()?;
let detection_started = Instant::now();
let detection = self.detector.detect(&source.working)?;
let detection_ms = detection_started.elapsed().as_millis();
request.check()?;
- let crops = crop_candidates(&source.working, &detection.candidates)?;
+ let crops = crop_candidates(&mut source.working, &detection.candidates)?;
request.check()?;
let recognition_started = Instant::now();
let mut recognized = recognize_crops(recognizer, crops, &request)?;
@@ -379,22 +379,43 @@ impl RecognizedCrop {
}
fn crop_candidates(
- working: &ImageU8,
+ working: &mut ImageU8,
candidates: &[DetectionCandidate],
) -> MlResult<Vec<TextCrop>> {
if candidates.is_empty() {
return Ok(Vec::new());
}
+ let expected_len = (working.width as usize)
+ .checked_mul(working.height as usize)
+ .and_then(|pixels| pixels.checked_mul(3));
+ if working.width <= 0
+ || working.height <= 0
+ || working.channels != 3
+ || expected_len != Some(working.data.len())
+ {
+ return Err(MlError::Preprocess(
+ "OCR crop source buffer mismatch".to_string(),
+ ));
+ }
let source = image::RgbImage::from_raw(
working.width as u32,
working.height as u32,
- working.data.clone(),
+ std::mem::take(&mut working.data),
)
- .ok_or_else(|| MlError::Preprocess("OCR crop source buffer mismatch".to_string()))?;
- candidates
- .iter()
- .map(|candidate| crop_text(&source, &candidate.points))
- .collect()
+ .expect("validated RGB source dimensions and length");
+ let prepare = || {
+ candidates
+ .iter()
+ .map(|candidate| crop_text(&source, &candidate.points))
+ .collect()
+ };
+ let crops = if cfg!(target_os = "android") {
+ tensor::prepare_crops(prepare)
+ } else {
+ prepare()
+ };
+ working.data = source.into_raw();
+ crops
}
fn recognize_crops(
@@ -575,8 +596,51 @@ mod tests {
#[test]
fn no_detected_regions_produce_no_crops() {
- let image = ImageU8::zeros(32, 32, 3).unwrap();
- assert!(crop_candidates(&image, &[]).unwrap().is_empty());
+ let mut image = ImageU8::zeros(32, 32, 3).unwrap();
+ assert!(crop_candidates(&mut image, &[]).unwrap().is_empty());
+ }
+
+ #[test]
+ fn cropping_restores_the_source_buffer_on_success_and_error() {
+ let data = (0..32 * 32 * 3).map(|i| (i * 37 % 256) as u8).collect();
+ let original = ImageU8::new(32, 32, 3, data).unwrap();
+ let mut image = original.clone();
+ let pointer = image.data.as_ptr();
+ let candidate = DetectionCandidate {
+ points: quad(3.0, 4.0),
+ score: 0.9,
+ };
+ let rgb = image::RgbImage::from_raw(32, 32, original.data.clone()).unwrap();
+ let expected = crop_text(&rgb, &candidate.points).unwrap();
+ let crops = crop_candidates(&mut image, std::slice::from_ref(&candidate)).unwrap();
+ assert_eq!(crops[0].image, expected.image);
+ assert_eq!(image, original);
+ assert_eq!(image.data.as_ptr(), pointer);
+
+ let invalid = DetectionCandidate {
+ points: [Point::new(0.0, 0.0); 4],
+ score: 0.9,
+ };
+ assert!(crop_candidates(&mut image, &[candidate, invalid]).is_err());
+ assert_eq!(image, original);
+ assert_eq!(image.data.as_ptr(), pointer);
+ }
+
+ #[test]
+ fn cropping_rejects_invalid_source_geometry_without_taking_the_buffer() {
+ let candidate = DetectionCandidate {
+ points: quad(3.0, 4.0),
+ score: 0.9,
+ };
+ for (width, height, channels) in [(33, 32, 3), (-1, 32, 3), (32, 32, 4)] {
+ let mut image = ImageU8::zeros(32, 32, 3).unwrap();
+ image.width = width;
+ image.height = height;
+ image.channels = channels;
+ let original = image.clone();
+ assert!(crop_candidates(&mut image, std::slice::from_ref(&candidate)).is_err());
+ assert_eq!(image, original);
+ }
}
#[test]
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index 8443ae3cc7..e1f6897eb5 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -4,7 +4,7 @@ use std::sync::{Mutex, OnceLock, PoisonError};
use super::OcrError;
use super::cancel::RequestGuard;
use super::dictionary::load_dictionary;
-use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};
+use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};
use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
@@ -89,7 +89,7 @@ impl TextRecognizer {
fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {
let layout = BatchLayout::new(batch);
- let input = PreparedF32Input::new(prepare_crop_tensor(|| layout.tensor(batch))?);
+ let input = PreparedF32Input::new(prepare_crops(|| layout.tensor(batch))?);
let count = batch.len();
let input_shape = [
count as i64,
@@ -300,18 +300,33 @@ where
logits.chunks_exact(vocabulary).map(argmax).collect()
}
-fn argmax(step: impl FloatTensorData) -> StepBest {
- let mut best = StepBest {
+fn argmax<'a, T>(step: &'a [T]) -> StepBest
+where
+ &'a [T]: FloatTensorData,
+{
+ let mut maxima = [f32::NEG_INFINITY; 8];
+ let (chunks, remainder) = step.as_chunks::<8>();
+ for chunk in chunks {
+ for (lane, maximum) in maxima.iter_mut().enumerate() {
+ *maximum = maximum.max(chunk.as_slice().value(lane));
+ }
+ }
+ let mut maximum = maxima.into_iter().fold(f32::NEG_INFINITY, f32::max);
+ for index in 0..remainder.len() {
+ maximum = maximum.max(remainder.value(index));
+ }
+ if maximum > f32::NEG_INFINITY {
+ for index in 0..step.len() {
+ let probability = step.value(index);
+ if probability == maximum {
+ return StepBest { index, probability };
+ }
+ }
+ }
+ StepBest {
index: 0,
probability: f32::NEG_INFINITY,
- };
- for index in 0..step.len() {
- let probability = step.value(index);
- if probability > best.probability {
- best = StepBest { index, probability };
- }
}
- best
}
struct CharacterRun {
@@ -554,10 +569,71 @@ mod tests {
.build()
.unwrap();
let expected = parallel.install(|| layout.tensor(&batch)).unwrap();
- let prepared = prepare_crop_tensor(|| layout.tensor(&batch)).unwrap();
+ let prepared = prepare_crops(|| layout.tensor(&batch)).unwrap();
assert_eq!(prepared, expected);
}
+ #[test]
+ fn argmax_matches_scalar_for_lane_boundaries_ties_and_both_precisions() {
+ fn reference(values: impl FloatTensorData) -> StepBest {
+ let mut best = StepBest {
+ index: 0,
+ probability: f32::NEG_INFINITY,
+ };
+ for index in 0..values.len() {
+ let probability = values.value(index);
+ if probability > best.probability {
+ best = StepBest { index, probability };
+ }
+ }
+ best
+ }
+
+ fn check<'a, T>(values: &'a [T])
+ where
+ &'a [T]: FloatTensorData,
+ {
+ let expected = reference(values);
+ let actual = argmax(values);
+ assert_eq!(actual.index, expected.index);
+ assert_eq!(actual.probability.to_bits(), expected.probability.to_bits());
+ }
+
+ let mut state = 37u32;
+ for len in (0..40).chain([191, 320, REC_VOCABULARY_SIZE]) {
+ let mut values: Vec<f32> = (0..len)
+ .map(|_| {
+ state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+ (state >> 16) as f32 / 65536.0 - 0.5
+ })
+ .collect();
+ for first in 0..len.min(17) {
+ for last in [first, len / 2, len - 1] {
+ values[first] = 1.0;
+ values[last] = 1.0;
+ check(values.as_slice());
+ let half_values: Vec<_> =
+ values.iter().map(|&v| half::f16::from_f32(v)).collect();
+ check(half_values.as_slice());
+ values[first] = -0.25;
+ values[last] = -0.25;
+ }
+ }
+ check(values.as_slice());
+ }
+ for values in [
+ vec![],
+ vec![-0.0, 0.0, -0.0, 0.0, -0.0, 0.0, -0.0, 0.0, 0.0],
+ vec![f32::NEG_INFINITY; 19],
+ vec![f32::NAN; 19],
+ vec![f32::NAN, 0.25, f32::INFINITY, f32::INFINITY],
+ ] {
+ check(values.as_slice());
+ let half_values: Vec<_> = values.iter().map(|&v| half::f16::from_f32(v)).collect();
+ check(half_values.as_slice());
+ }
+ }
+
#[test]
fn borrowed_outputs_preserve_ties_blanks_and_half_precision_confidences() {
let values = [
diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rs
index 62fad88983..53986e025e 100644
--- a/rust/crates/ml/src/ocr/tensor.rs
+++ b/rust/crates/ml/src/ocr/tensor.rs
@@ -5,9 +5,7 @@ use rayon::{ThreadPool, ThreadPoolBuilder};
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
-pub(crate) fn prepare_crop_tensor(
- prepare: impl FnOnce() -> MlResult<Vec<f32>> + Send,
-) -> MlResult<Vec<f32>> {
+pub(crate) fn prepare_crops<T: Send>(prepare: impl FnOnce() -> MlResult<T> + Send) -> MlResult<T> {
static POOL: OnceLock<Option<ThreadPool>> = OnceLock::new();
let pool = POOL.get_or_init(|| {
ThreadPoolBuilder::new()
@@ -89,13 +87,19 @@ mod tests {
fn preparation_keeps_the_callers_worker_pool_unchanged() {
let caller = ThreadPoolBuilder::new().num_threads(4).build().unwrap();
caller.install(|| {
- let values =
- prepare_crop_tensor(|| Ok(vec![rayon::current_num_threads() as f32])).unwrap();
+ let values = prepare_crops(|| Ok(vec![rayon::current_num_threads() as f32])).unwrap();
assert_eq!(values, [1.0]);
assert_eq!(rayon::current_num_threads(), 4);
});
}
+ #[test]
+ fn preparation_reuses_the_worker_across_return_types() {
+ let first = prepare_crops(|| Ok(std::thread::current().id())).unwrap();
+ let second = prepare_crops(|| Ok(vec![std::thread::current().id()])).unwrap();
+ assert_eq!(second, [first]);
+ }
+
#[test]
fn planes_are_bgr_left_aligned_per_row_and_zero_padded() {
let rgb = ImageU8::new(