Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Optimize Rust OCR preprocessing and decoding

laurenspriemdbd249014514
diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rs
index 81882e6b4b..de0adc365e 100644
--- a/rust/crates/ml/src/cv/mod.rs
+++ b/rust/crates/ml/src/cv/mod.rs
@@ -36,7 +36,7 @@ pub(crate) use stats::{
};
pub(crate) use structuring::ellipse_kernel;
pub(crate) use transform::rotate_u8;
-pub(crate) use warp::warp_perspective;
+pub(crate) use warp::{warp_perspective, warp_rgb_perspective};
pub(crate) const PARALLEL_MIN_ELEMS: usize = 200_000;
const PIXELS_PER_CHUNK: usize = 65_536;
diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rs
index 122d8a3f7f..73cfd778c2 100644
--- a/rust/crates/ml/src/cv/warp.rs
+++ b/rust/crates/ml/src/cv/warp.rs
@@ -22,15 +22,29 @@ pub(crate) fn warp_perspective(
src.channels
));
}
+ let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())
+ .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;
+ warp_rgb_perspective(&source, src_corners, dst_corners, width, height)
+}
+
+pub(crate) fn warp_rgb_perspective(
+ source: &RgbImage,
+ src_corners: [(f64, f64); 4],
+ dst_corners: [(f64, f64); 4],
+ width: i32,
+ height: i32,
+) -> OpResult<ImageU8> {
+ if width <= 0 || height <= 0 {
+ return Err(format!(
+ "warp_perspective: invalid destination size {width}x{height}"
+ ));
+ }
let narrow = |c: [(f64, f64); 4]| c.map(|(x, y)| (x as f32, y as f32));
let projection = Projection::from_control_points(narrow(src_corners), narrow(dst_corners))
.ok_or_else(|| "warp_perspective: the corner pairs are degenerate".to_string())?;
-
- let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())
- .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;
let mut out = RgbImage::new(width as u32, height as u32);
warp_into(
- &source,
+ source,
&projection,
Interpolation::Bilinear,
Rgb([0, 0, 0]),
diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rs
index de07e72444..1ef9e2c91f 100644
--- a/rust/crates/ml/src/ocr/classify.rs
+++ b/rust/crates/ml/src/ocr/classify.rs
@@ -1,6 +1,6 @@
use std::sync::{Mutex, PoisonError};
-use super::tensor::{BgrNormalization, write_bgr_planes};
+use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};
use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
@@ -51,7 +51,7 @@ impl AngleClassifier {
}
fn score_batch(&self, batch: &[ImageU8]) -> MlResult<Vec<AngleScores>> {
- let input = PreparedF32Input::new(batch_tensor(batch)?);
+ let input = PreparedF32Input::new(prepare_crop_tensor(|| batch_tensor(batch))?);
let count = batch.len() as i64;
let expected_shape = [count, CLASS_COUNT as i64];
let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
@@ -180,6 +180,26 @@ mod tests {
assert!(!scores(0.15, 0.85).needs_rotation());
}
+ #[test]
+ fn crop_preparation_preserves_resized_classifier_values() {
+ let batch: Vec<_> = [(7, 3), (390, 25), (3, 71)]
+ .into_iter()
+ .map(|(width, height)| {
+ let pixels = (0..width * height * 3)
+ .map(|index| (index * 29 % 256) as u8)
+ .collect();
+ ImageU8::new(width, height, 3, pixels).unwrap()
+ })
+ .collect();
+ let parallel = rayon::ThreadPoolBuilder::new()
+ .num_threads(4)
+ .build()
+ .unwrap();
+ let expected = parallel.install(|| batch_tensor(&batch)).unwrap();
+ let prepared = prepare_crop_tensor(|| batch_tensor(&batch)).unwrap();
+ assert_eq!(prepared, expected);
+ }
+
#[test]
fn decisions_rotate_only_flagged_crops_in_place() {
let mut crop = ImageU8::new(2, 1, 3, vec![1, 2, 3, 4, 5, 6]).unwrap();
diff --git a/rust/crates/ml/src/ocr/crop.rs b/rust/crates/ml/src/ocr/crop.rs
index 2bb7926eaa..2442c17b82 100644
--- a/rust/crates/ml/src/ocr/crop.rs
+++ b/rust/crates/ml/src/ocr/crop.rs
@@ -1,3 +1,5 @@
+use image::RgbImage;
+
use super::Point;
use crate::cv;
use crate::cv::image::ImageU8;
@@ -18,7 +20,7 @@ pub(crate) struct TextCrop {
pub(crate) orientation: Orientation,
}
-pub(crate) fn crop_text(working: &ImageU8, quad: &[Point; 4]) -> MlResult<TextCrop> {
+pub(crate) fn crop_text(working: &RgbImage, quad: &[Point; 4]) -> MlResult<TextCrop> {
let (width, height) = crop_size(quad);
let warped = warp_to_rectangle(working, quad, width, height)?;
if is_vertical(width, height) {
@@ -50,7 +52,7 @@ fn is_vertical(width: i32, height: i32) -> bool {
}
fn warp_to_rectangle(
- working: &ImageU8,
+ working: &RgbImage,
quad: &[Point; 4],
width: i32,
height: i32,
@@ -58,7 +60,7 @@ fn warp_to_rectangle(
let source_corners = quad.map(|p| (f64::from(p.x), f64::from(p.y)));
let (w, h) = (f64::from(width), f64::from(height));
let target_corners = [(0.0, 0.0), (w, 0.0), (w, h), (0.0, h)];
- cv::warp_perspective(working, source_corners, target_corners, width, height)
+ cv::warp_rgb_perspective(working, source_corners, target_corners, width, height)
.map_err(MlError::Preprocess)
}
@@ -74,11 +76,11 @@ mod tests {
points.map(|(x, y)| Point::new(x, y))
}
- fn gradient_image(width: i32, height: i32) -> ImageU8 {
+ fn gradient_image(width: i32, height: i32) -> RgbImage {
let data = (0..height)
.flat_map(|y| (0..width).flat_map(move |x| [x as u8, y as u8, (x + y) as u8]))
.collect();
- ImageU8::new(width, height, 3, data).unwrap()
+ RgbImage::from_raw(width as u32, height as u32, data).unwrap()
}
fn pixel(image: &ImageU8, x: i32, y: i32) -> [u8; 3] {
@@ -115,7 +117,7 @@ mod tests {
for x in 0..20 {
assert_eq!(
pixel(&crop.image, x, y),
- pixel(&source, x + 8, y + 5),
+ source.get_pixel((x + 8) as u32, (y + 5) as u32).0,
"pixel ({x}, {y})"
);
}
@@ -132,9 +134,9 @@ mod tests {
.unwrap();
assert_eq!(crop.orientation, Orientation::Vertical);
assert_eq!((crop.image.width, crop.image.height), (3, 2));
- assert_eq!(pixel(&crop.image, 0, 0), pixel(&source, 5, 2));
- assert_eq!(pixel(&crop.image, 2, 0), pixel(&source, 5, 4));
- assert_eq!(pixel(&crop.image, 0, 1), pixel(&source, 4, 2));
- assert_eq!(pixel(&crop.image, 2, 1), pixel(&source, 4, 4));
+ assert_eq!(pixel(&crop.image, 0, 0), source.get_pixel(5, 2).0);
+ assert_eq!(pixel(&crop.image, 2, 0), source.get_pixel(5, 4).0);
+ assert_eq!(pixel(&crop.image, 0, 1), source.get_pixel(4, 2).0);
+ assert_eq!(pixel(&crop.image, 2, 1), source.get_pixel(4, 4).0);
}
}
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index e2b12c1f4c..2d8295e72d 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -382,9 +382,18 @@ fn crop_candidates(
working: &ImageU8,
candidates: &[DetectionCandidate],
) -> MlResult<Vec<TextCrop>> {
+ if candidates.is_empty() {
+ return Ok(Vec::new());
+ }
+ let source = image::RgbImage::from_raw(
+ working.width as u32,
+ working.height as u32,
+ working.data.clone(),
+ )
+ .ok_or_else(|| MlError::Preprocess("OCR crop source buffer mismatch".to_string()))?;
candidates
.iter()
- .map(|candidate| crop_text(working, &candidate.points))
+ .map(|candidate| crop_text(&source, &candidate.points))
.collect()
}
@@ -564,6 +573,12 @@ mod tests {
}
}
+ #[test]
+ fn no_detected_regions_produce_no_crops() {
+ let image = ImageU8::zeros(32, 32, 3).unwrap();
+ assert!(crop_candidates(&image, &[]).unwrap().is_empty());
+ }
+
#[test]
fn engine_accepts_a_detector_only_path_set_without_loading_it() {
let engine = detector_only_engine();
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index 33ff2ae47d..8443ae3cc7 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -4,11 +4,14 @@ use std::sync::{Mutex, OnceLock, PoisonError};
use super::OcrError;
use super::cancel::RequestGuard;
use super::dictionary::load_dictionary;
-use super::tensor::{BgrNormalization, write_bgr_planes};
+use super::tensor::{BgrNormalization, prepare_crop_tensor, write_bgr_planes};
use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
-use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};
+use crate::onnx::{
+ BorrowedFloatTensor, ExecutionMode, FloatTensorData, OnnxSession, PreparedF32Input,
+ with_prepared_float_output,
+};
const MODEL_NAMESPACE: &str = "ocr-recognition";
const REC_VOCABULARY_SIZE: usize = 18385;
@@ -86,7 +89,7 @@ impl TextRecognizer {
fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {
let layout = BatchLayout::new(batch);
- let input = PreparedF32Input::new(layout.tensor(batch)?);
+ let input = PreparedF32Input::new(prepare_crop_tensor(|| layout.tensor(batch))?);
let count = batch.len();
let input_shape = [
count as i64,
@@ -95,27 +98,45 @@ impl TextRecognizer {
i64::from(layout.target_width),
];
let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
- let (output, _usage) = session.run(|session| {
- let (shape, values) = run_f32(session, &input, input_shape)?;
- SequenceOutput::new(&shape, values, count, dictionary.len())
- .map_err(SessionRunError::from)
- })?;
- Ok(layout
- .content_widths
- .iter()
- .zip(output.sequences())
- .map(|(&content_width, logits)| {
- ctc_decode(
- logits,
- output.vocabulary,
- dictionary,
- layout.padding_scale(content_width),
- )
+ let (recognized, _usage) = session.run(|session| {
+ with_prepared_float_output(session, &input, input_shape, |shape, values| match values {
+ BorrowedFloatTensor::F32(values) => {
+ decode_output(shape, values, &layout, dictionary)
+ }
+ BorrowedFloatTensor::F16(values) => {
+ decode_output(shape, values, &layout, dictionary)
+ }
})
- .collect())
+ })?;
+ Ok(recognized)
}
}
+fn decode_output<'a, T>(
+ shape: &[i64],
+ values: &'a [T],
+ layout: &BatchLayout,
+ dictionary: &[String],
+) -> MlResult<Vec<Recognition>>
+where
+ &'a [T]: FloatTensorData,
+{
+ let output = SequenceOutput::new(shape, values, layout.content_widths.len(), dictionary.len())?;
+ Ok(layout
+ .content_widths
+ .iter()
+ .zip(output.sequences())
+ .map(|(&content_width, logits)| {
+ ctc_decode(
+ logits,
+ output.vocabulary,
+ dictionary,
+ layout.padding_scale(content_width),
+ )
+ })
+ .collect())
+}
+
fn recognize_in_batches(
crops: &[&ImageU8],
request: &RequestGuard<'_>,
@@ -209,14 +230,14 @@ fn content_width(crop: &ImageU8, target_width: i32) -> i32 {
}
#[derive(Debug)]
-struct SequenceOutput {
+struct SequenceOutput<'a, T> {
steps: usize,
vocabulary: usize,
- values: Vec<f32>,
+ values: &'a [T],
}
-impl SequenceOutput {
- fn new(shape: &[i64], values: Vec<f32>, count: usize, vocabulary: usize) -> MlResult<Self> {
+impl<'a, T> SequenceOutput<'a, T> {
+ fn new(shape: &[i64], values: &'a [T], count: usize, vocabulary: usize) -> MlResult<Self> {
match *shape {
[n, steps, v]
if n == count as i64
@@ -237,17 +258,20 @@ impl SequenceOutput {
}
}
- fn sequences(&self) -> impl Iterator<Item = &[f32]> {
+ fn sequences(&self) -> impl Iterator<Item = &'a [T]> {
self.values.chunks_exact(self.steps * self.vocabulary)
}
}
-fn ctc_decode(
- logits: &[f32],
+fn ctc_decode<'a, T>(
+ logits: &'a [T],
vocabulary: usize,
dictionary: &[String],
padding_scale: f32,
-) -> Recognition {
+) -> Recognition
+where
+ &'a [T]: FloatTensorData,
+{
let best = best_per_step(logits, vocabulary);
let spans: Vec<CharacterSpan> = character_runs(&best)
.iter()
@@ -266,19 +290,23 @@ struct StepBest {
probability: f32,
}
-fn best_per_step(logits: &[f32], vocabulary: usize) -> Vec<StepBest> {
+fn best_per_step<'a, T>(logits: &'a [T], vocabulary: usize) -> Vec<StepBest>
+where
+ &'a [T]: FloatTensorData,
+{
if vocabulary == 0 {
return Vec::new();
}
logits.chunks_exact(vocabulary).map(argmax).collect()
}
-fn argmax(step: &[f32]) -> StepBest {
+fn argmax(step: impl FloatTensorData) -> StepBest {
let mut best = StepBest {
index: 0,
probability: f32::NEG_INFINITY,
};
- for (index, &probability) in step.iter().enumerate() {
+ for index in 0..step.len() {
+ let probability = step.value(index);
if probability > best.probability {
best = StepBest { index, probability };
}
@@ -497,17 +525,58 @@ mod tests {
#[test]
fn sequence_output_requires_the_dictionary_vocabulary() {
- let ok = SequenceOutput::new(&[1, 2, 4], vec![0.0; 8], 1, 4).unwrap();
+ let ok = SequenceOutput::new(&[1, 2, 4], &[0.0f32; 8], 1, 4).unwrap();
assert_eq!((ok.steps, ok.vocabulary), (2, 4));
assert_eq!(ok.sequences().count(), 1);
for shape in [[1, 2, 5], [2, 2, 4], [1, 0, 4]] {
- let error = SequenceOutput::new(&shape, vec![0.0; 8], 1, 4).unwrap_err();
+ let error = SequenceOutput::new(&shape, &[0.0f32; 8], 1, 4).unwrap_err();
assert!(matches!(error, MlError::CorruptModel(_)), "{error}");
}
- let short = SequenceOutput::new(&[1, 2, 4], vec![0.0; 7], 1, 4).unwrap_err();
+ let short = SequenceOutput::new(&[1, 2, 4], &[0.0f32; 7], 1, 4).unwrap_err();
assert!(matches!(short, MlError::CorruptModel(_)), "{short}");
}
+ #[test]
+ fn crop_preparation_preserves_resized_batch_values() {
+ let crops: Vec<_> = [(79, 11), (12, 160), (611, 53)]
+ .into_iter()
+ .map(|(width, height)| {
+ let pixels = (0..width * height * 3)
+ .map(|index| (index * 37 % 256) as u8)
+ .collect();
+ ImageU8::new(width, height, 3, pixels).unwrap()
+ })
+ .collect();
+ let batch = crops.iter().collect::<Vec<_>>();
+ let layout = BatchLayout::new(&batch);
+ let parallel = rayon::ThreadPoolBuilder::new()
+ .num_threads(4)
+ .build()
+ .unwrap();
+ let expected = parallel.install(|| layout.tensor(&batch)).unwrap();
+ let prepared = prepare_crop_tensor(|| layout.tensor(&batch)).unwrap();
+ assert_eq!(prepared, expected);
+ }
+
+ #[test]
+ fn borrowed_outputs_preserve_ties_blanks_and_half_precision_confidences() {
+ let values = [
+ 0.0, 0.75, 0.75, 0.0, 0.0, 0.5, 0.0, 0.0, 0.875, 0.0, 0.0, 0.0, 0.0, 0.625, 0.0, 0.0,
+ ];
+ let half_values = values.map(half::f16::from_f32);
+ let layout = BatchLayout {
+ target_width: 320,
+ content_widths: vec![320],
+ };
+ let full = decode_output(&[1, 4, 4], &values, &layout, &dictionary()).unwrap();
+ let half = decode_output(&[1, 4, 4], &half_values, &layout, &dictionary()).unwrap();
+ assert_eq!(full, half);
+ assert_eq!(full[0].text, "aa");
+ assert_span(&full[0].spans[0], "a", 0.625, 0.0, 0.5);
+ assert_span(&full[0].spans[1], "a", 0.625, 0.75, 1.0);
+ assert_eq!(full[0].confidence, 0.625);
+ }
+
#[test]
fn crops_are_batched_by_ascending_aspect_and_restored_to_input_order() {
let widths = [80, 10, 60, 30, 70, 20, 50, 40];
diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rs
index 82cffa9ac8..62fad88983 100644
--- a/rust/crates/ml/src/ocr/tensor.rs
+++ b/rust/crates/ml/src/ocr/tensor.rs
@@ -1,6 +1,27 @@
+use std::sync::OnceLock;
+
+use rayon::{ThreadPool, ThreadPoolBuilder};
+
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
+pub(crate) fn prepare_crop_tensor(
+ prepare: impl FnOnce() -> MlResult<Vec<f32>> + Send,
+) -> MlResult<Vec<f32>> {
+ static POOL: OnceLock<Option<ThreadPool>> = OnceLock::new();
+ let pool = POOL.get_or_init(|| {
+ ThreadPoolBuilder::new()
+ .num_threads(1)
+ .thread_name(|_| "ocr-crop-preparation".to_string())
+ .build()
+ .ok()
+ });
+ match pool {
+ Some(pool) => pool.install(prepare),
+ None => prepare(),
+ }
+}
+
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct BgrNormalization {
mean: [f32; 3],
@@ -64,6 +85,17 @@ mod tests {
(value as f32 / 255.0 - 0.5) / 0.5
}
+ #[test]
+ fn preparation_keeps_the_callers_worker_pool_unchanged() {
+ let caller = ThreadPoolBuilder::new().num_threads(4).build().unwrap();
+ caller.install(|| {
+ let values =
+ prepare_crop_tensor(|| Ok(vec![rayon::current_num_threads() as f32])).unwrap();
+ assert_eq!(values, [1.0]);
+ assert_eq!(rayon::current_num_threads(), 4);
+ });
+ }
+
#[test]
fn planes_are_bgr_left_aligned_per_row_and_zero_padded() {
let rgb = ImageU8::new(