Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Bound Rust OCR recognition memory

laurenspriemda0496c7917d
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index e1f6897eb5..7a30c2b3f8 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -17,6 +17,8 @@ const MODEL_NAMESPACE: &str = "ocr-recognition";
const REC_VOCABULARY_SIZE: usize = 18385;
const REC_HEIGHT: i32 = 48;
const REC_BASE_WIDTH: i32 = 320;
+const REC_MAX_WIDTH: i32 = 7168;
+const REC_BATCH_COLUMNS: usize = 7168;
const REC_BATCH_SIZE: usize = 6;
const BLANK_INDEX: usize = 0;
const MIN_SPAN: f32 = 1e-3;
@@ -84,11 +86,17 @@ impl TextRecognizer {
return Ok(Vec::new());
}
let dictionary = self.dictionary.entries()?;
- recognize_in_batches(crops, request, |batch| self.infer_batch(batch, dictionary))
+ recognize_in_batches(crops, request, |batch, layout| {
+ self.infer_batch(batch, layout, dictionary)
+ })
}
- fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {
- let layout = BatchLayout::new(batch);
+ fn infer_batch(
+ &self,
+ batch: &[&ImageU8],
+ layout: &BatchLayout,
+ dictionary: &[String],
+ ) -> MlResult<Vec<Recognition>> {
let input = PreparedF32Input::new(prepare_crops(|| layout.tensor(batch))?);
let count = batch.len();
let input_shape = [
@@ -101,10 +109,10 @@ impl TextRecognizer {
let (recognized, _usage) = session.run(|session| {
with_prepared_float_output(session, &input, input_shape, |shape, values| match values {
BorrowedFloatTensor::F32(values) => {
- decode_output(shape, values, &layout, dictionary)
+ decode_output(shape, values, layout, dictionary)
}
BorrowedFloatTensor::F16(values) => {
- decode_output(shape, values, &layout, dictionary)
+ decode_output(shape, values, layout, dictionary)
}
})
})?;
@@ -140,23 +148,34 @@ where
fn recognize_in_batches(
crops: &[&ImageU8],
request: &RequestGuard<'_>,
- mut infer_batch: impl FnMut(&[&ImageU8]) -> MlResult<Vec<Recognition>>,
+ mut infer_batch: impl FnMut(&[&ImageU8], &BatchLayout) -> MlResult<Vec<Recognition>>,
) -> Result<Vec<Recognition>, OcrError> {
let mut results: Vec<Option<Recognition>> = vec![None; crops.len()];
for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {
request.check()?;
let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();
- let recognized = infer_batch(&batch)?;
- if recognized.len() != batch.len() {
- return Err(MlError::CorruptModel(format!(
- "text recognizer decoded {} crops out of {}",
- recognized.len(),
- batch.len()
- ))
- .into());
- }
- for (&index, recognition) in indices.iter().zip(recognized) {
- results[index] = Some(recognition);
+ let layout = BatchLayout::new(&batch)?;
+ let batch_size = (REC_BATCH_COLUMNS / layout.target_width as usize).min(REC_BATCH_SIZE);
+ for (offset, batch) in batch.chunks(batch_size).enumerate() {
+ request.check()?;
+ let start = offset * batch_size;
+ let end = start + batch.len();
+ let layout = BatchLayout {
+ target_width: layout.target_width,
+ content_widths: layout.content_widths[start..end].to_vec(),
+ };
+ let recognized = infer_batch(batch, &layout)?;
+ if recognized.len() != batch.len() {
+ return Err(MlError::CorruptModel(format!(
+ "text recognizer decoded {} crops out of {}",
+ recognized.len(),
+ batch.len()
+ ))
+ .into());
+ }
+ for (&index, recognition) in indices[start..end].iter().zip(recognized) {
+ results[index] = Some(recognition);
+ }
}
}
results
@@ -181,7 +200,27 @@ struct BatchLayout {
}
impl BatchLayout {
- fn new(batch: &[&ImageU8]) -> Self {
+ fn new(batch: &[&ImageU8]) -> MlResult<Self> {
+ for crop in batch {
+ let expected = usize::try_from(crop.width)
+ .ok()
+ .zip(usize::try_from(crop.height).ok())
+ .and_then(|(width, height)| width.checked_mul(height))
+ .and_then(|pixels| pixels.checked_mul(3));
+ if crop.width <= 0
+ || crop.height <= 0
+ || crop.channels != 3
+ || expected != Some(crop.data.len())
+ {
+ return Err(MlError::Preprocess(format!(
+ "invalid text crop {}x{}x{} with {} bytes",
+ crop.width,
+ crop.height,
+ crop.channels,
+ crop.data.len()
+ )));
+ }
+ }
let base_ratio = f64::from(REC_BASE_WIDTH) / f64::from(REC_HEIGHT);
let max_wh_ratio = batch
.iter()
@@ -192,10 +231,10 @@ impl BatchLayout {
.iter()
.map(|crop| content_width(crop, target_width))
.collect();
- Self {
+ Ok(Self {
target_width,
content_widths,
- }
+ })
}
fn padding_scale(&self, content_width: i32) -> f32 {
@@ -203,12 +242,33 @@ impl BatchLayout {
}
fn tensor(&self, batch: &[&ImageU8]) -> MlResult<Vec<f32>> {
- let plane = (self.target_width * REC_HEIGHT) as usize;
- let mut tensor = vec![0.0f32; batch.len() * 3 * plane];
+ let columns = batch.len().checked_mul(self.target_width as usize);
+ if !(REC_BASE_WIDTH..=REC_MAX_WIDTH).contains(&self.target_width)
+ || batch.is_empty()
+ || batch.len() > REC_BATCH_SIZE
+ || batch.len() != self.content_widths.len()
+ || columns.is_none_or(|columns| columns > REC_BATCH_COLUMNS)
+ {
+ return Err(MlError::Preprocess(
+ "text recognition batch exceeds its tensor budget".to_string(),
+ ));
+ }
+ let plane = self.target_width as usize * REC_HEIGHT as usize;
+ let mut tensor = Vec::new();
+ let length = batch.len() * 3 * plane;
+ tensor.try_reserve_exact(length).map_err(|error| {
+ MlError::Preprocess(format!("cannot allocate text recognition input: {error}"))
+ })?;
+ tensor.resize(length, 0.0f32);
let slots = tensor.chunks_exact_mut(3 * plane);
for ((crop, &content_width), slot) in batch.iter().zip(&self.content_widths).zip(slots) {
- let resized = cv::resize_u8(crop, content_width, REC_HEIGHT, cv::Interp::Bilinear)
- .map_err(MlError::Preprocess)?;
+ let resized = cv::resize_u8(
+ crop,
+ content_width,
+ content_height(crop),
+ cv::Interp::Bilinear,
+ )
+ .map_err(MlError::Preprocess)?;
write_bgr_planes(
&resized,
slot,
@@ -221,14 +281,20 @@ impl BatchLayout {
}
fn target_width(max_wh_ratio: f64) -> i32 {
- ((f64::from(REC_HEIGHT) * max_wh_ratio).trunc() as i32).max(1)
+ ((f64::from(REC_HEIGHT) * max_wh_ratio).trunc() as i32).clamp(1, REC_MAX_WIDTH)
}
fn content_width(crop: &ImageU8, target_width: i32) -> i32 {
- let scaled = (f64::from(REC_HEIGHT * crop.width) / f64::from(crop.height)).ceil() as i32;
+ let scaled =
+ (f64::from(REC_HEIGHT) * f64::from(crop.width) / f64::from(crop.height)).ceil() as i32;
scaled.min(target_width)
}
+fn content_height(crop: &ImageU8) -> i32 {
+ ((f64::from(REC_MAX_WIDTH) * f64::from(crop.height) / f64::from(crop.width)).round() as i32)
+ .clamp(1, REC_HEIGHT)
+}
+
#[derive(Debug)]
struct SequenceOutput<'a, T> {
steps: usize,
@@ -504,13 +570,13 @@ mod tests {
solid(400, 48, [0; 3]),
solid(7, 3, [0; 3]),
];
- let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());
+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>()).unwrap();
assert_eq!(layout.target_width, 400);
assert_eq!(layout.content_widths, [100, 400, 112]);
assert_close(layout.padding_scale(100), 4.0);
let narrow = [solid(10, 48, [0; 3])];
- let layout = BatchLayout::new(&narrow.iter().collect::<Vec<_>>());
+ let layout = BatchLayout::new(&narrow.iter().collect::<Vec<_>>()).unwrap();
assert_eq!(layout.target_width, 320);
assert_eq!(layout.content_widths, [10]);
}
@@ -518,7 +584,7 @@ mod tests {
#[test]
fn batch_tensor_left_aligns_each_crop_and_zero_pads_the_rest() {
let crops = [solid(100, 48, [10, 20, 30]), solid(400, 48, [0; 3])];
- let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());
+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>()).unwrap();
let tensor = layout.tensor(&crops.iter().collect::<Vec<_>>()).unwrap();
let plane = 48 * 400;
assert_eq!(tensor.len(), 2 * 3 * plane);
@@ -563,7 +629,7 @@ mod tests {
})
.collect();
let batch = crops.iter().collect::<Vec<_>>();
- let layout = BatchLayout::new(&batch);
+ let layout = BatchLayout::new(&batch).unwrap();
let parallel = rayon::ThreadPoolBuilder::new()
.num_threads(4)
.build()
@@ -661,17 +727,18 @@ mod tests {
let request = registry.begin(None);
let mut batches = Vec::new();
- let results = recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch| {
- batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());
- Ok(batch
- .iter()
- .map(|crop| Recognition {
- text: crop.width.to_string(),
- ..Recognition::default()
- })
- .collect())
- })
- .unwrap();
+ let results =
+ recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch, _| {
+ batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());
+ Ok(batch
+ .iter()
+ .map(|crop| Recognition {
+ text: crop.width.to_string(),
+ ..Recognition::default()
+ })
+ .collect())
+ })
+ .unwrap();
assert_eq!(batches, [vec![10, 20, 30, 40, 50, 60], vec![70, 80]]);
let texts: Vec<String> = results.into_iter().map(|r| r.text).collect();
@@ -686,8 +753,250 @@ mod tests {
let request = registry.begin(Some("cancelled"));
registry.cancel("cancelled");
- let error = recognize_in_batches(&[&crop], &request, |_| unreachable!()).unwrap_err();
+ let error = recognize_in_batches(&[&crop], &request, |_, _| unreachable!()).unwrap_err();
assert!(matches!(error, OcrError::Cancelled), "{error}");
}
+
+ fn legacy_tensor(batch: &[&ImageU8]) -> Vec<f32> {
+ let ratio = batch
+ .iter()
+ .map(|crop| aspect(crop))
+ .fold(320.0 / 48.0, f64::max);
+ let width = (48.0 * ratio).trunc() as usize;
+ let plane = width * 48;
+ let mut tensor = vec![0.0; batch.len() * 3 * plane];
+ for (crop, slot) in batch.iter().zip(tensor.chunks_exact_mut(3 * plane)) {
+ let content_width = ((48 * crop.width) as f64 / crop.height as f64).ceil() as i32;
+ let resized = cv::resize_u8(
+ crop,
+ content_width.min(width as i32),
+ 48,
+ cv::Interp::Bilinear,
+ )
+ .unwrap();
+ for (y, row) in resized
+ .data
+ .chunks_exact(resized.width as usize * 3)
+ .enumerate()
+ {
+ for (x, pixel) in row.as_chunks::<3>().0.iter().enumerate() {
+ for channel in 0..3 {
+ slot[channel * plane + y * width + x] =
+ (pixel[2 - channel] as f32 / 255.0 - 0.5) / 0.5;
+ }
+ }
+ }
+ }
+ tensor
+ }
+
+ #[test]
+ fn bounded_batches_preserve_ordinary_pixels_padding_and_input_order() {
+ for sizes in [
+ vec![(79, 11), (12, 160), (611, 53)],
+ vec![
+ (1381, 48),
+ (1200, 48),
+ (287, 10),
+ (1300, 48),
+ (1100, 48),
+ (1350, 48),
+ ],
+ vec![(7167, 48), (7168, 48), (6281, 48)],
+ ] {
+ let crops: Vec<_> = sizes
+ .into_iter()
+ .map(|(width, height)| {
+ ImageU8::new(
+ width,
+ height,
+ 3,
+ (0..width * height * 3)
+ .map(|index| (index * 37 % 256) as u8)
+ .collect(),
+ )
+ .unwrap()
+ })
+ .collect();
+ let references: Vec<_> = crops.iter().collect();
+ let order = ascending_aspect_order(&references);
+ let sorted: Vec<_> = order.iter().map(|&index| references[index]).collect();
+ let expected = legacy_tensor(&sorted);
+ let registry = RequestRegistry::default();
+ let request = registry.begin(None);
+ let mut actual = Vec::new();
+ let mut counts = Vec::new();
+ let results = recognize_in_batches(&references, &request, |batch, layout| {
+ assert!(batch.len() <= REC_BATCH_SIZE);
+ assert!(batch.len() * layout.target_width as usize <= REC_BATCH_COLUMNS);
+ counts.push(batch.len());
+ actual.extend(layout.tensor(batch)?);
+ Ok(batch
+ .iter()
+ .map(|crop| Recognition {
+ text: crop.width.to_string(),
+ ..Recognition::default()
+ })
+ .collect())
+ })
+ .unwrap();
+ assert_eq!(actual, expected);
+ assert_eq!(
+ results
+ .iter()
+ .map(|result| result.text.clone())
+ .collect::<Vec<_>>(),
+ crops
+ .iter()
+ .map(|crop| crop.width.to_string())
+ .collect::<Vec<_>>()
+ );
+ match crops.len() {
+ 6 => assert_eq!(counts, [5, 1]),
+ _ if crops[0].width == 7167 => assert_eq!(counts, [1, 1, 1]),
+ _ => assert_eq!(counts, [3]),
+ }
+ }
+ }
+
+ #[test]
+ fn oversized_crops_fit_proportionally_and_keep_all_columns_with_vertical_padding() {
+ for (width, height, resized_height) in [
+ (7168, 48, 48),
+ (7169, 48, 48),
+ (8192, 48, 42),
+ (14336, 48, 24),
+ (4096, 1, 2),
+ (200000, 1, 1),
+ ] {
+ let mut crop = solid(width, height, [0; 3]);
+ for row in crop.data.chunks_exact_mut(width as usize * 3) {
+ row[width as usize / 2 * 3..].fill(255);
+ }
+ let layout = BatchLayout::new(&[&crop]).unwrap();
+ assert_eq!(layout.target_width, REC_MAX_WIDTH);
+ assert_eq!(layout.content_widths, [REC_MAX_WIDTH]);
+ assert_eq!(content_height(&crop), resized_height);
+ assert_eq!(layout.padding_scale(REC_MAX_WIDTH), 1.0);
+ let tensor = layout.tensor(&[&crop]).unwrap();
+ assert_eq!(tensor.len(), 3 * 48 * 7168);
+ let top = (48 - resized_height) as usize / 2;
+ for plane in tensor.as_chunks::<{ 48 * 7168 }>().0 {
+ for (y, row) in plane.as_chunks::<7168>().0.iter().enumerate() {
+ if (top..top + resized_height as usize).contains(&y) {
+ assert_eq!(row[0], -1.0);
+ assert_eq!(row[7167], 1.0);
+ } else {
+ assert!(row.iter().all(|&value| value == 0.0));
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn invalid_crop_geometry_is_rejected_before_tensor_allocation() {
+ for (width, height, channels, len) in [
+ (0, 48, 3, 0),
+ (320, -1, 3, 0),
+ (1, 1, 4, 4),
+ (1, 1, 3, 2),
+ (i32::MAX, i32::MAX, 3, 0),
+ ] {
+ let crop = ImageU8 {
+ width,
+ height,
+ channels,
+ data: vec![0; len],
+ };
+ assert!(matches!(
+ BatchLayout::new(&[&crop]),
+ Err(MlError::Preprocess(_))
+ ));
+ }
+ let geometry = ImageU8 {
+ width: i32::MAX,
+ height: 1,
+ channels: 3,
+ data: Vec::new(),
+ };
+ assert_eq!(target_width(aspect(&geometry)), REC_MAX_WIDTH);
+ assert_eq!(content_width(&geometry, REC_MAX_WIDTH), REC_MAX_WIDTH);
+ assert_eq!(content_height(&geometry), 1);
+ }
+
+ #[test]
+ fn vertical_padding_keeps_character_spans_on_the_original_quad_in_both_orientations() {
+ use super::super::characters::character_boxes;
+ use super::super::{Orientation, Point};
+
+ let crop = solid(4096, 1, [0; 3]);
+ let layout = BatchLayout::new(&[&crop]).unwrap();
+ let values = logits(4, &[(0, 1.0), (1, 0.9), (0, 1.0), (0, 1.0)]);
+ let results = decode_output(&[1, 4, 4], &values, &layout, &dictionary()).unwrap();
+ assert_span(&results[0].spans[0], "a", 0.9, 0.25, 0.5);
+ for rotated in [false, true] {
+ for orientation in [Orientation::Horizontal, Orientation::Vertical] {
+ let (width, height) = match orientation {
+ Orientation::Horizontal => (4096.0, 1.0),
+ Orientation::Vertical => (1.0, 4096.0),
+ };
+ let quad = [
+ Point::new(0.0, 0.0),
+ Point::new(width, 0.0),
+ Point::new(width, height),
+ Point::new(0.0, height),
+ ];
+ let boxes = character_boxes(&quad, &results[0].spans, orientation, rotated);
+ assert_eq!(boxes.len(), 1);
+ let (start, end) = if rotated {
+ (2048.0, 3072.0)
+ } else {
+ (1024.0, 2048.0)
+ };
+ let points = boxes[0].points;
+ match orientation {
+ Orientation::Horizontal => {
+ assert_eq!(points[0], Point::new(start, 0.0));
+ assert_eq!(points[2], Point::new(end, 1.0));
+ }
+ Orientation::Vertical => {
+ assert_eq!(points[0], Point::new(0.0, start));
+ assert_eq!(points[2], Point::new(1.0, end));
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn extreme_aspect_batches_are_bounded_and_cancellation_stops_the_next_split() {
+ let crop = solid(4096, 1, [0; 3]);
+ let crops = [&crop; 6];
+ let layout = BatchLayout::new(&crops).unwrap();
+ assert!(matches!(layout.tensor(&crops), Err(MlError::Preprocess(_))));
+ let registry = RequestRegistry::default();
+ let request = registry.begin(None);
+ let mut calls = 0;
+ recognize_in_batches(&crops, &request, |batch, layout| {
+ calls += 1;
+ assert_eq!(batch.len(), 1);
+ assert_eq!(layout.tensor(batch)?.len(), 3 * 48 * 7168);
+ Ok(vec![Recognition::default()])
+ })
+ .unwrap();
+ assert_eq!(calls, 6);
+
+ let request = registry.begin(Some("split"));
+ calls = 0;
+ let error = recognize_in_batches(&crops, &request, |_, _| {
+ calls += 1;
+ registry.cancel("split");
+ Ok(vec![Recognition::default()])
+ })
+ .unwrap_err();
+ assert_eq!(calls, 1);
+ assert!(matches!(error, OcrError::Cancelled));
+ }
}
diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rs
index 53986e025e..0f8c976d7b 100644
--- a/rust/crates/ml/src/ocr/tensor.rs
+++ b/rust/crates/ml/src/ocr/tensor.rs
@@ -50,8 +50,21 @@ pub(crate) fn write_bgr_planes(
) -> MlResult<()> {
let width = rgb.width as usize;
let height = rgb.height as usize;
- let plane = plane_width * height;
- if rgb.channels != 3 || width > plane_width || planes.len() != 3 * plane {
+ let plane = planes.len() / 3;
+ let plane_height = plane.checked_div(plane_width).unwrap_or(0);
+ let expected = width
+ .checked_mul(height)
+ .and_then(|size| size.checked_mul(3));
+ if rgb.width <= 0
+ || rgb.height <= 0
+ || rgb.channels != 3
+ || width > plane_width
+ || height > plane_height
+ || !planes.len().is_multiple_of(3)
+ || plane_width == 0
+ || !plane.is_multiple_of(plane_width)
+ || expected != Some(rgb.data.len())
+ {
return Err(MlError::Preprocess(format!(
"cannot write a {width}x{height}x{} image into {} values as {plane_width}-wide BGR planes",
rgb.channels,
@@ -60,8 +73,14 @@ pub(crate) fn write_bgr_planes(
}
let (blue, rest) = planes.split_at_mut(plane);
let (green, red) = rest.split_at_mut(plane);
+ let top = (plane_height - height) / 2 * plane_width;
+ let bottom = top + height * plane_width;
+ for channel in [&mut *blue, &mut *green, &mut *red] {
+ channel[..top].fill(0.0);
+ channel[bottom..].fill(0.0);
+ }
for (y, row) in rgb.data.chunks_exact(width * 3).enumerate() {
- let offset = y * plane_width;
+ let offset = top + y * plane_width;
for (x, px) in row.as_chunks::<3>().0.iter().enumerate() {
blue[offset + x] = normalization.apply(0, px[2]);
green[offset + x] = normalization.apply(1, px[1]);
@@ -146,4 +165,34 @@ mod tests {
assert!((actual - expected).abs() <= 1e-6, "{planes:?}");
}
}
+
+ #[test]
+ fn smaller_images_are_centered_vertically_with_zero_padding_on_every_side() {
+ let rgb = ImageU8::new(1, 2, 3, vec![10, 20, 30, 40, 50, 60]).unwrap();
+ for height in [4, 5] {
+ let mut planes = vec![f32::NAN; 3 * 2 * height];
+ write_bgr_planes(&rgb, &mut planes, 2, BgrNormalization::CENTERED).unwrap();
+ for (channel, plane) in planes.chunks_exact(2 * height).enumerate() {
+ for (y, row) in plane.as_chunks::<2>().0.iter().enumerate() {
+ let expected = match y {
+ 1 => centered([30, 20, 10][channel]),
+ 2 => centered([60, 50, 40][channel]),
+ _ => 0.0,
+ };
+ assert_eq!(*row, [expected, 0.0]);
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn invalid_destination_planes_return_an_error() {
+ let rgb = ImageU8::new(2, 2, 3, vec![0; 12]).unwrap();
+ for (width, len) in [(0, 12), (1, 12), (2, 6), (2, 13), (3, 12)] {
+ assert!(
+ write_bgr_planes(&rgb, &mut vec![0.0; len], width, BgrNormalization::CENTERED,)
+ .is_err()
+ );
+ }
+ }
}