π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][photos] Bound Rust OCR recognition memory
laurenspriemda0496c7917d
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex e1f6897eb5..7a30c2b3f8 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -17,6 +17,8 @@ const MODEL_NAMESPACE: &str = "ocr-recognition";const REC_VOCABULARY_SIZE: usize = 18385;const REC_HEIGHT: i32 = 48;const REC_BASE_WIDTH: i32 = 320;+const REC_MAX_WIDTH: i32 = 7168;+const REC_BATCH_COLUMNS: usize = 7168;const REC_BATCH_SIZE: usize = 6;const BLANK_INDEX: usize = 0;const MIN_SPAN: f32 = 1e-3;@@ -84,11 +86,17 @@ impl TextRecognizer {return Ok(Vec::new());}let dictionary = self.dictionary.entries()?;- recognize_in_batches(crops, request, |batch| self.infer_batch(batch, dictionary))+ recognize_in_batches(crops, request, |batch, layout| {+ self.infer_batch(batch, layout, dictionary)+ })}- fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {- let layout = BatchLayout::new(batch);+ fn infer_batch(+ &self,+ batch: &[&ImageU8],+ layout: &BatchLayout,+ dictionary: &[String],+ ) -> MlResult<Vec<Recognition>> {let input = PreparedF32Input::new(prepare_crops(|| layout.tensor(batch))?);let count = batch.len();let input_shape = [@@ -101,10 +109,10 @@ impl TextRecognizer {let (recognized, _usage) = session.run(|session| {with_prepared_float_output(session, &input, input_shape, |shape, values| match values {BorrowedFloatTensor::F32(values) => {- decode_output(shape, values, &layout, dictionary)+ decode_output(shape, values, layout, dictionary)}BorrowedFloatTensor::F16(values) => {- decode_output(shape, values, &layout, dictionary)+ decode_output(shape, values, layout, dictionary)}})})?;@@ -140,23 +148,34 @@ wherefn recognize_in_batches(crops: &[&ImageU8],request: &RequestGuard<'_>,- mut infer_batch: impl FnMut(&[&ImageU8]) -> MlResult<Vec<Recognition>>,+ mut infer_batch: impl FnMut(&[&ImageU8], &BatchLayout) -> MlResult<Vec<Recognition>>,) -> Result<Vec<Recognition>, OcrError> {let mut results: Vec<Option<Recognition>> = vec![None; crops.len()];for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {request.check()?;let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();- let recognized = infer_batch(&batch)?;- if recognized.len() != batch.len() {- return Err(MlError::CorruptModel(format!(- "text recognizer decoded {} crops out of {}",- recognized.len(),- batch.len()- ))- .into());- }- for (&index, recognition) in indices.iter().zip(recognized) {- results[index] = Some(recognition);+ let layout = BatchLayout::new(&batch)?;+ let batch_size = (REC_BATCH_COLUMNS / layout.target_width as usize).min(REC_BATCH_SIZE);+ for (offset, batch) in batch.chunks(batch_size).enumerate() {+ request.check()?;+ let start = offset * batch_size;+ let end = start + batch.len();+ let layout = BatchLayout {+ target_width: layout.target_width,+ content_widths: layout.content_widths[start..end].to_vec(),+ };+ let recognized = infer_batch(batch, &layout)?;+ if recognized.len() != batch.len() {+ return Err(MlError::CorruptModel(format!(+ "text recognizer decoded {} crops out of {}",+ recognized.len(),+ batch.len()+ ))+ .into());+ }+ for (&index, recognition) in indices[start..end].iter().zip(recognized) {+ results[index] = Some(recognition);+ }}}results@@ -181,7 +200,27 @@ struct BatchLayout {}impl BatchLayout {- fn new(batch: &[&ImageU8]) -> Self {+ fn new(batch: &[&ImageU8]) -> MlResult<Self> {+ for crop in batch {+ let expected = usize::try_from(crop.width)+ .ok()+ .zip(usize::try_from(crop.height).ok())+ .and_then(|(width, height)| width.checked_mul(height))+ .and_then(|pixels| pixels.checked_mul(3));+ if crop.width <= 0+ || crop.height <= 0+ || crop.channels != 3+ || expected != Some(crop.data.len())+ {+ return Err(MlError::Preprocess(format!(+ "invalid text crop {}x{}x{} with {} bytes",+ crop.width,+ crop.height,+ crop.channels,+ crop.data.len()+ )));+ }+ }let base_ratio = f64::from(REC_BASE_WIDTH) / f64::from(REC_HEIGHT);let max_wh_ratio = batch.iter()@@ -192,10 +231,10 @@ impl BatchLayout {.iter().map(|crop| content_width(crop, target_width)).collect();- Self {+ Ok(Self {target_width,content_widths,- }+ })}fn padding_scale(&self, content_width: i32) -> f32 {@@ -203,12 +242,33 @@ impl BatchLayout {}fn tensor(&self, batch: &[&ImageU8]) -> MlResult<Vec<f32>> {- let plane = (self.target_width * REC_HEIGHT) as usize;- let mut tensor = vec![0.0f32; batch.len() * 3 * plane];+ let columns = batch.len().checked_mul(self.target_width as usize);+ if !(REC_BASE_WIDTH..=REC_MAX_WIDTH).contains(&self.target_width)+ || batch.is_empty()+ || batch.len() > REC_BATCH_SIZE+ || batch.len() != self.content_widths.len()+ || columns.is_none_or(|columns| columns > REC_BATCH_COLUMNS)+ {+ return Err(MlError::Preprocess(+ "text recognition batch exceeds its tensor budget".to_string(),+ ));+ }+ let plane = self.target_width as usize * REC_HEIGHT as usize;+ let mut tensor = Vec::new();+ let length = batch.len() * 3 * plane;+ tensor.try_reserve_exact(length).map_err(|error| {+ MlError::Preprocess(format!("cannot allocate text recognition input: {error}"))+ })?;+ tensor.resize(length, 0.0f32);let slots = tensor.chunks_exact_mut(3 * plane);for ((crop, &content_width), slot) in batch.iter().zip(&self.content_widths).zip(slots) {- let resized = cv::resize_u8(crop, content_width, REC_HEIGHT, cv::Interp::Bilinear)- .map_err(MlError::Preprocess)?;+ let resized = cv::resize_u8(+ crop,+ content_width,+ content_height(crop),+ cv::Interp::Bilinear,+ )+ .map_err(MlError::Preprocess)?;write_bgr_planes(&resized,slot,@@ -221,14 +281,20 @@ impl BatchLayout {}fn target_width(max_wh_ratio: f64) -> i32 {- ((f64::from(REC_HEIGHT) * max_wh_ratio).trunc() as i32).max(1)+ ((f64::from(REC_HEIGHT) * max_wh_ratio).trunc() as i32).clamp(1, REC_MAX_WIDTH)}fn content_width(crop: &ImageU8, target_width: i32) -> i32 {- let scaled = (f64::from(REC_HEIGHT * crop.width) / f64::from(crop.height)).ceil() as i32;+ let scaled =+ (f64::from(REC_HEIGHT) * f64::from(crop.width) / f64::from(crop.height)).ceil() as i32;scaled.min(target_width)}+fn content_height(crop: &ImageU8) -> i32 {+ ((f64::from(REC_MAX_WIDTH) * f64::from(crop.height) / f64::from(crop.width)).round() as i32)+ .clamp(1, REC_HEIGHT)+}+#[derive(Debug)]struct SequenceOutput<'a, T> {steps: usize,@@ -504,13 +570,13 @@ mod tests {solid(400, 48, [0; 3]),solid(7, 3, [0; 3]),];- let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>()).unwrap();assert_eq!(layout.target_width, 400);assert_eq!(layout.content_widths, [100, 400, 112]);assert_close(layout.padding_scale(100), 4.0);let narrow = [solid(10, 48, [0; 3])];- let layout = BatchLayout::new(&narrow.iter().collect::<Vec<_>>());+ let layout = BatchLayout::new(&narrow.iter().collect::<Vec<_>>()).unwrap();assert_eq!(layout.target_width, 320);assert_eq!(layout.content_widths, [10]);}@@ -518,7 +584,7 @@ mod tests {#[test]fn batch_tensor_left_aligns_each_crop_and_zero_pads_the_rest() {let crops = [solid(100, 48, [10, 20, 30]), solid(400, 48, [0; 3])];- let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>()).unwrap();let tensor = layout.tensor(&crops.iter().collect::<Vec<_>>()).unwrap();let plane = 48 * 400;assert_eq!(tensor.len(), 2 * 3 * plane);@@ -563,7 +629,7 @@ mod tests {}).collect();let batch = crops.iter().collect::<Vec<_>>();- let layout = BatchLayout::new(&batch);+ let layout = BatchLayout::new(&batch).unwrap();let parallel = rayon::ThreadPoolBuilder::new().num_threads(4).build()@@ -661,17 +727,18 @@ mod tests {let request = registry.begin(None);let mut batches = Vec::new();- let results = recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch| {- batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());- Ok(batch- .iter()- .map(|crop| Recognition {- text: crop.width.to_string(),- ..Recognition::default()- })- .collect())- })- .unwrap();+ let results =+ recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch, _| {+ batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());+ Ok(batch+ .iter()+ .map(|crop| Recognition {+ text: crop.width.to_string(),+ ..Recognition::default()+ })+ .collect())+ })+ .unwrap();assert_eq!(batches, [vec![10, 20, 30, 40, 50, 60], vec![70, 80]]);let texts: Vec<String> = results.into_iter().map(|r| r.text).collect();@@ -686,8 +753,250 @@ mod tests {let request = registry.begin(Some("cancelled"));registry.cancel("cancelled");- let error = recognize_in_batches(&[&crop], &request, |_| unreachable!()).unwrap_err();+ let error = recognize_in_batches(&[&crop], &request, |_, _| unreachable!()).unwrap_err();assert!(matches!(error, OcrError::Cancelled), "{error}");}++ fn legacy_tensor(batch: &[&ImageU8]) -> Vec<f32> {+ let ratio = batch+ .iter()+ .map(|crop| aspect(crop))+ .fold(320.0 / 48.0, f64::max);+ let width = (48.0 * ratio).trunc() as usize;+ let plane = width * 48;+ let mut tensor = vec![0.0; batch.len() * 3 * plane];+ for (crop, slot) in batch.iter().zip(tensor.chunks_exact_mut(3 * plane)) {+ let content_width = ((48 * crop.width) as f64 / crop.height as f64).ceil() as i32;+ let resized = cv::resize_u8(+ crop,+ content_width.min(width as i32),+ 48,+ cv::Interp::Bilinear,+ )+ .unwrap();+ for (y, row) in resized+ .data+ .chunks_exact(resized.width as usize * 3)+ .enumerate()+ {+ for (x, pixel) in row.as_chunks::<3>().0.iter().enumerate() {+ for channel in 0..3 {+ slot[channel * plane + y * width + x] =+ (pixel[2 - channel] as f32 / 255.0 - 0.5) / 0.5;+ }+ }+ }+ }+ tensor+ }++ #[test]+ fn bounded_batches_preserve_ordinary_pixels_padding_and_input_order() {+ for sizes in [+ vec![(79, 11), (12, 160), (611, 53)],+ vec![+ (1381, 48),+ (1200, 48),+ (287, 10),+ (1300, 48),+ (1100, 48),+ (1350, 48),+ ],+ vec![(7167, 48), (7168, 48), (6281, 48)],+ ] {+ let crops: Vec<_> = sizes+ .into_iter()+ .map(|(width, height)| {+ ImageU8::new(+ width,+ height,+ 3,+ (0..width * height * 3)+ .map(|index| (index * 37 % 256) as u8)+ .collect(),+ )+ .unwrap()+ })+ .collect();+ let references: Vec<_> = crops.iter().collect();+ let order = ascending_aspect_order(&references);+ let sorted: Vec<_> = order.iter().map(|&index| references[index]).collect();+ let expected = legacy_tensor(&sorted);+ let registry = RequestRegistry::default();+ let request = registry.begin(None);+ let mut actual = Vec::new();+ let mut counts = Vec::new();+ let results = recognize_in_batches(&references, &request, |batch, layout| {+ assert!(batch.len() <= REC_BATCH_SIZE);+ assert!(batch.len() * layout.target_width as usize <= REC_BATCH_COLUMNS);+ counts.push(batch.len());+ actual.extend(layout.tensor(batch)?);+ Ok(batch+ .iter()+ .map(|crop| Recognition {+ text: crop.width.to_string(),+ ..Recognition::default()+ })+ .collect())+ })+ .unwrap();+ assert_eq!(actual, expected);+ assert_eq!(+ results+ .iter()+ .map(|result| result.text.clone())+ .collect::<Vec<_>>(),+ crops+ .iter()+ .map(|crop| crop.width.to_string())+ .collect::<Vec<_>>()+ );+ match crops.len() {+ 6 => assert_eq!(counts, [5, 1]),+ _ if crops[0].width == 7167 => assert_eq!(counts, [1, 1, 1]),+ _ => assert_eq!(counts, [3]),+ }+ }+ }++ #[test]+ fn oversized_crops_fit_proportionally_and_keep_all_columns_with_vertical_padding() {+ for (width, height, resized_height) in [+ (7168, 48, 48),+ (7169, 48, 48),+ (8192, 48, 42),+ (14336, 48, 24),+ (4096, 1, 2),+ (200000, 1, 1),+ ] {+ let mut crop = solid(width, height, [0; 3]);+ for row in crop.data.chunks_exact_mut(width as usize * 3) {+ row[width as usize / 2 * 3..].fill(255);+ }+ let layout = BatchLayout::new(&[&crop]).unwrap();+ assert_eq!(layout.target_width, REC_MAX_WIDTH);+ assert_eq!(layout.content_widths, [REC_MAX_WIDTH]);+ assert_eq!(content_height(&crop), resized_height);+ assert_eq!(layout.padding_scale(REC_MAX_WIDTH), 1.0);+ let tensor = layout.tensor(&[&crop]).unwrap();+ assert_eq!(tensor.len(), 3 * 48 * 7168);+ let top = (48 - resized_height) as usize / 2;+ for plane in tensor.as_chunks::<{ 48 * 7168 }>().0 {+ for (y, row) in plane.as_chunks::<7168>().0.iter().enumerate() {+ if (top..top + resized_height as usize).contains(&y) {+ assert_eq!(row[0], -1.0);+ assert_eq!(row[7167], 1.0);+ } else {+ assert!(row.iter().all(|&value| value == 0.0));+ }+ }+ }+ }+ }++ #[test]+ fn invalid_crop_geometry_is_rejected_before_tensor_allocation() {+ for (width, height, channels, len) in [+ (0, 48, 3, 0),+ (320, -1, 3, 0),+ (1, 1, 4, 4),+ (1, 1, 3, 2),+ (i32::MAX, i32::MAX, 3, 0),+ ] {+ let crop = ImageU8 {+ width,+ height,+ channels,+ data: vec![0; len],+ };+ assert!(matches!(+ BatchLayout::new(&[&crop]),+ Err(MlError::Preprocess(_))+ ));+ }+ let geometry = ImageU8 {+ width: i32::MAX,+ height: 1,+ channels: 3,+ data: Vec::new(),+ };+ assert_eq!(target_width(aspect(&geometry)), REC_MAX_WIDTH);+ assert_eq!(content_width(&geometry, REC_MAX_WIDTH), REC_MAX_WIDTH);+ assert_eq!(content_height(&geometry), 1);+ }++ #[test]+ fn vertical_padding_keeps_character_spans_on_the_original_quad_in_both_orientations() {+ use super::super::characters::character_boxes;+ use super::super::{Orientation, Point};++ let crop = solid(4096, 1, [0; 3]);+ let layout = BatchLayout::new(&[&crop]).unwrap();+ let values = logits(4, &[(0, 1.0), (1, 0.9), (0, 1.0), (0, 1.0)]);+ let results = decode_output(&[1, 4, 4], &values, &layout, &dictionary()).unwrap();+ assert_span(&results[0].spans[0], "a", 0.9, 0.25, 0.5);+ for rotated in [false, true] {+ for orientation in [Orientation::Horizontal, Orientation::Vertical] {+ let (width, height) = match orientation {+ Orientation::Horizontal => (4096.0, 1.0),+ Orientation::Vertical => (1.0, 4096.0),+ };+ let quad = [+ Point::new(0.0, 0.0),+ Point::new(width, 0.0),+ Point::new(width, height),+ Point::new(0.0, height),+ ];+ let boxes = character_boxes(&quad, &results[0].spans, orientation, rotated);+ assert_eq!(boxes.len(), 1);+ let (start, end) = if rotated {+ (2048.0, 3072.0)+ } else {+ (1024.0, 2048.0)+ };+ let points = boxes[0].points;+ match orientation {+ Orientation::Horizontal => {+ assert_eq!(points[0], Point::new(start, 0.0));+ assert_eq!(points[2], Point::new(end, 1.0));+ }+ Orientation::Vertical => {+ assert_eq!(points[0], Point::new(0.0, start));+ assert_eq!(points[2], Point::new(1.0, end));+ }+ }+ }+ }+ }++ #[test]+ fn extreme_aspect_batches_are_bounded_and_cancellation_stops_the_next_split() {+ let crop = solid(4096, 1, [0; 3]);+ let crops = [&crop; 6];+ let layout = BatchLayout::new(&crops).unwrap();+ assert!(matches!(layout.tensor(&crops), Err(MlError::Preprocess(_))));+ let registry = RequestRegistry::default();+ let request = registry.begin(None);+ let mut calls = 0;+ recognize_in_batches(&crops, &request, |batch, layout| {+ calls += 1;+ assert_eq!(batch.len(), 1);+ assert_eq!(layout.tensor(batch)?.len(), 3 * 48 * 7168);+ Ok(vec![Recognition::default()])+ })+ .unwrap();+ assert_eq!(calls, 6);++ let request = registry.begin(Some("split"));+ calls = 0;+ let error = recognize_in_batches(&crops, &request, |_, _| {+ calls += 1;+ registry.cancel("split");+ Ok(vec![Recognition::default()])+ })+ .unwrap_err();+ assert_eq!(calls, 1);+ assert!(matches!(error, OcrError::Cancelled));+ }}diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rsindex 53986e025e..0f8c976d7b 100644--- a/rust/crates/ml/src/ocr/tensor.rs+++ b/rust/crates/ml/src/ocr/tensor.rs@@ -50,8 +50,21 @@ pub(crate) fn write_bgr_planes() -> MlResult<()> {let width = rgb.width as usize;let height = rgb.height as usize;- let plane = plane_width * height;- if rgb.channels != 3 || width > plane_width || planes.len() != 3 * plane {+ let plane = planes.len() / 3;+ let plane_height = plane.checked_div(plane_width).unwrap_or(0);+ let expected = width+ .checked_mul(height)+ .and_then(|size| size.checked_mul(3));+ if rgb.width <= 0+ || rgb.height <= 0+ || rgb.channels != 3+ || width > plane_width+ || height > plane_height+ || !planes.len().is_multiple_of(3)+ || plane_width == 0+ || !plane.is_multiple_of(plane_width)+ || expected != Some(rgb.data.len())+ {return Err(MlError::Preprocess(format!("cannot write a {width}x{height}x{} image into {} values as {plane_width}-wide BGR planes",rgb.channels,@@ -60,8 +73,14 @@ pub(crate) fn write_bgr_planes(}let (blue, rest) = planes.split_at_mut(plane);let (green, red) = rest.split_at_mut(plane);+ let top = (plane_height - height) / 2 * plane_width;+ let bottom = top + height * plane_width;+ for channel in [&mut *blue, &mut *green, &mut *red] {+ channel[..top].fill(0.0);+ channel[bottom..].fill(0.0);+ }for (y, row) in rgb.data.chunks_exact(width * 3).enumerate() {- let offset = y * plane_width;+ let offset = top + y * plane_width;for (x, px) in row.as_chunks::<3>().0.iter().enumerate() {blue[offset + x] = normalization.apply(0, px[2]);green[offset + x] = normalization.apply(1, px[1]);@@ -146,4 +165,34 @@ mod tests {assert!((actual - expected).abs() <= 1e-6, "{planes:?}");}}++ #[test]+ fn smaller_images_are_centered_vertically_with_zero_padding_on_every_side() {+ let rgb = ImageU8::new(1, 2, 3, vec![10, 20, 30, 40, 50, 60]).unwrap();+ for height in [4, 5] {+ let mut planes = vec![f32::NAN; 3 * 2 * height];+ write_bgr_planes(&rgb, &mut planes, 2, BgrNormalization::CENTERED).unwrap();+ for (channel, plane) in planes.chunks_exact(2 * height).enumerate() {+ for (y, row) in plane.as_chunks::<2>().0.iter().enumerate() {+ let expected = match y {+ 1 => centered([30, 20, 10][channel]),+ 2 => centered([60, 50, 40][channel]),+ _ => 0.0,+ };+ assert_eq!(*row, [expected, 0.0]);+ }+ }+ }+ }++ #[test]+ fn invalid_destination_planes_return_an_error() {+ let rgb = ImageU8::new(2, 2, 3, vec![0; 12]).unwrap();+ for (width, len) in [(0, 12), (1, 12), (2, 6), (2, 13), (3, 12)] {+ assert!(+ write_bgr_planes(&rgb, &mut vec![0.0; len], width, BgrNormalization::CENTERED,)+ .is_err()+ );+ }+ }}