π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Simplify fixed OCR planning decoding and model generation
laurenspriemd7edb89b1c5b
diff --git a/infra/ml/playground/optimizations/README.md b/infra/ml/playground/optimizations/README.mdindex 63d58792f1..0a37537f9c 100644--- a/infra/ml/playground/optimizations/README.md+++ b/infra/ml/playground/optimizations/README.md@@ -32,7 +32,8 @@ input adapter; they cannot replace the original models independently.Missing sources are downloaded from `https://models.ente.com/PP-OCRv5`. Source andoutput hashes are verified. Upload only `det_fixed_v1.onnx`, `cls_fixed_v1.onnx`,-and `rec_fixed_v1.onnx`; the dictionary is unchanged. The generated+and `rec_fixed_v1.onnx`; the dictionary is unchanged. Unused legacy constants are+retained for byte-for-byte CDN reproducibility. The generated`ocr_model_manifest.json` records the input shapes, hashes, and sizes.## Rebuilding the modelsdiff --git a/infra/ml/playground/optimizations/optimize_ocr_models.py b/infra/ml/playground/optimizations/optimize_ocr_models.pyindex 443f249ed4..66bc859b10 100644--- a/infra/ml/playground/optimizations/optimize_ocr_models.py+++ b/infra/ml/playground/optimizations/optimize_ocr_models.py@@ -157,47 +157,6 @@ def fold_input_affine(model):return (remove_unused_nodes(model), count)-def align_recognizer_projection(model, width):- values = constant_arrays(model)- node = next(n for n in model.graph.node if n.name == "MatMul.12")- weights = values[node.input[1]]- assert weights.shape == (120, 18385)- padded = np.pad(weights, ((0, 0), (0, width - 18385)))- model.graph.initializer.extend(- [- numpy_helper.from_array(padded, "modelopt_projection_weight"),- numpy_helper.from_array(- np.array([0], dtype=np.int64), "modelopt_slice_start"- ),- numpy_helper.from_array(- np.array([18385], dtype=np.int64), "modelopt_slice_end"- ),- numpy_helper.from_array(- np.array([2], dtype=np.int64), "modelopt_slice_axis"- ),- ]- )- node.input[1] = "modelopt_projection_weight"- output = node.output[0]- node.output[0] = "modelopt_padded_projection"- index = next((i for i, n in enumerate(model.graph.node) if n.name == node.name))- model.graph.node.insert(- index + 1,- helper.make_node(- "Slice",- [- node.output[0],- "modelopt_slice_start",- "modelopt_slice_end",- "modelopt_slice_axis",- ],- [output],- name="modelopt_projection_slice",- ),- )- return remove_unused_nodes(model)--def fuse_recognizer_layer_norms(model):model = onnx.version_converter.convert_version(model, 17)values = constant_arrays(model)@@ -683,33 +642,60 @@ def recognizer_model(source_model, width, slots):return model, metadata-def pad_vocabulary(original):- model = copy.deepcopy(original)- values = {v.name: numpy_helper.to_array(v) for v in model.graph.initializer}- for node in model.graph.node:- if node.op_type == "Constant":- for attribute in node.attribute:- if attribute.name == "value":- values[node.output[0]] = numpy_helper.to_array(attribute.t)- removed = next(- node- for node in model.graph.node- if node.op_type == "Slice" and node.input[0] == "modelopt_padded_projection"- )+def pad_vocabulary(model):+ values = constant_arrays(model)+ projection = next(n for n in model.graph.node if n.name == "MatMul.12")+ weights = values[projection.input[1]]+ assert weights.shape == (120, 18385)add = next(nodefor node in model.graph.node- if node.op_type == "Add" and removed.output[0] in node.input+ if node.op_type == "Add" and projection.output[0] in node.input)- bias_name = next(name for name in add.input if name in values)- bias = values[bias_name].reshape(-1)+ bias = values[next(name for name in add.input if name in values)].reshape(-1)assert bias.size == 18385- extended = np.pad(bias, (0, 18432 - bias.size), constant_values=-np.inf)+ insertion = next(+ i+ for i, value in enumerate(model.graph.initializer)+ if value.name == "shared_positions"+ )+ model.graph.initializer.insert(+ insertion,+ numpy_helper.from_array(+ np.pad(weights, ((0, 0), (0, 47))), "modelopt_projection_weight"+ ),+ )+ projection.input[1] = "modelopt_projection_weight"+ original = projection.output[0]+ projection.output[0] = "modelopt_padded_projection"+ add.input[:] = [+ projection.output[0] if name == original else name for name in add.input+ ]+ model = remove_unused_nodes(model)+ insertion = next(+ i+ for i, value in enumerate(model.graph.initializer)+ if value.name == "shared_positions"+ )+ published_slice_constants = [("start", 0), ("end", 18385), ("axis", 2)]+ for offset, (name, value) in enumerate(published_slice_constants):+ model.graph.initializer.insert(+ insertion + offset,+ numpy_helper.from_array(+ np.array([value], dtype=np.int64), f"modelopt_slice_{name}"+ ),+ )model.graph.initializer.append(- numpy_helper.from_array(extended, "vocabulary_bias_padded")+ numpy_helper.from_array(+ np.pad(bias, (0, 47), constant_values=-np.inf), "vocabulary_bias_padded"+ )+ )+ add = next(+ node+ for node in model.graph.node+ if node.op_type == "Add" and "modelopt_padded_projection" in node.input)add.input[:] = ["modelopt_padded_projection", "vocabulary_bias_padded"]- model.graph.node.remove(removed)for value in model.graph.initializer:if value.name == "shared_positions":value.CopyFrom(@@ -1012,7 +998,6 @@ def optimize(models):f"Unexpected recognizer rewrites: folded={folded!r}, normalized={normalized!r}")recognition = expand_hardswish(recognition)- recognition = align_recognizer_projection(recognition, 18432)recognition = compact_recognizer_output(recognition)detection = fold_detector_head(copy.deepcopy(models["det"]))detection = expand_hardswish(detection)diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex 40600dff6f..f3aa49918b 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -8,7 +8,7 @@ use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};-use crate::onnx::{FloatTensorData, SessionRunError};+use crate::onnx::SessionRunError;use super::session::{OcrModel, OcrSession};const REC_VOCABULARY_SIZE: usize = 18385;@@ -79,22 +79,19 @@ impl TextRecognizer {return Ok(Vec::new());}let dictionary = self.dictionary.entries()?;- let mut lines = Vec::new();- let mut line_crops = Vec::new();- let ordered = recognize_in_batches(crops, request, |batch, layout| {- let mut indices = Vec::with_capacity(batch.len());- for (&crop, &content_width) in batch.iter().zip(&layout.content_widths) {- indices.push(lines.len());- line_crops.push(crop);+ let mut lines = Vec::with_capacity(crops.len());+ let mut crop_indices = Vec::with_capacity(crops.len());+ for batch in plan_batches(crops, request)? {+ for (&index, &content_width) in batch.indices.iter().zip(&batch.layout.content_widths) {+ crop_indices.push(index);lines.push(super::context::Line {values: Vec::new(),- width: layout.target_width as usize,+ width: batch.layout.target_width as usize,content_width,});}- Ok(indices)- })?;- let mut results = vec![Recognition::default(); lines.len()];+ }+ let mut results = vec![Recognition::default(); crops.len()];let mut pending: Vec<usize> = (0..lines.len()).collect();pending.sort_by_key(|&i| std::cmp::Reverse(lines[i].width));while !pending.is_empty() {@@ -125,7 +122,7 @@ impl TextRecognizer {target_width: line.width as i32,content_widths: vec![line.content_width],}- .tensor(&[line_crops[index]])?;+ .tensor(&[crops[crop_indices[index]]])?;}Ok(())})?;@@ -150,72 +147,61 @@ impl TextRecognizer {target_width: line.width as i32,content_widths: vec![line.content_width],};- let mut decoded = decode_output(+ let mut decoded = decode_compact_output(&[1, count as i64, 2],&values[start..start + count * 2],&layout,dictionary,)?;- results[index] = decoded.remove(0);+ results[crop_indices[index]] = decoded.remove(0);lines[index].values = Vec::new();}}- ordered- .into_iter()- .map(|index| {- results.get(index).cloned().ok_or_else(|| {- MlError::Preprocess("packed line index out of range".to_owned()).into()- })- })- .collect()+ Ok(results)}}-fn decode_output<'a, T>(+fn decode_compact_output(shape: &[i64],- values: &'a [T],+ values: &[f32],layout: &BatchLayout,dictionary: &[String],-) -> MlResult<Vec<Recognition>>-where- &'a [T]: FloatTensorData,-{- let compact = shape.get(2) == Some(&2);- let values_per_step = if compact { 2 } else { dictionary.len() };- let output = SequenceOutput::new(shape, values, layout.content_widths.len(), values_per_step)?;+) -> MlResult<Vec<Recognition>> {+ let count = layout.content_widths.len();+ let steps = match *shape {+ [n, steps, 2]+ if n == count as i64 && steps > 0 && values.len() == count * steps as usize * 2 =>+ {+ steps as usize+ }+ _ => {+ return Err(MlError::CorruptModel(format!(+ "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, 2]",+ values.len()+ )));+ }+ };layout.content_widths.iter()- .zip(output.sequences())- .map(|(&content_width, logits)| {- let padding_scale = layout.padding_scale(content_width);- if compact {- let best = compact_steps(logits, dictionary.len())?;- Ok(decode_steps(&best, dictionary, padding_scale))- } else {- Ok(ctc_decode(- logits,- output.vocabulary,- dictionary,- padding_scale,- ))- }+ .zip(values.chunks_exact(steps * 2))+ .map(|(&content_width, sequence)| {+ let best = compact_steps(sequence, dictionary.len())?;+ Ok(decode_steps(+ &best,+ dictionary,+ layout.padding_scale(content_width),+ ))}).collect()}-fn compact_steps<'a, T>(values: &'a [T], vocabulary: usize) -> MlResult<Vec<StepBest>>-where- &'a [T]: FloatTensorData,-{+fn compact_steps(values: &[f32], vocabulary: usize) -> MlResult<Vec<StepBest>> {values.as_chunks::<2>().0.iter()- .map(|step| {- let step = step.as_slice();- let index = step.value(0);- let probability = step.value(1);+ .map(|&[index, probability]| {if !index.is_finite()|| index < 0.0|| index >= vocabulary as f32@@ -235,12 +221,16 @@ where.collect()}-fn recognize_in_batches<'a, T>(- crops: &[&'a ImageU8],+struct PlannedBatch {+ indices: Vec<usize>,+ layout: BatchLayout,+}++fn plan_batches(+ crops: &[&ImageU8],request: &RequestGuard<'_>,- mut infer_batch: impl FnMut(&[&'a ImageU8], &BatchLayout) -> MlResult<Vec<T>>,-) -> Result<Vec<T>, OcrError> {- let mut results: Vec<Option<T>> = std::iter::repeat_with(|| None).take(crops.len()).collect();+) -> Result<Vec<PlannedBatch>, OcrError> {+ let mut batches = Vec::new();for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {request.check()?;let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();@@ -263,30 +253,17 @@ fn recognize_in_batches<'a, T>(target_width = next_width;end += 1;}- let split = &batch[start..end];- let split_layout = BatchLayout {- target_width,- content_widths: layout.content_widths[start..end].to_vec(),- };- let recognized = infer_batch(split, &split_layout)?;- if recognized.len() != split.len() {- return Err(MlError::CorruptModel(format!(- "text recognizer decoded {} crops out of {}",- recognized.len(),- split.len()- ))- .into());- }- for (&index, recognition) in indices[start..end].iter().zip(recognized) {- results[index] = Some(recognition);- }+ batches.push(PlannedBatch {+ indices: indices[start..end].to_vec(),+ layout: BatchLayout {+ target_width,+ content_widths: layout.content_widths[start..end].to_vec(),+ },+ });start = end;}}- results- .into_iter()- .collect::<Option<Vec<_>>>()- .ok_or_else(|| MlError::CorruptModel("text recognizer skipped a crop".to_string()).into())+ Ok(batches)}fn ascending_aspect_order(crops: &[&ImageU8]) -> Vec<usize> {@@ -400,53 +377,6 @@ fn content_height(crop: &ImageU8) -> i32 {.clamp(1, REC_HEIGHT)}-#[derive(Debug)]-struct SequenceOutput<'a, T> {- steps: usize,- vocabulary: usize,- values: &'a [T],-}--impl<'a, T> SequenceOutput<'a, T> {- fn new(shape: &[i64], values: &'a [T], count: usize, vocabulary: usize) -> MlResult<Self> {- match *shape {- [n, steps, v]- if n == count as i64- && steps > 0- && v == vocabulary as i64- && values.len() == count * steps as usize * vocabulary =>- {- Ok(Self {- steps: steps as usize,- vocabulary,- values,- })- }- _ => Err(MlError::CorruptModel(format!(- "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, {vocabulary}]",- values.len()- ))),- }- }-- fn sequences(&self) -> impl Iterator<Item = &'a [T]> {- self.values.chunks_exact(self.steps * self.vocabulary)- }-}--fn ctc_decode<'a, T>(- logits: &'a [T],- vocabulary: usize,- dictionary: &[String],- padding_scale: f32,-) -> Recognition-where- &'a [T]: FloatTensorData,-{- let best = best_per_step(logits, vocabulary);- decode_steps(&best, dictionary, padding_scale)-}-fn decode_steps(best: &[StepBest], dictionary: &[String], padding_scale: f32) -> Recognition {let spans: Vec<CharacterSpan> = character_runs(best).iter()@@ -465,45 +395,6 @@ struct StepBest {probability: f32,}-fn best_per_step<'a, T>(logits: &'a [T], vocabulary: usize) -> Vec<StepBest>-where- &'a [T]: FloatTensorData,-{- if vocabulary == 0 {- return Vec::new();- }- logits.chunks_exact(vocabulary).map(argmax).collect()-}--fn argmax<'a, T>(step: &'a [T]) -> StepBest-where- &'a [T]: FloatTensorData,-{- let mut maxima = [f32::NEG_INFINITY; 8];- let (chunks, remainder) = step.as_chunks::<8>();- for chunk in chunks {- for (lane, maximum) in maxima.iter_mut().enumerate() {- *maximum = maximum.max(chunk.as_slice().value(lane));- }- }- let mut maximum = maxima.into_iter().fold(f32::NEG_INFINITY, f32::max);- for index in 0..remainder.len() {- maximum = maximum.max(remainder.value(index));- }- if maximum > f32::NEG_INFINITY {- for index in 0..step.len() {- let probability = step.value(index);- if probability == maximum {- return StepBest { index, probability };- }- }- }- StepBest {- index: 0,- probability: f32::NEG_INFINITY,- }-}-struct CharacterRun {index: usize,start: usize,@@ -578,6 +469,136 @@ fn mean_confidence(spans: &[CharacterSpan]) -> f32 {mod tests {use super::*;use crate::ocr::cancel::RequestRegistry;+ use crate::onnx::FloatTensorData;+ #[derive(Debug)]+ struct SequenceOutput<'a, T> {+ steps: usize,+ vocabulary: usize,+ values: &'a [T],+ }++ impl<'a, T> SequenceOutput<'a, T> {+ fn new(shape: &[i64], values: &'a [T], count: usize, vocabulary: usize) -> MlResult<Self> {+ match *shape {+ [n, steps, v]+ if n == count as i64+ && steps > 0+ && v == vocabulary as i64+ && values.len() == count * steps as usize * vocabulary =>+ {+ Ok(Self {+ steps: steps as usize,+ vocabulary,+ values,+ })+ }+ _ => Err(MlError::CorruptModel(format!(+ "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, {vocabulary}]",+ values.len()+ ))),+ }+ }++ fn sequences(&self) -> impl Iterator<Item = &'a [T]> {+ self.values.chunks_exact(self.steps * self.vocabulary)+ }+ }++ fn ctc_decode<'a, T>(+ logits: &'a [T],+ vocabulary: usize,+ dictionary: &[String],+ padding_scale: f32,+ ) -> Recognition+ where+ &'a [T]: FloatTensorData,+ {+ let best = best_per_step(logits, vocabulary);+ decode_steps(&best, dictionary, padding_scale)+ }++ fn best_per_step<'a, T>(logits: &'a [T], vocabulary: usize) -> Vec<StepBest>+ where+ &'a [T]: FloatTensorData,+ {+ if vocabulary == 0 {+ return Vec::new();+ }+ logits.chunks_exact(vocabulary).map(argmax).collect()+ }++ fn argmax<'a, T>(step: &'a [T]) -> StepBest+ where+ &'a [T]: FloatTensorData,+ {+ let mut maxima = [f32::NEG_INFINITY; 8];+ let (chunks, remainder) = step.as_chunks::<8>();+ for chunk in chunks {+ for (lane, maximum) in maxima.iter_mut().enumerate() {+ *maximum = maximum.max(chunk.as_slice().value(lane));+ }+ }+ let mut maximum = maxima.into_iter().fold(f32::NEG_INFINITY, f32::max);+ for index in 0..remainder.len() {+ maximum = maximum.max(remainder.value(index));+ }+ if maximum > f32::NEG_INFINITY {+ for index in 0..step.len() {+ let probability = step.value(index);+ if probability == maximum {+ return StepBest { index, probability };+ }+ }+ }+ StepBest {+ index: 0,+ probability: f32::NEG_INFINITY,+ }+ }++ fn decode_output<'a, T>(+ shape: &[i64],+ values: &'a [T],+ layout: &BatchLayout,+ dictionary: &[String],+ ) -> MlResult<Vec<Recognition>>+ where+ &'a [T]: FloatTensorData,+ {+ let output =+ SequenceOutput::new(shape, values, layout.content_widths.len(), dictionary.len())?;+ Ok(layout+ .content_widths+ .iter()+ .zip(output.sequences())+ .map(|(&content_width, logits)| {+ ctc_decode(+ logits,+ output.vocabulary,+ dictionary,+ layout.padding_scale(content_width),+ )+ })+ .collect())+ }++ fn run_planned_batches(+ crops: &[&ImageU8],+ request: &RequestGuard<'_>,+ mut infer_batch: impl FnMut(&[&ImageU8], &BatchLayout) -> MlResult<Vec<Recognition>>,+ ) -> Result<Vec<Recognition>, OcrError> {+ let mut results = vec![Recognition::default(); crops.len()];+ for batch in plan_batches(crops, request)? {+ request.check()?;+ let images: Vec<_> = batch.indices.iter().map(|&index| crops[index]).collect();+ let recognized = infer_batch(&images, &batch.layout)?;+ assert_eq!(recognized.len(), images.len());+ for (&index, recognition) in batch.indices.iter().zip(recognized) {+ results[index] = recognition;+ }+ }+ Ok(results)+ }const TOLERANCE: f32 = 1e-6;@@ -648,7 +669,7 @@ mod tests {content_widths: vec![160, 320],};let expected = decode_output(&[2, 6, 4], &full, &layout, &dictionary()).unwrap();- let actual = decode_output(&[2, 6, 2], &compact, &layout, &dictionary()).unwrap();+ let actual = decode_compact_output(&[2, 6, 2], &compact, &layout, &dictionary()).unwrap();assert_eq!(actual, expected);assert_eq!(actual[0].text, "aabc");assert_eq!(actual[1], Recognition::default());@@ -670,10 +691,10 @@ mod tests {[1.0, f32::INFINITY],[1.0, -0.1],] {- assert!(decode_output(&[1, 1, 2], &values, &layout, &dictionary()).is_err());+ assert!(decode_compact_output(&[1, 1, 2], &values, &layout, &dictionary()).is_err());}- for shape in [[2, 1, 2], [1, 0, 2], [1, 2, 2]] {- assert!(decode_output(&shape, &[1.0, 0.9], &layout, &dictionary()).is_err());+ for shape in [[2, 1, 2], [1, 0, 2], [1, 2, 2], [1, 1, 4]] {+ assert!(decode_compact_output(&shape, &[1.0, 0.9], &layout, &dictionary()).is_err());}}@@ -886,7 +907,7 @@ mod tests {let mut batches = Vec::new();let results =- recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch, _| {+ run_planned_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch, _| {batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());Ok(batch.iter()@@ -911,8 +932,7 @@ mod tests {let request = registry.begin(Some("cancelled"));registry.cancel("cancelled");- let error = recognize_in_batches::<Recognition>(&[&crop], &request, |_, _| unreachable!())- .unwrap_err();+ let error = run_planned_batches(&[&crop], &request, |_, _| unreachable!()).unwrap_err();assert!(matches!(error, OcrError::Cancelled), "{error}");}@@ -1005,7 +1025,7 @@ mod tests {let registry = RequestRegistry::default();let request = registry.begin(None);let mut shapes = Vec::new();- let results = recognize_in_batches(&references, &request, |batch, layout| {+ let results = run_planned_batches(&references, &request, |batch, layout| {assert!(batch.len() <= REC_BATCH_SIZE);assert!(batch.len() * layout.target_width as usize <= REC_BATCH_COLUMNS);shapes.push((batch.len(), layout.target_width));@@ -1048,7 +1068,7 @@ mod tests {let registry = RequestRegistry::default();let request = registry.begin(Some("mixed"));let mut shapes = Vec::new();- let result = recognize_in_batches(&crops, &request, |batch, layout| {+ let result = run_planned_batches(&crops, &request, |batch, layout| {shapes.push((batch.len(), layout.target_width));assert_eq!(layout.tensor(batch)?.len(),@@ -1188,7 +1208,7 @@ mod tests {let registry = RequestRegistry::default();let request = registry.begin(None);let mut calls = 0;- recognize_in_batches(&crops, &request, |batch, layout| {+ run_planned_batches(&crops, &request, |batch, layout| {calls += 1;assert_eq!(batch.len(), 1);assert_eq!(layout.tensor(batch)?.len(), 3 * 48 * 7168);@@ -1199,7 +1219,7 @@ mod tests {let request = registry.begin(Some("split"));calls = 0;- let error = recognize_in_batches(&crops, &request, |_, _| {+ let error = run_planned_batches(&crops, &request, |_, _| {calls += 1;registry.cancel("split");Ok(vec![Recognition::default()])