Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Simplify fixed OCR planning decoding and model generation

laurenspriemd7edb89b1c5b
diff --git a/infra/ml/playground/optimizations/README.md b/infra/ml/playground/optimizations/README.md
index 63d58792f1..0a37537f9c 100644
--- a/infra/ml/playground/optimizations/README.md
+++ b/infra/ml/playground/optimizations/README.md
@@ -32,7 +32,8 @@ input adapter; they cannot replace the original models independently.
Missing sources are downloaded from `https://models.ente.com/PP-OCRv5`. Source and
output hashes are verified. Upload only `det_fixed_v1.onnx`, `cls_fixed_v1.onnx`,
-and `rec_fixed_v1.onnx`; the dictionary is unchanged. The generated
+and `rec_fixed_v1.onnx`; the dictionary is unchanged. Unused legacy constants are
+retained for byte-for-byte CDN reproducibility. The generated
`ocr_model_manifest.json` records the input shapes, hashes, and sizes.
## Rebuilding the models
diff --git a/infra/ml/playground/optimizations/optimize_ocr_models.py b/infra/ml/playground/optimizations/optimize_ocr_models.py
index 443f249ed4..66bc859b10 100644
--- a/infra/ml/playground/optimizations/optimize_ocr_models.py
+++ b/infra/ml/playground/optimizations/optimize_ocr_models.py
@@ -157,47 +157,6 @@ def fold_input_affine(model):
return (remove_unused_nodes(model), count)
-def align_recognizer_projection(model, width):
- values = constant_arrays(model)
- node = next(n for n in model.graph.node if n.name == "MatMul.12")
- weights = values[node.input[1]]
- assert weights.shape == (120, 18385)
- padded = np.pad(weights, ((0, 0), (0, width - 18385)))
- model.graph.initializer.extend(
- [
- numpy_helper.from_array(padded, "modelopt_projection_weight"),
- numpy_helper.from_array(
- np.array([0], dtype=np.int64), "modelopt_slice_start"
- ),
- numpy_helper.from_array(
- np.array([18385], dtype=np.int64), "modelopt_slice_end"
- ),
- numpy_helper.from_array(
- np.array([2], dtype=np.int64), "modelopt_slice_axis"
- ),
- ]
- )
- node.input[1] = "modelopt_projection_weight"
- output = node.output[0]
- node.output[0] = "modelopt_padded_projection"
- index = next((i for i, n in enumerate(model.graph.node) if n.name == node.name))
- model.graph.node.insert(
- index + 1,
- helper.make_node(
- "Slice",
- [
- node.output[0],
- "modelopt_slice_start",
- "modelopt_slice_end",
- "modelopt_slice_axis",
- ],
- [output],
- name="modelopt_projection_slice",
- ),
- )
- return remove_unused_nodes(model)
-
-
def fuse_recognizer_layer_norms(model):
model = onnx.version_converter.convert_version(model, 17)
values = constant_arrays(model)
@@ -683,33 +642,60 @@ def recognizer_model(source_model, width, slots):
return model, metadata
-def pad_vocabulary(original):
- model = copy.deepcopy(original)
- values = {v.name: numpy_helper.to_array(v) for v in model.graph.initializer}
- for node in model.graph.node:
- if node.op_type == "Constant":
- for attribute in node.attribute:
- if attribute.name == "value":
- values[node.output[0]] = numpy_helper.to_array(attribute.t)
- removed = next(
- node
- for node in model.graph.node
- if node.op_type == "Slice" and node.input[0] == "modelopt_padded_projection"
- )
+def pad_vocabulary(model):
+ values = constant_arrays(model)
+ projection = next(n for n in model.graph.node if n.name == "MatMul.12")
+ weights = values[projection.input[1]]
+ assert weights.shape == (120, 18385)
add = next(
node
for node in model.graph.node
- if node.op_type == "Add" and removed.output[0] in node.input
+ if node.op_type == "Add" and projection.output[0] in node.input
)
- bias_name = next(name for name in add.input if name in values)
- bias = values[bias_name].reshape(-1)
+ bias = values[next(name for name in add.input if name in values)].reshape(-1)
assert bias.size == 18385
- extended = np.pad(bias, (0, 18432 - bias.size), constant_values=-np.inf)
+ insertion = next(
+ i
+ for i, value in enumerate(model.graph.initializer)
+ if value.name == "shared_positions"
+ )
+ model.graph.initializer.insert(
+ insertion,
+ numpy_helper.from_array(
+ np.pad(weights, ((0, 0), (0, 47))), "modelopt_projection_weight"
+ ),
+ )
+ projection.input[1] = "modelopt_projection_weight"
+ original = projection.output[0]
+ projection.output[0] = "modelopt_padded_projection"
+ add.input[:] = [
+ projection.output[0] if name == original else name for name in add.input
+ ]
+ model = remove_unused_nodes(model)
+ insertion = next(
+ i
+ for i, value in enumerate(model.graph.initializer)
+ if value.name == "shared_positions"
+ )
+ published_slice_constants = [("start", 0), ("end", 18385), ("axis", 2)]
+ for offset, (name, value) in enumerate(published_slice_constants):
+ model.graph.initializer.insert(
+ insertion + offset,
+ numpy_helper.from_array(
+ np.array([value], dtype=np.int64), f"modelopt_slice_{name}"
+ ),
+ )
model.graph.initializer.append(
- numpy_helper.from_array(extended, "vocabulary_bias_padded")
+ numpy_helper.from_array(
+ np.pad(bias, (0, 47), constant_values=-np.inf), "vocabulary_bias_padded"
+ )
+ )
+ add = next(
+ node
+ for node in model.graph.node
+ if node.op_type == "Add" and "modelopt_padded_projection" in node.input
)
add.input[:] = ["modelopt_padded_projection", "vocabulary_bias_padded"]
- model.graph.node.remove(removed)
for value in model.graph.initializer:
if value.name == "shared_positions":
value.CopyFrom(
@@ -1012,7 +998,6 @@ def optimize(models):
f"Unexpected recognizer rewrites: folded={folded!r}, normalized={normalized!r}"
)
recognition = expand_hardswish(recognition)
- recognition = align_recognizer_projection(recognition, 18432)
recognition = compact_recognizer_output(recognition)
detection = fold_detector_head(copy.deepcopy(models["det"]))
detection = expand_hardswish(detection)
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index 40600dff6f..f3aa49918b 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -8,7 +8,7 @@ use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};
use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
-use crate::onnx::{FloatTensorData, SessionRunError};
+use crate::onnx::SessionRunError;
use super::session::{OcrModel, OcrSession};
const REC_VOCABULARY_SIZE: usize = 18385;
@@ -79,22 +79,19 @@ impl TextRecognizer {
return Ok(Vec::new());
}
let dictionary = self.dictionary.entries()?;
- let mut lines = Vec::new();
- let mut line_crops = Vec::new();
- let ordered = recognize_in_batches(crops, request, |batch, layout| {
- let mut indices = Vec::with_capacity(batch.len());
- for (&crop, &content_width) in batch.iter().zip(&layout.content_widths) {
- indices.push(lines.len());
- line_crops.push(crop);
+ let mut lines = Vec::with_capacity(crops.len());
+ let mut crop_indices = Vec::with_capacity(crops.len());
+ for batch in plan_batches(crops, request)? {
+ for (&index, &content_width) in batch.indices.iter().zip(&batch.layout.content_widths) {
+ crop_indices.push(index);
lines.push(super::context::Line {
values: Vec::new(),
- width: layout.target_width as usize,
+ width: batch.layout.target_width as usize,
content_width,
});
}
- Ok(indices)
- })?;
- let mut results = vec![Recognition::default(); lines.len()];
+ }
+ let mut results = vec![Recognition::default(); crops.len()];
let mut pending: Vec<usize> = (0..lines.len()).collect();
pending.sort_by_key(|&i| std::cmp::Reverse(lines[i].width));
while !pending.is_empty() {
@@ -125,7 +122,7 @@ impl TextRecognizer {
target_width: line.width as i32,
content_widths: vec![line.content_width],
}
- .tensor(&[line_crops[index]])?;
+ .tensor(&[crops[crop_indices[index]]])?;
}
Ok(())
})?;
@@ -150,72 +147,61 @@ impl TextRecognizer {
target_width: line.width as i32,
content_widths: vec![line.content_width],
};
- let mut decoded = decode_output(
+ let mut decoded = decode_compact_output(
&[1, count as i64, 2],
&values[start..start + count * 2],
&layout,
dictionary,
)?;
- results[index] = decoded.remove(0);
+ results[crop_indices[index]] = decoded.remove(0);
lines[index].values = Vec::new();
}
}
- ordered
- .into_iter()
- .map(|index| {
- results.get(index).cloned().ok_or_else(|| {
- MlError::Preprocess("packed line index out of range".to_owned()).into()
- })
- })
- .collect()
+ Ok(results)
}
}
-fn decode_output<'a, T>(
+fn decode_compact_output(
shape: &[i64],
- values: &'a [T],
+ values: &[f32],
layout: &BatchLayout,
dictionary: &[String],
-) -> MlResult<Vec<Recognition>>
-where
- &'a [T]: FloatTensorData,
-{
- let compact = shape.get(2) == Some(&2);
- let values_per_step = if compact { 2 } else { dictionary.len() };
- let output = SequenceOutput::new(shape, values, layout.content_widths.len(), values_per_step)?;
+) -> MlResult<Vec<Recognition>> {
+ let count = layout.content_widths.len();
+ let steps = match *shape {
+ [n, steps, 2]
+ if n == count as i64 && steps > 0 && values.len() == count * steps as usize * 2 =>
+ {
+ steps as usize
+ }
+ _ => {
+ return Err(MlError::CorruptModel(format!(
+ "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, 2]",
+ values.len()
+ )));
+ }
+ };
layout
.content_widths
.iter()
- .zip(output.sequences())
- .map(|(&content_width, logits)| {
- let padding_scale = layout.padding_scale(content_width);
- if compact {
- let best = compact_steps(logits, dictionary.len())?;
- Ok(decode_steps(&best, dictionary, padding_scale))
- } else {
- Ok(ctc_decode(
- logits,
- output.vocabulary,
- dictionary,
- padding_scale,
- ))
- }
+ .zip(values.chunks_exact(steps * 2))
+ .map(|(&content_width, sequence)| {
+ let best = compact_steps(sequence, dictionary.len())?;
+ Ok(decode_steps(
+ &best,
+ dictionary,
+ layout.padding_scale(content_width),
+ ))
})
.collect()
}
-fn compact_steps<'a, T>(values: &'a [T], vocabulary: usize) -> MlResult<Vec<StepBest>>
-where
- &'a [T]: FloatTensorData,
-{
+fn compact_steps(values: &[f32], vocabulary: usize) -> MlResult<Vec<StepBest>> {
values
.as_chunks::<2>()
.0
.iter()
- .map(|step| {
- let step = step.as_slice();
- let index = step.value(0);
- let probability = step.value(1);
+ .map(|&[index, probability]| {
if !index.is_finite()
|| index < 0.0
|| index >= vocabulary as f32
@@ -235,12 +221,16 @@ where
.collect()
}
-fn recognize_in_batches<'a, T>(
- crops: &[&'a ImageU8],
+struct PlannedBatch {
+ indices: Vec<usize>,
+ layout: BatchLayout,
+}
+
+fn plan_batches(
+ crops: &[&ImageU8],
request: &RequestGuard<'_>,
- mut infer_batch: impl FnMut(&[&'a ImageU8], &BatchLayout) -> MlResult<Vec<T>>,
-) -> Result<Vec<T>, OcrError> {
- let mut results: Vec<Option<T>> = std::iter::repeat_with(|| None).take(crops.len()).collect();
+) -> Result<Vec<PlannedBatch>, OcrError> {
+ let mut batches = Vec::new();
for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {
request.check()?;
let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();
@@ -263,30 +253,17 @@ fn recognize_in_batches<'a, T>(
target_width = next_width;
end += 1;
}
- let split = &batch[start..end];
- let split_layout = BatchLayout {
- target_width,
- content_widths: layout.content_widths[start..end].to_vec(),
- };
- let recognized = infer_batch(split, &split_layout)?;
- if recognized.len() != split.len() {
- return Err(MlError::CorruptModel(format!(
- "text recognizer decoded {} crops out of {}",
- recognized.len(),
- split.len()
- ))
- .into());
- }
- for (&index, recognition) in indices[start..end].iter().zip(recognized) {
- results[index] = Some(recognition);
- }
+ batches.push(PlannedBatch {
+ indices: indices[start..end].to_vec(),
+ layout: BatchLayout {
+ target_width,
+ content_widths: layout.content_widths[start..end].to_vec(),
+ },
+ });
start = end;
}
}
- results
- .into_iter()
- .collect::<Option<Vec<_>>>()
- .ok_or_else(|| MlError::CorruptModel("text recognizer skipped a crop".to_string()).into())
+ Ok(batches)
}
fn ascending_aspect_order(crops: &[&ImageU8]) -> Vec<usize> {
@@ -400,53 +377,6 @@ fn content_height(crop: &ImageU8) -> i32 {
.clamp(1, REC_HEIGHT)
}
-#[derive(Debug)]
-struct SequenceOutput<'a, T> {
- steps: usize,
- vocabulary: usize,
- values: &'a [T],
-}
-
-impl<'a, T> SequenceOutput<'a, T> {
- fn new(shape: &[i64], values: &'a [T], count: usize, vocabulary: usize) -> MlResult<Self> {
- match *shape {
- [n, steps, v]
- if n == count as i64
- && steps > 0
- && v == vocabulary as i64
- && values.len() == count * steps as usize * vocabulary =>
- {
- Ok(Self {
- steps: steps as usize,
- vocabulary,
- values,
- })
- }
- _ => Err(MlError::CorruptModel(format!(
- "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, {vocabulary}]",
- values.len()
- ))),
- }
- }
-
- fn sequences(&self) -> impl Iterator<Item = &'a [T]> {
- self.values.chunks_exact(self.steps * self.vocabulary)
- }
-}
-
-fn ctc_decode<'a, T>(
- logits: &'a [T],
- vocabulary: usize,
- dictionary: &[String],
- padding_scale: f32,
-) -> Recognition
-where
- &'a [T]: FloatTensorData,
-{
- let best = best_per_step(logits, vocabulary);
- decode_steps(&best, dictionary, padding_scale)
-}
-
fn decode_steps(best: &[StepBest], dictionary: &[String], padding_scale: f32) -> Recognition {
let spans: Vec<CharacterSpan> = character_runs(best)
.iter()
@@ -465,45 +395,6 @@ struct StepBest {
probability: f32,
}
-fn best_per_step<'a, T>(logits: &'a [T], vocabulary: usize) -> Vec<StepBest>
-where
- &'a [T]: FloatTensorData,
-{
- if vocabulary == 0 {
- return Vec::new();
- }
- logits.chunks_exact(vocabulary).map(argmax).collect()
-}
-
-fn argmax<'a, T>(step: &'a [T]) -> StepBest
-where
- &'a [T]: FloatTensorData,
-{
- let mut maxima = [f32::NEG_INFINITY; 8];
- let (chunks, remainder) = step.as_chunks::<8>();
- for chunk in chunks {
- for (lane, maximum) in maxima.iter_mut().enumerate() {
- *maximum = maximum.max(chunk.as_slice().value(lane));
- }
- }
- let mut maximum = maxima.into_iter().fold(f32::NEG_INFINITY, f32::max);
- for index in 0..remainder.len() {
- maximum = maximum.max(remainder.value(index));
- }
- if maximum > f32::NEG_INFINITY {
- for index in 0..step.len() {
- let probability = step.value(index);
- if probability == maximum {
- return StepBest { index, probability };
- }
- }
- }
- StepBest {
- index: 0,
- probability: f32::NEG_INFINITY,
- }
-}
-
struct CharacterRun {
index: usize,
start: usize,
@@ -578,6 +469,136 @@ fn mean_confidence(spans: &[CharacterSpan]) -> f32 {
mod tests {
use super::*;
use crate::ocr::cancel::RequestRegistry;
+ use crate::onnx::FloatTensorData;
+ #[derive(Debug)]
+ struct SequenceOutput<'a, T> {
+ steps: usize,
+ vocabulary: usize,
+ values: &'a [T],
+ }
+
+ impl<'a, T> SequenceOutput<'a, T> {
+ fn new(shape: &[i64], values: &'a [T], count: usize, vocabulary: usize) -> MlResult<Self> {
+ match *shape {
+ [n, steps, v]
+ if n == count as i64
+ && steps > 0
+ && v == vocabulary as i64
+ && values.len() == count * steps as usize * vocabulary =>
+ {
+ Ok(Self {
+ steps: steps as usize,
+ vocabulary,
+ values,
+ })
+ }
+ _ => Err(MlError::CorruptModel(format!(
+ "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, {vocabulary}]",
+ values.len()
+ ))),
+ }
+ }
+
+ fn sequences(&self) -> impl Iterator<Item = &'a [T]> {
+ self.values.chunks_exact(self.steps * self.vocabulary)
+ }
+ }
+
+ fn ctc_decode<'a, T>(
+ logits: &'a [T],
+ vocabulary: usize,
+ dictionary: &[String],
+ padding_scale: f32,
+ ) -> Recognition
+ where
+ &'a [T]: FloatTensorData,
+ {
+ let best = best_per_step(logits, vocabulary);
+ decode_steps(&best, dictionary, padding_scale)
+ }
+
+ fn best_per_step<'a, T>(logits: &'a [T], vocabulary: usize) -> Vec<StepBest>
+ where
+ &'a [T]: FloatTensorData,
+ {
+ if vocabulary == 0 {
+ return Vec::new();
+ }
+ logits.chunks_exact(vocabulary).map(argmax).collect()
+ }
+
+ fn argmax<'a, T>(step: &'a [T]) -> StepBest
+ where
+ &'a [T]: FloatTensorData,
+ {
+ let mut maxima = [f32::NEG_INFINITY; 8];
+ let (chunks, remainder) = step.as_chunks::<8>();
+ for chunk in chunks {
+ for (lane, maximum) in maxima.iter_mut().enumerate() {
+ *maximum = maximum.max(chunk.as_slice().value(lane));
+ }
+ }
+ let mut maximum = maxima.into_iter().fold(f32::NEG_INFINITY, f32::max);
+ for index in 0..remainder.len() {
+ maximum = maximum.max(remainder.value(index));
+ }
+ if maximum > f32::NEG_INFINITY {
+ for index in 0..step.len() {
+ let probability = step.value(index);
+ if probability == maximum {
+ return StepBest { index, probability };
+ }
+ }
+ }
+ StepBest {
+ index: 0,
+ probability: f32::NEG_INFINITY,
+ }
+ }
+
+ fn decode_output<'a, T>(
+ shape: &[i64],
+ values: &'a [T],
+ layout: &BatchLayout,
+ dictionary: &[String],
+ ) -> MlResult<Vec<Recognition>>
+ where
+ &'a [T]: FloatTensorData,
+ {
+ let output =
+ SequenceOutput::new(shape, values, layout.content_widths.len(), dictionary.len())?;
+ Ok(layout
+ .content_widths
+ .iter()
+ .zip(output.sequences())
+ .map(|(&content_width, logits)| {
+ ctc_decode(
+ logits,
+ output.vocabulary,
+ dictionary,
+ layout.padding_scale(content_width),
+ )
+ })
+ .collect())
+ }
+
+ fn run_planned_batches(
+ crops: &[&ImageU8],
+ request: &RequestGuard<'_>,
+ mut infer_batch: impl FnMut(&[&ImageU8], &BatchLayout) -> MlResult<Vec<Recognition>>,
+ ) -> Result<Vec<Recognition>, OcrError> {
+ let mut results = vec![Recognition::default(); crops.len()];
+ for batch in plan_batches(crops, request)? {
+ request.check()?;
+ let images: Vec<_> = batch.indices.iter().map(|&index| crops[index]).collect();
+ let recognized = infer_batch(&images, &batch.layout)?;
+ assert_eq!(recognized.len(), images.len());
+ for (&index, recognition) in batch.indices.iter().zip(recognized) {
+ results[index] = recognition;
+ }
+ }
+ Ok(results)
+ }
const TOLERANCE: f32 = 1e-6;
@@ -648,7 +669,7 @@ mod tests {
content_widths: vec![160, 320],
};
let expected = decode_output(&[2, 6, 4], &full, &layout, &dictionary()).unwrap();
- let actual = decode_output(&[2, 6, 2], &compact, &layout, &dictionary()).unwrap();
+ let actual = decode_compact_output(&[2, 6, 2], &compact, &layout, &dictionary()).unwrap();
assert_eq!(actual, expected);
assert_eq!(actual[0].text, "aabc");
assert_eq!(actual[1], Recognition::default());
@@ -670,10 +691,10 @@ mod tests {
[1.0, f32::INFINITY],
[1.0, -0.1],
] {
- assert!(decode_output(&[1, 1, 2], &values, &layout, &dictionary()).is_err());
+ assert!(decode_compact_output(&[1, 1, 2], &values, &layout, &dictionary()).is_err());
}
- for shape in [[2, 1, 2], [1, 0, 2], [1, 2, 2]] {
- assert!(decode_output(&shape, &[1.0, 0.9], &layout, &dictionary()).is_err());
+ for shape in [[2, 1, 2], [1, 0, 2], [1, 2, 2], [1, 1, 4]] {
+ assert!(decode_compact_output(&shape, &[1.0, 0.9], &layout, &dictionary()).is_err());
}
}
@@ -886,7 +907,7 @@ mod tests {
let mut batches = Vec::new();
let results =
- recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch, _| {
+ run_planned_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch, _| {
batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());
Ok(batch
.iter()
@@ -911,8 +932,7 @@ mod tests {
let request = registry.begin(Some("cancelled"));
registry.cancel("cancelled");
- let error = recognize_in_batches::<Recognition>(&[&crop], &request, |_, _| unreachable!())
- .unwrap_err();
+ let error = run_planned_batches(&[&crop], &request, |_, _| unreachable!()).unwrap_err();
assert!(matches!(error, OcrError::Cancelled), "{error}");
}
@@ -1005,7 +1025,7 @@ mod tests {
let registry = RequestRegistry::default();
let request = registry.begin(None);
let mut shapes = Vec::new();
- let results = recognize_in_batches(&references, &request, |batch, layout| {
+ let results = run_planned_batches(&references, &request, |batch, layout| {
assert!(batch.len() <= REC_BATCH_SIZE);
assert!(batch.len() * layout.target_width as usize <= REC_BATCH_COLUMNS);
shapes.push((batch.len(), layout.target_width));
@@ -1048,7 +1068,7 @@ mod tests {
let registry = RequestRegistry::default();
let request = registry.begin(Some("mixed"));
let mut shapes = Vec::new();
- let result = recognize_in_batches(&crops, &request, |batch, layout| {
+ let result = run_planned_batches(&crops, &request, |batch, layout| {
shapes.push((batch.len(), layout.target_width));
assert_eq!(
layout.tensor(batch)?.len(),
@@ -1188,7 +1208,7 @@ mod tests {
let registry = RequestRegistry::default();
let request = registry.begin(None);
let mut calls = 0;
- recognize_in_batches(&crops, &request, |batch, layout| {
+ run_planned_batches(&crops, &request, |batch, layout| {
calls += 1;
assert_eq!(batch.len(), 1);
assert_eq!(layout.tensor(batch)?.len(), 3 * 48 * 7168);
@@ -1199,7 +1219,7 @@ mod tests {
let request = registry.begin(Some("split"));
calls = 0;
- let error = recognize_in_batches(&crops, &request, |_, _| {
+ let error = run_planned_batches(&crops, &request, |_, _| {
calls += 1;
registry.cancel("split");
Ok(vec![Recognition::default()])