Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Add OCR text recognition

laurenspriem86ead8087d20
diff --git a/rust/crates/ml/examples/ocr.rs b/rust/crates/ml/examples/ocr.rs
index 4ae088696e..849c434776 100644
--- a/rust/crates/ml/examples/ocr.rs
+++ b/rust/crates/ml/examples/ocr.rs
@@ -9,29 +9,58 @@ use anyhow::{Context, Result, bail};
use ente_assets::AssetStore;
use ente_image::decode::decode_image_from_path;
use ente_ml::ocr::{
- CropDebug, DetectRegionsRequest, OcrEngine, OcrModelPaths, Orientation, Point, ProbabilityMap,
- RegionDetectionDebug, TextRegion, assets,
+ CropDebug, DetectRegionsRequest, DetectTextRequest, OcrEngine, OcrModelPaths, Orientation,
+ Point, ProbabilityMap, RegionDetectionDebug, TextBlock, TextDetectionDebug,
+ TextDetectionResult, TextRegion, assets,
};
use image::{GrayImage, Rgb, RgbImage};
use imageproc::drawing::draw_line_segment_mut;
use support::ml_indexing::{asset_cache_dir, load_onnx_runtime, run_with_large_stack};
-const OVERLAY_COLOR: Rgb<u8> = Rgb([255, 0, 0]);
+const BLOCK_COLOR: Rgb<u8> = Rgb([255, 0, 0]);
+const CHARACTER_COLOR: Rgb<u8> = Rgb([0, 0, 255]);
struct Options {
+ regions: bool,
+ all_confidences: bool,
dump_dir: Option<PathBuf>,
images: Vec<String>,
}
+struct OcrLogger;
+
+impl log::Log for OcrLogger {
+ fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
+ metadata.level() <= log::Level::Info && metadata.target().starts_with("ente_ml::ocr")
+ }
+
+ fn log(&self, record: &log::Record<'_>) {
+ if self.enabled(record.metadata()) {
+ eprintln!("{}", record.args());
+ }
+ }
+
+ fn flush(&self) {}
+}
+
+static LOGGER: OcrLogger = OcrLogger;
+
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
let options = parse_args(std::env::args().skip(1))?;
+ install_logger();
load_onnx_runtime().await?;
let store = AssetStore::new(asset_cache_dir()?);
let paths = assets::ensure_models(&store).await?;
run_with_large_stack("ocr", move || run(options, paths))
}
+fn install_logger() {
+ if log::set_logger(&LOGGER).is_ok() {
+ log::set_max_level(log::LevelFilter::Info);
+ }
+}
+
fn usage() -> &'static str {
"usage: cargo run -p ente-ml --example ocr -- [--regions] [--all-confidences] \
[--dump-dir DIR] <image>..."
@@ -39,12 +68,15 @@ fn usage() -> &'static str {
fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Options> {
let mut options = Options {
+ regions: false,
+ all_confidences: false,
dump_dir: None,
images: Vec::new(),
};
while let Some(arg) = args.next() {
match arg.as_str() {
- "--regions" | "--all-confidences" => {}
+ "--regions" => options.regions = true,
+ "--all-confidences" => options.all_confidences = true,
"--dump-dir" => {
let dir = args.next().context("--dump-dir needs a directory")?;
options.dump_dir = Some(PathBuf::from(dir));
@@ -63,12 +95,19 @@ fn run(options: Options, paths: OcrModelPaths) -> Result<()> {
let engine = OcrEngine::new(paths)?;
let mut failures = 0usize;
for image_path in &options.images {
- match detect_regions(&engine, image_path, options.dump_dir.as_deref()) {
- Ok(()) => {}
- Err(error) => {
- failures += 1;
- eprintln!("{image_path}: {error:#}");
- }
+ let outcome = if options.regions {
+ detect_regions(&engine, image_path, options.dump_dir.as_deref())
+ } else {
+ detect_text(
+ &engine,
+ image_path,
+ options.all_confidences,
+ options.dump_dir.as_deref(),
+ )
+ };
+ if let Err(error) = outcome {
+ failures += 1;
+ eprintln!("{image_path}: {error:#}");
}
}
if failures > 0 {
@@ -77,6 +116,57 @@ fn run(options: Options, paths: OcrModelPaths) -> Result<()> {
Ok(())
}
+fn detect_text(
+ engine: &OcrEngine,
+ image_path: &str,
+ all_confidences: bool,
+ dump_dir: Option<&Path>,
+) -> Result<()> {
+ let request = DetectTextRequest {
+ image_path: image_path.to_string(),
+ include_all_confidence_scores: all_confidences,
+ request_id: None,
+ };
+ let started = Instant::now();
+ match dump_dir {
+ None => {
+ let result = engine.detect_text(&request)?;
+ print_text_result(image_path, &result, None, started.elapsed().as_millis());
+ }
+ Some(dir) => {
+ let debug = engine.detect_text_debug(&request)?;
+ print_text_result(
+ image_path,
+ &debug.result,
+ Some((debug.working_width, debug.working_height)),
+ started.elapsed().as_millis(),
+ );
+ let target = dump_target(dir, image_path)?;
+ dump_text(&target, image_path, &debug)?;
+ }
+ }
+ Ok(())
+}
+
+fn print_text_result(
+ image_path: &str,
+ result: &TextDetectionResult,
+ working: Option<(u32, u32)>,
+ elapsed_ms: u128,
+) {
+ let working = working
+ .map(|(width, height)| format!(", working {width}x{height}"))
+ .unwrap_or_default();
+ println!(
+ "{image_path}: decoded {}x{}{working}, {elapsed_ms}ms",
+ result.image_width, result.image_height
+ );
+ for block in &result.blocks {
+ println!(" {}", format_block(block));
+ }
+ println!(" blocks: {}", result.blocks.len());
+}
+
fn detect_regions(engine: &OcrEngine, image_path: &str, dump_dir: Option<&Path>) -> Result<()> {
let started = Instant::now();
let debug = engine.detect_text_regions_debug(&DetectRegionsRequest {
@@ -100,20 +190,34 @@ fn detect_regions(engine: &OcrEngine, image_path: &str, dump_dir: Option<&Path>)
}
if let Some(dir) = dump_dir {
let target = dump_target(dir, image_path)?;
- dump(&target, image_path, &debug)?;
- dump_crops(engine, image_path, &target)?;
+ dump_regions(&target, image_path, &debug)?;
}
Ok(())
}
+fn format_block(block: &TextBlock) -> String {
+ format!(
+ "[{:.3}] {} {}",
+ block.confidence,
+ block.text,
+ format_points(&block.points)
+ )
+}
+
fn format_region(region: &TextRegion) -> String {
- let corners = region
- .points
+ format!(
+ "[{:.3}] {}",
+ region.confidence,
+ format_points(&region.points)
+ )
+}
+
+fn format_points(points: &[Point; 4]) -> String {
+ points
.iter()
.map(|point| format!("({:.0},{:.0})", point.x, point.y))
.collect::<Vec<_>>()
- .join(" ");
- format!("[{:.3}] {corners}", region.confidence)
+ .join(" ")
}
fn dump_target(dir: &Path, image_path: &str) -> Result<PathBuf> {
@@ -126,38 +230,45 @@ fn dump_target(dir: &Path, image_path: &str) -> Result<PathBuf> {
Ok(target)
}
-fn dump(target: &Path, image_path: &str, debug: &RegionDetectionDebug) -> Result<()> {
+fn dump_regions(target: &Path, image_path: &str, debug: &RegionDetectionDebug) -> Result<()> {
probability_map_image(&debug.probability_map)?
.save(target.join("probmap.png"))
.context("write probmap.png")?;
- overlay_image(image_path, &debug.result.regions)?
+ let mut canvas = decoded_canvas(image_path)?;
+ let thickness = line_thickness(&canvas);
+ for region in &debug.result.regions {
+ draw_quad(&mut canvas, &region.points, thickness, BLOCK_COLOR);
+ }
+ canvas
.save(target.join("overlay.png"))
.context("write overlay.png")?;
println!(" wrote {}", target.display());
Ok(())
}
-fn dump_crops(engine: &OcrEngine, image_path: &str, target: &Path) -> Result<()> {
- let debug = engine.detect_and_crop_debug(&DetectRegionsRequest {
- image_path: image_path.to_string(),
- request_id: None,
- })?;
+fn dump_text(target: &Path, image_path: &str, debug: &TextDetectionDebug) -> Result<()> {
+ probability_map_image(&debug.probability_map)?
+ .save(target.join("probmap.png"))
+ .context("write probmap.png")?;
+ block_overlay(image_path, &debug.result.blocks)?
+ .save(target.join("overlay.png"))
+ .context("write overlay.png")?;
+ dump_crops(target, &debug.crops)?;
+ println!(" wrote {}", target.display());
+ Ok(())
+}
+
+fn dump_crops(target: &Path, crops: &[CropDebug]) -> Result<()> {
let crops_dir = target.join("crops");
std::fs::create_dir_all(&crops_dir)
.with_context(|| format!("create {}", crops_dir.display()))?;
- for (index, crop) in debug.crops.iter().enumerate() {
+ for (index, crop) in crops.iter().enumerate() {
println!(" {}", format_crop(index, crop));
let name = crop_file_name(index, crop);
crop_image(crop)?
.save(crops_dir.join(&name))
.with_context(|| format!("write {name}"))?;
}
- println!(
- " wrote {} crops for {} candidates to {}",
- debug.crops.len(),
- debug.candidates.len(),
- crops_dir.display()
- );
Ok(())
}
@@ -171,20 +282,21 @@ fn orientation_label(orientation: Orientation) -> &'static str {
fn format_crop(index: usize, crop: &CropDebug) -> String {
let rotated = if crop.rotated { " rotated" } else { "" };
format!(
- "{index:03} {}x{} {} p180={:.3}{rotated}",
+ "{index:03} {}x{} {}{rotated} [{:.3}] {}",
crop.width,
crop.height,
orientation_label(crop.orientation),
- crop.p180
+ crop.confidence,
+ crop.text
)
}
fn crop_file_name(index: usize, crop: &CropDebug) -> String {
let rotated = if crop.rotated { "_rot180" } else { "" };
format!(
- "{index:03}_{}_{:.2}{rotated}.png",
+ "{index:03}_{}{rotated}_{:.2}.png",
orientation_label(crop.orientation),
- crop.p180
+ crop.confidence
)
}
@@ -203,22 +315,33 @@ fn probability_map_image(map: &ProbabilityMap) -> Result<GrayImage> {
.context("probability map size does not match its values")
}
-fn overlay_image(image_path: &str, regions: &[TextRegion]) -> Result<RgbImage> {
+fn decoded_canvas(image_path: &str) -> Result<RgbImage> {
let decoded = decode_image_from_path(image_path)?;
- let mut canvas = RgbImage::from_raw(
+ RgbImage::from_raw(
decoded.dimensions.width,
decoded.dimensions.height,
decoded.rgb,
)
- .context("decoded image size does not match its buffer")?;
- let thickness = (canvas.width().max(canvas.height()) / 800).clamp(1, 6);
- for region in regions {
- draw_quad(&mut canvas, &region.points, thickness);
+ .context("decoded image size does not match its buffer")
+}
+
+fn line_thickness(canvas: &RgbImage) -> u32 {
+ (canvas.width().max(canvas.height()) / 800).clamp(1, 6)
+}
+
+fn block_overlay(image_path: &str, blocks: &[TextBlock]) -> Result<RgbImage> {
+ let mut canvas = decoded_canvas(image_path)?;
+ let thickness = line_thickness(&canvas);
+ for character in blocks.iter().flat_map(|block| &block.characters) {
+ draw_quad(&mut canvas, &character.points, thickness, CHARACTER_COLOR);
+ }
+ for block in blocks {
+ draw_quad(&mut canvas, &block.points, thickness, BLOCK_COLOR);
}
Ok(canvas)
}
-fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32) {
+fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32, color: Rgb<u8>) {
for index in 0..4 {
let start = points[index];
let end = points[(index + 1) % 4];
@@ -228,13 +351,13 @@ fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32) {
canvas,
(start.x + shift, start.y),
(end.x + shift, end.y),
- OVERLAY_COLOR,
+ color,
);
draw_line_segment_mut(
canvas,
(start.x, start.y + shift),
(end.x, end.y + shift),
- OVERLAY_COLOR,
+ color,
);
}
}
diff --git a/rust/crates/ml/src/ocr/characters.rs b/rust/crates/ml/src/ocr/characters.rs
new file mode 100644
index 0000000000..8edc0509be
--- /dev/null
+++ b/rust/crates/ml/src/ocr/characters.rs
@@ -0,0 +1,181 @@
+use super::recognize::CharacterSpan;
+use super::{CharacterBox, Orientation, Point};
+
+const MIN_SPAN_LENGTH: f32 = 1e-4;
+
+pub(crate) fn character_boxes(
+ quad: &[Point; 4],
+ spans: &[CharacterSpan],
+ orientation: Orientation,
+ rotated: bool,
+) -> Vec<CharacterBox> {
+ spans
+ .iter()
+ .filter_map(|span| {
+ let (start, end) = span_range(span, rotated)?;
+ Some(CharacterBox {
+ text: span.text.clone(),
+ confidence: span.confidence,
+ points: span_polygon(quad, orientation, start, end),
+ })
+ })
+ .collect()
+}
+
+fn span_range(span: &CharacterSpan, rotated: bool) -> Option<(f32, f32)> {
+ let (start, end) = if rotated {
+ (1.0 - span.end, 1.0 - span.start)
+ } else {
+ (span.start, span.end)
+ };
+ let start = start.clamp(0.0, 1.0);
+ let end = end.clamp(start, 1.0);
+ (end - start > MIN_SPAN_LENGTH).then_some((start, end))
+}
+
+fn span_polygon(quad: &[Point; 4], orientation: Orientation, start: f32, end: f32) -> [Point; 4] {
+ let [tl, tr, br, bl] = *quad;
+ match orientation {
+ Orientation::Horizontal => [
+ lerp(tl, tr, start),
+ lerp(tl, tr, end),
+ lerp(bl, br, end),
+ lerp(bl, br, start),
+ ],
+ Orientation::Vertical => [
+ lerp(tl, bl, start),
+ lerp(tr, br, start),
+ lerp(tr, br, end),
+ lerp(tl, bl, end),
+ ],
+ }
+}
+
+fn lerp(from: Point, to: Point, ratio: f32) -> Point {
+ from + (to - from) * ratio
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn span(text: &str, start: f32, end: f32) -> CharacterSpan {
+ CharacterSpan {
+ text: text.to_string(),
+ confidence: 0.9,
+ start,
+ end,
+ }
+ }
+
+ fn quad(points: [(f32, f32); 4]) -> [Point; 4] {
+ points.map(|(x, y)| Point::new(x, y))
+ }
+
+ fn wide_quad() -> [Point; 4] {
+ quad([(0.0, 0.0), (100.0, 0.0), (100.0, 10.0), (0.0, 10.0)])
+ }
+
+ fn tall_quad() -> [Point; 4] {
+ quad([(0.0, 0.0), (10.0, 0.0), (10.0, 100.0), (0.0, 100.0)])
+ }
+
+ #[test]
+ fn horizontal_spans_interpolate_along_the_top_and_bottom_edges() {
+ let boxes = character_boxes(
+ &wide_quad(),
+ &[span("a", 0.25, 0.5), span("b", 0.5, 1.0)],
+ Orientation::Horizontal,
+ false,
+ );
+ assert_eq!(boxes.len(), 2);
+ assert_eq!(boxes[0].text, "a");
+ assert_eq!(boxes[0].confidence, 0.9);
+ assert_eq!(
+ boxes[0].points,
+ quad([(25.0, 0.0), (50.0, 0.0), (50.0, 10.0), (25.0, 10.0)])
+ );
+ assert_eq!(
+ boxes[1].points,
+ quad([(50.0, 0.0), (100.0, 0.0), (100.0, 10.0), (50.0, 10.0)])
+ );
+ }
+
+ #[test]
+ fn sheared_quads_interpolate_each_edge_separately() {
+ let sheared = quad([(10.0, 10.0), (50.0, 30.0), (40.0, 50.0), (0.0, 30.0)]);
+ let boxes = character_boxes(
+ &sheared,
+ &[span("a", 0.5, 1.0)],
+ Orientation::Horizontal,
+ false,
+ );
+ assert_eq!(
+ boxes[0].points,
+ quad([(30.0, 20.0), (50.0, 30.0), (40.0, 50.0), (20.0, 40.0)])
+ );
+ }
+
+ #[test]
+ fn vertical_spans_interpolate_along_the_left_and_right_edges() {
+ let boxes = character_boxes(
+ &tall_quad(),
+ &[span("a", 0.25, 0.5)],
+ Orientation::Vertical,
+ false,
+ );
+ assert_eq!(
+ boxes[0].points,
+ quad([(0.0, 25.0), (10.0, 25.0), (10.0, 50.0), (0.0, 50.0)])
+ );
+ }
+
+ #[test]
+ fn rotated_horizontal_spans_are_mirrored() {
+ let boxes = character_boxes(
+ &wide_quad(),
+ &[span("a", 0.25, 0.5)],
+ Orientation::Horizontal,
+ true,
+ );
+ assert_eq!(
+ boxes[0].points,
+ quad([(50.0, 0.0), (75.0, 0.0), (75.0, 10.0), (50.0, 10.0)])
+ );
+ }
+
+ #[test]
+ fn rotated_vertical_spans_are_mirrored_bottom_to_top() {
+ let boxes = character_boxes(
+ &tall_quad(),
+ &[span("a", 0.0, 0.25)],
+ Orientation::Vertical,
+ true,
+ );
+ assert_eq!(
+ boxes[0].points,
+ quad([(0.0, 75.0), (10.0, 75.0), (10.0, 100.0), (0.0, 100.0)])
+ );
+ }
+
+ #[test]
+ fn empty_and_out_of_range_spans_are_dropped() {
+ let boxes = character_boxes(
+ &wide_quad(),
+ &[
+ span("a", 0.5, 0.5),
+ span("b", 0.7, 0.6),
+ span("c", 1.2, 1.5),
+ span("d", -0.5, 0.1),
+ ],
+ Orientation::Horizontal,
+ false,
+ );
+ assert_eq!(boxes.len(), 1);
+ assert_eq!(boxes[0].text, "d");
+ assert_eq!(
+ boxes[0].points,
+ quad([(0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0)])
+ );
+ }
+}
diff --git a/rust/crates/ml/src/ocr/dictionary.rs b/rust/crates/ml/src/ocr/dictionary.rs
index 295be010f0..b0da00b2b8 100644
--- a/rust/crates/ml/src/ocr/dictionary.rs
+++ b/rust/crates/ml/src/ocr/dictionary.rs
@@ -5,7 +5,7 @@ use crate::error::{MlError, MlResult};
const BLANK_TOKEN: &str = "blank";
const SPACE_TOKEN: &str = " ";
-pub fn load_dictionary(path: &Path, expected_size: usize) -> MlResult<Vec<String>> {
+pub(crate) fn load_dictionary(path: &Path, expected_size: usize) -> MlResult<Vec<String>> {
let contents = std::fs::read_to_string(path).map_err(|error| {
MlError::CorruptModel(format!(
"cannot read OCR dictionary {}: {error}",
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index fb6ae9bfe4..8aac580be1 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -1,34 +1,41 @@
pub mod assets;
mod cancel;
+mod characters;
mod classify;
mod crop;
mod detect;
mod dictionary;
mod geometry;
+mod recognize;
mod source;
mod tensor;
pub use crop::Orientation;
pub use detect::ProbabilityMap;
-pub use dictionary::load_dictionary;
pub use geometry::Point;
use std::time::Instant;
use thiserror::Error;
-use cancel::RequestRegistry;
-use classify::{AngleClassifier, AngleDecision};
-use crop::crop_text;
+use cancel::{RequestGuard, RequestRegistry};
+use characters::character_boxes;
+use classify::AngleClassifier;
+use crop::{TextCrop, crop_text};
use detect::{DetectionCandidate, TextDetector};
use geometry::scale_points;
-use source::{REGIONS_CAP, SourceImage, load_source};
+use recognize::{Recognition, TextRecognizer};
+use source::{FULL_TEXT_CAP, REGIONS_CAP, SourceImage, load_source};
+use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
const REGION_MIN_SCORE: f32 = 0.5;
const MAX_REGIONS: usize = 1000;
+const MIN_BLOCK_CONFIDENCE: f32 = 0.8;
+const MIN_BLOCK_CONFIDENCE_WITH_ALL_SCORES: f32 = 0.5;
+const RETRY_BELOW_CONFIDENCE: f32 = 0.65;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OcrModelPaths {
@@ -101,15 +108,16 @@ pub struct CropDebug {
pub rgb: Vec<u8>,
pub orientation: Orientation,
pub rotated: bool,
- pub p0: f32,
- pub p180: f32,
+ pub text: String,
+ pub confidence: f32,
}
#[derive(Clone, Debug, PartialEq)]
-pub struct CropDetectionDebug {
- pub image_width: u32,
- pub image_height: u32,
- pub candidates: Vec<TextRegion>,
+pub struct TextDetectionDebug {
+ pub result: TextDetectionResult,
+ pub working_width: u32,
+ pub working_height: u32,
+ pub probability_map: ProbabilityMap,
pub crops: Vec<CropDebug>,
}
@@ -126,6 +134,8 @@ pub enum OcrError {
pub struct OcrEngine {
detector: TextDetector,
classifier: Option<AngleClassifier>,
+ recognizer: Option<TextRecognizer>,
+ missing_text_model_paths: Vec<&'static str>,
requests: RequestRegistry,
}
@@ -138,67 +148,98 @@ impl OcrEngine {
}
let classifier =
(!paths.classification.is_empty()).then(|| AngleClassifier::new(&paths.classification));
+ let recognizer = (!paths.recognition.is_empty())
+ .then(|| TextRecognizer::new(&paths.recognition, &paths.dictionary));
Ok(Self {
detector: TextDetector::new(&paths.detection),
classifier,
+ recognizer,
+ missing_text_model_paths: missing_text_model_paths(&paths),
requests: RequestRegistry::default(),
})
}
- fn classifier(&self) -> MlResult<&AngleClassifier> {
- self.classifier.as_ref().ok_or_else(|| {
- MlError::InvalidRequest("OCR classification model path is empty".to_string())
- })
+ fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {
+ match (&self.classifier, &self.recognizer) {
+ (Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {
+ Ok((classifier, recognizer))
+ }
+ _ => Err(MlError::InvalidRequest(format!(
+ "OCR model paths are empty: {}",
+ self.missing_text_model_paths.join(", ")
+ ))),
+ }
}
- pub fn detect_text_regions(
- &self,
- req: &DetectRegionsRequest,
- ) -> Result<TextRegionDetectionResult, OcrError> {
- self.detect_text_regions_debug(req)
- .map(|debug| debug.result)
+ pub fn detect_text(&self, req: &DetectTextRequest) -> Result<TextDetectionResult, OcrError> {
+ self.run_text_pipeline(req).map(TextPipeline::into_result)
}
- pub fn detect_text_regions_debug(
+ pub fn detect_text_debug(
&self,
- req: &DetectRegionsRequest,
- ) -> Result<RegionDetectionDebug, OcrError> {
+ req: &DetectTextRequest,
+ ) -> Result<TextDetectionDebug, OcrError> {
+ self.run_text_pipeline(req).map(TextPipeline::into_debug)
+ }
+
+ fn run_text_pipeline(&self, req: &DetectTextRequest) -> Result<TextPipeline, OcrError> {
+ let (classifier, recognizer) = self.text_models()?;
let started = Instant::now();
let request = self.requests.begin(req.request_id.as_deref());
- let source = load_source(&req.image_path, REGIONS_CAP)?;
+ let source = load_source(&req.image_path, FULL_TEXT_CAP)?;
request.check()?;
let detection_started = Instant::now();
let detection = self.detector.detect(&source.working)?;
let detection_ms = detection_started.elapsed().as_millis();
request.check()?;
- let regions = regions_in_decoded_pixels(&detection.candidates, &source);
+ let crops = crop_candidates(&source.working, &detection.candidates)?;
+ request.check()?;
+ let recognition_started = Instant::now();
+ let mut recognized = recognize_crops(recognizer, crops, &request)?;
+ request.check()?;
+ let retries = retry_low_confidence(classifier, recognizer, &mut recognized, &request)?;
+ let recognition_ms = recognition_started.elapsed().as_millis();
+ request.check()?;
+ let blocks = text_blocks(
+ &detection.candidates,
+ &recognized,
+ &source,
+ min_block_confidence(req.include_all_confidence_scores),
+ );
log::info!(
- "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",
+ "ocr detect_text: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, rec {} crops in {recognition_ms}ms, cls retries {retries}, kept {}, total {}ms",
source.decoded_width,
source.decoded_height,
source.working.width,
source.working.height,
detection.candidates.len(),
- regions.len(),
+ recognized.len(),
+ blocks.len(),
started.elapsed().as_millis()
);
- Ok(RegionDetectionDebug {
- result: TextRegionDetectionResult {
- regions,
- image_width: source.decoded_width,
- image_height: source.decoded_height,
- },
+ Ok(TextPipeline {
+ image_width: source.decoded_width,
+ image_height: source.decoded_height,
working_width: source.working.width as u32,
working_height: source.working.height as u32,
probability_map: detection.probability_map,
+ crops: recognized,
+ blocks,
})
}
- pub fn detect_and_crop_debug(
+ pub fn detect_text_regions(
&self,
req: &DetectRegionsRequest,
- ) -> Result<CropDetectionDebug, OcrError> {
- let classifier = self.classifier()?;
+ ) -> Result<TextRegionDetectionResult, OcrError> {
+ self.detect_text_regions_debug(req)
+ .map(|debug| debug.result)
+ }
+
+ pub fn detect_text_regions_debug(
+ &self,
+ req: &DetectRegionsRequest,
+ ) -> Result<RegionDetectionDebug, OcrError> {
let started = Instant::now();
let request = self.requests.begin(req.request_id.as_deref());
let source = load_source(&req.image_path, REGIONS_CAP)?;
@@ -207,29 +248,26 @@ impl OcrEngine {
let detection = self.detector.detect(&source.working)?;
let detection_ms = detection_started.elapsed().as_millis();
request.check()?;
- let (mut images, orientations) = crop_candidates(&source.working, &detection.candidates)?;
- request.check()?;
- let classification_started = Instant::now();
- let decisions = classifier.classify(&mut images)?;
- let classification_ms = classification_started.elapsed().as_millis();
- request.check()?;
- let crops = crop_debug_entries(images, orientations, &decisions);
+ let regions = regions_in_decoded_pixels(&detection.candidates, &source);
log::info!(
- "ocr detect_and_crop_debug: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, cls {} crops in {classification_ms}ms, rotated {}, total {}ms",
+ "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",
source.decoded_width,
source.decoded_height,
source.working.width,
source.working.height,
detection.candidates.len(),
- crops.len(),
- decisions.iter().filter(|d| d.rotated).count(),
+ regions.len(),
started.elapsed().as_millis()
);
- Ok(CropDetectionDebug {
- image_width: source.decoded_width,
- image_height: source.decoded_height,
- candidates: candidates_in_decoded_pixels(&detection.candidates, &source),
- crops,
+ Ok(RegionDetectionDebug {
+ result: TextRegionDetectionResult {
+ regions,
+ image_width: source.decoded_width,
+ image_height: source.decoded_height,
+ },
+ working_width: source.working.width as u32,
+ working_height: source.working.height as u32,
+ probability_map: detection.probability_map,
})
}
@@ -238,71 +276,293 @@ impl OcrEngine {
}
}
+fn missing_text_model_paths(paths: &OcrModelPaths) -> Vec<&'static str> {
+ [
+ ("classification", &paths.classification),
+ ("recognition", &paths.recognition),
+ ("dictionary", &paths.dictionary),
+ ]
+ .into_iter()
+ .filter(|(_, path)| path.is_empty())
+ .map(|(name, _)| name)
+ .collect()
+}
+
+fn min_block_confidence(include_all_confidence_scores: bool) -> f32 {
+ if include_all_confidence_scores {
+ MIN_BLOCK_CONFIDENCE_WITH_ALL_SCORES
+ } else {
+ MIN_BLOCK_CONFIDENCE
+ }
+}
+
+struct TextPipeline {
+ image_width: u32,
+ image_height: u32,
+ working_width: u32,
+ working_height: u32,
+ probability_map: ProbabilityMap,
+ crops: Vec<RecognizedCrop>,
+ blocks: Vec<TextBlock>,
+}
+
+impl TextPipeline {
+ fn into_result(self) -> TextDetectionResult {
+ TextDetectionResult {
+ blocks: self.blocks,
+ image_width: self.image_width,
+ image_height: self.image_height,
+ }
+ }
+
+ fn into_debug(self) -> TextDetectionDebug {
+ let crops = self
+ .crops
+ .into_iter()
+ .map(RecognizedCrop::into_debug)
+ .collect();
+ TextDetectionDebug {
+ result: TextDetectionResult {
+ blocks: self.blocks,
+ image_width: self.image_width,
+ image_height: self.image_height,
+ },
+ working_width: self.working_width,
+ working_height: self.working_height,
+ probability_map: self.probability_map,
+ crops,
+ }
+ }
+}
+
+struct RecognizedCrop {
+ image: ImageU8,
+ orientation: Orientation,
+ rotated: bool,
+ recognition: Recognition,
+}
+
+impl RecognizedCrop {
+ fn upright(crop: TextCrop, recognition: Recognition) -> Self {
+ Self {
+ image: crop.image,
+ orientation: crop.orientation,
+ rotated: false,
+ recognition,
+ }
+ }
+
+ fn prefer_rotated(&mut self, image: ImageU8, recognition: Recognition) {
+ if recognition.confidence > self.recognition.confidence {
+ self.image = image;
+ self.rotated = true;
+ self.recognition = recognition;
+ }
+ }
+
+ fn needs_retry(&self) -> bool {
+ self.orientation == Orientation::Horizontal
+ && self.recognition.confidence < RETRY_BELOW_CONFIDENCE
+ }
+
+ fn into_debug(self) -> CropDebug {
+ CropDebug {
+ width: self.image.width as u32,
+ height: self.image.height as u32,
+ rgb: self.image.data,
+ orientation: self.orientation,
+ rotated: self.rotated,
+ text: self.recognition.text,
+ confidence: self.recognition.confidence,
+ }
+ }
+}
+
fn crop_candidates(
working: &ImageU8,
candidates: &[DetectionCandidate],
-) -> MlResult<(Vec<ImageU8>, Vec<Orientation>)> {
+) -> MlResult<Vec<TextCrop>> {
candidates
.iter()
- .map(|candidate| {
- crop_text(working, &candidate.points).map(|crop| (crop.image, crop.orientation))
- })
+ .map(|candidate| crop_text(working, &candidate.points))
.collect()
}
-fn crop_debug_entries(
- images: Vec<ImageU8>,
- orientations: Vec<Orientation>,
- decisions: &[AngleDecision],
-) -> Vec<CropDebug> {
- images
+fn recognize_crops(
+ recognizer: &TextRecognizer,
+ crops: Vec<TextCrop>,
+ request: &RequestGuard<'_>,
+) -> Result<Vec<RecognizedCrop>, OcrError> {
+ let flipped = crops
+ .iter()
+ .map(flipped_if_vertical)
+ .collect::<MlResult<Vec<_>>>()?;
+ let inputs: Vec<&ImageU8> = crops
+ .iter()
+ .zip(&flipped)
+ .flat_map(|(crop, flipped)| std::iter::once(&crop.image).chain(flipped))
+ .collect();
+ let mut results = recognizer.recognize(&inputs, request)?.into_iter();
+ crops
.into_iter()
- .zip(orientations)
- .zip(decisions)
- .map(|((image, orientation), decision)| CropDebug {
- width: image.width as u32,
- height: image.height as u32,
- rgb: image.data,
- orientation,
- rotated: decision.rotated,
- p0: decision.scores.p0,
- p180: decision.scores.p180,
+ .zip(flipped)
+ .map(|(crop, flipped)| {
+ let mut recognized = RecognizedCrop::upright(crop, next_recognition(&mut results)?);
+ if let Some(image) = flipped {
+ recognized.prefer_rotated(image, next_recognition(&mut results)?);
+ }
+ Ok(recognized)
})
.collect()
}
-fn regions_in_decoded_pixels(
+fn flipped_if_vertical(crop: &TextCrop) -> MlResult<Option<ImageU8>> {
+ match crop.orientation {
+ Orientation::Vertical => cv::rotate_u8(&crop.image, 180)
+ .map(Some)
+ .map_err(MlError::Preprocess),
+ Orientation::Horizontal => Ok(None),
+ }
+}
+
+fn next_recognition(results: &mut impl Iterator<Item = Recognition>) -> MlResult<Recognition> {
+ results.next().ok_or_else(|| {
+ MlError::CorruptModel("text recognizer returned fewer results than crops".to_string())
+ })
+}
+
+fn retry_low_confidence(
+ classifier: &AngleClassifier,
+ recognizer: &TextRecognizer,
+ crops: &mut [RecognizedCrop],
+ request: &RequestGuard<'_>,
+) -> Result<usize, OcrError> {
+ let indices: Vec<usize> = crops
+ .iter()
+ .enumerate()
+ .filter(|(_, crop)| crop.needs_retry())
+ .map(|(index, _)| index)
+ .collect();
+ if indices.is_empty() {
+ return Ok(0);
+ }
+ let mut images: Vec<ImageU8> = indices
+ .iter()
+ .map(|&index| crops[index].image.clone())
+ .collect();
+ let decisions = classifier.classify(&mut images)?;
+ request.check()?;
+ let flipped: Vec<(usize, ImageU8)> = indices
+ .iter()
+ .copied()
+ .zip(images)
+ .zip(decisions)
+ .filter(|(_, decision)| decision.rotated)
+ .map(|(flipped, _)| flipped)
+ .collect();
+ let inputs: Vec<&ImageU8> = flipped.iter().map(|(_, image)| image).collect();
+ let results = recognizer.recognize(&inputs, request)?;
+ for ((index, image), retry) in flipped.into_iter().zip(results) {
+ crops[index].prefer_rotated(image, retry);
+ }
+ Ok(indices.len())
+}
+
+fn text_blocks(
candidates: &[DetectionCandidate],
+ crops: &[RecognizedCrop],
source: &SourceImage,
-) -> Vec<TextRegion> {
- let to_region = decoded_region_of(source);
+ min_confidence: f32,
+) -> Vec<TextBlock> {
+ let (scale_x, scale_y) = decoded_scale(source);
candidates
.iter()
- .filter(|candidate| candidate.score >= REGION_MIN_SCORE)
- .take(MAX_REGIONS)
- .map(to_region)
+ .zip(crops)
+ .filter(|(_, crop)| crop.recognition.confidence >= min_confidence)
+ .map(|(candidate, crop)| TextBlock {
+ text: crop.recognition.text.clone(),
+ confidence: crop.recognition.confidence,
+ points: scale_points(&candidate.points, scale_x, scale_y),
+ characters: character_boxes(
+ &candidate.points,
+ &crop.recognition.spans,
+ crop.orientation,
+ crop.rotated,
+ )
+ .into_iter()
+ .map(|character| CharacterBox {
+ points: scale_points(&character.points, scale_x, scale_y),
+ ..character
+ })
+ .collect(),
+ })
.collect()
}
-fn candidates_in_decoded_pixels(
+fn regions_in_decoded_pixels(
candidates: &[DetectionCandidate],
source: &SourceImage,
) -> Vec<TextRegion> {
- candidates.iter().map(decoded_region_of(source)).collect()
+ let (scale_x, scale_y) = decoded_scale(source);
+ candidates
+ .iter()
+ .filter(|candidate| candidate.score >= REGION_MIN_SCORE)
+ .take(MAX_REGIONS)
+ .map(|candidate| TextRegion {
+ confidence: candidate.score,
+ points: scale_points(&candidate.points, scale_x, scale_y),
+ })
+ .collect()
}
-fn decoded_region_of(source: &SourceImage) -> impl Fn(&DetectionCandidate) -> TextRegion {
- let scale_x = source.decoded_width as f32 / source.working.width as f32;
- let scale_y = source.decoded_height as f32 / source.working.height as f32;
- move |candidate| TextRegion {
- confidence: candidate.score,
- points: scale_points(&candidate.points, scale_x, scale_y),
- }
+fn decoded_scale(source: &SourceImage) -> (f32, f32) {
+ (
+ source.decoded_width as f32 / source.working.width as f32,
+ source.decoded_height as f32 / source.working.height as f32,
+ )
}
#[cfg(test)]
mod tests {
use super::*;
+ use recognize::CharacterSpan;
+
+ fn detector_only_engine() -> OcrEngine {
+ OcrEngine::new(OcrModelPaths {
+ detection: "missing/det.onnx".to_string(),
+ classification: String::new(),
+ recognition: String::new(),
+ dictionary: String::new(),
+ })
+ .unwrap()
+ }
+
+ fn quad(x: f32, y: f32) -> [Point; 4] {
+ [
+ Point::new(x, y),
+ Point::new(x + 10.0, y),
+ Point::new(x + 10.0, y + 5.0),
+ Point::new(x, y + 5.0),
+ ]
+ }
+
+ fn recognized(text: &str, confidence: f32, rotated: bool) -> RecognizedCrop {
+ RecognizedCrop {
+ image: ImageU8::zeros(2, 1, 3).unwrap(),
+ orientation: Orientation::Horizontal,
+ rotated,
+ recognition: Recognition {
+ text: text.to_string(),
+ confidence,
+ spans: vec![CharacterSpan {
+ text: text.to_string(),
+ confidence,
+ start: 0.0,
+ end: 0.5,
+ }],
+ },
+ }
+ }
#[test]
fn engine_requires_a_detection_model_path() {
@@ -319,31 +579,47 @@ mod tests {
#[test]
fn engine_accepts_a_detector_only_path_set_without_loading_it() {
- let engine = OcrEngine::new(OcrModelPaths {
- detection: "missing/det.onnx".to_string(),
- classification: String::new(),
- recognition: String::new(),
- dictionary: String::new(),
- });
- assert!(engine.is_ok());
+ let engine = detector_only_engine();
+ assert_eq!(
+ engine.missing_text_model_paths,
+ ["classification", "recognition", "dictionary"]
+ );
}
#[test]
fn missing_image_is_reported_before_the_detector_is_touched() {
+ let error = detector_only_engine()
+ .detect_text_regions(&DetectRegionsRequest {
+ image_path: "missing/image.jpg".to_string(),
+ request_id: Some("r1".to_string()),
+ })
+ .unwrap_err();
+ assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");
+ }
+
+ #[test]
+ fn detect_text_names_the_missing_model_paths_before_reading_the_image() {
let engine = OcrEngine::new(OcrModelPaths {
detection: "missing/det.onnx".to_string(),
- classification: String::new(),
+ classification: "missing/cls.onnx".to_string(),
recognition: String::new(),
dictionary: String::new(),
})
.unwrap();
let error = engine
- .detect_text_regions(&DetectRegionsRequest {
+ .detect_text(&DetectTextRequest {
image_path: "missing/image.jpg".to_string(),
- request_id: Some("r1".to_string()),
+ include_all_confidence_scores: false,
+ request_id: None,
})
.unwrap_err();
- assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");
+ let OcrError::Ml(MlError::InvalidRequest(message)) = error else {
+ panic!("{error}");
+ };
+ assert_eq!(
+ message,
+ "OCR model paths are empty: recognition, dictionary"
+ );
}
#[test]
@@ -351,15 +627,7 @@ mod tests {
let source = SourceImage {
decoded_width: 2000,
decoded_height: 1000,
- working: crate::cv::image::ImageU8::zeros(1000, 500, 3).unwrap(),
- };
- let quad = |x: f32, y: f32| {
- [
- Point::new(x, y),
- Point::new(x + 10.0, y),
- Point::new(x + 10.0, y + 5.0),
- Point::new(x, y + 5.0),
- ]
+ working: ImageU8::zeros(1000, 500, 3).unwrap(),
};
let candidates = vec![
DetectionCandidate {
@@ -379,4 +647,72 @@ mod tests {
assert_eq!(regions[0].points[0], Point::new(200.0, 100.0));
assert_eq!(regions[0].points[2], Point::new(220.0, 110.0));
}
+
+ #[test]
+ fn text_blocks_filter_by_confidence_and_scale_blocks_and_characters() {
+ let source = SourceImage {
+ decoded_width: 2000,
+ decoded_height: 1000,
+ working: ImageU8::zeros(1000, 500, 3).unwrap(),
+ };
+ let candidates = vec![
+ DetectionCandidate {
+ points: quad(100.0, 50.0),
+ score: 0.9,
+ },
+ DetectionCandidate {
+ points: quad(300.0, 50.0),
+ score: 0.9,
+ },
+ DetectionCandidate {
+ points: quad(500.0, 50.0),
+ score: 0.9,
+ },
+ ];
+ let crops = vec![
+ recognized("hi", 0.95, false),
+ recognized("lo", 0.6, false),
+ recognized("flip", 0.85, true),
+ ];
+
+ let blocks = text_blocks(&candidates, &crops, &source, MIN_BLOCK_CONFIDENCE);
+
+ assert_eq!(blocks.len(), 2);
+ assert_eq!(blocks[0].text, "hi");
+ assert_eq!(blocks[0].points[0], Point::new(200.0, 100.0));
+ assert_eq!(blocks[0].points[2], Point::new(220.0, 110.0));
+ assert_eq!(blocks[0].characters.len(), 1);
+ assert_eq!(blocks[0].characters[0].text, "hi");
+ assert_eq!(blocks[0].characters[0].points[0], Point::new(200.0, 100.0));
+ assert_eq!(blocks[0].characters[0].points[1], Point::new(210.0, 100.0));
+ assert_eq!(blocks[1].text, "flip");
+ assert_eq!(blocks[1].characters[0].points[0], Point::new(1010.0, 100.0));
+ assert_eq!(blocks[1].characters[0].points[1], Point::new(1020.0, 100.0));
+
+ let lenient = text_blocks(&candidates, &crops, &source, min_block_confidence(true));
+ assert_eq!(lenient.len(), 3);
+ }
+
+ #[test]
+ fn a_rotated_result_replaces_the_upright_one_only_when_more_confident() {
+ let mut crop = recognized("up", 0.7, false);
+ let same = recognized("same", 0.7, false);
+ crop.prefer_rotated(same.image, same.recognition);
+ assert!(!crop.rotated);
+ assert_eq!(crop.recognition.text, "up");
+
+ let better = recognized("down", 0.9, false);
+ crop.prefer_rotated(better.image, better.recognition);
+ assert!(crop.rotated);
+ assert_eq!(crop.recognition.text, "down");
+ }
+
+ #[test]
+ fn only_low_confidence_horizontal_crops_are_retried() {
+ assert!(recognized("a", 0.64, false).needs_retry());
+ assert!(!recognized("a", 0.65, false).needs_retry());
+ let mut vertical = recognized("a", 0.1, false);
+ vertical.orientation = Orientation::Vertical;
+ assert!(!vertical.needs_retry());
+ }
}
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
new file mode 100644
index 0000000000..333d66b758
--- /dev/null
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -0,0 +1,592 @@
+use std::path::Path;
+use std::sync::{Mutex, OnceLock, PoisonError};
+
+use super::OcrError;
+use super::cancel::RequestGuard;
+use super::dictionary::load_dictionary;
+use super::tensor::{BgrNormalization, write_bgr_planes};
+use crate::cv;
+use crate::cv::image::ImageU8;
+use crate::error::{MlError, MlResult};
+use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};
+
+const MODEL_NAMESPACE: &str = "ocr-recognition";
+const REC_VOCABULARY_SIZE: usize = 18385;
+const REC_HEIGHT: i32 = 48;
+const REC_BASE_WIDTH: i32 = 320;
+const REC_BATCH_SIZE: usize = 6;
+const BLANK_INDEX: usize = 0;
+const MIN_SPAN: f32 = 1e-3;
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct CharacterSpan {
+ pub(crate) text: String,
+ pub(crate) confidence: f32,
+ pub(crate) start: f32,
+ pub(crate) end: f32,
+}
+
+#[derive(Clone, Debug, Default, PartialEq)]
+pub(crate) struct Recognition {
+ pub(crate) text: String,
+ pub(crate) confidence: f32,
+ pub(crate) spans: Vec<CharacterSpan>,
+}
+
+struct LazyDictionary {
+ path: String,
+ entries: OnceLock<Vec<String>>,
+}
+
+impl LazyDictionary {
+ fn new(path: &str) -> Self {
+ Self {
+ path: path.to_string(),
+ entries: OnceLock::new(),
+ }
+ }
+
+ fn entries(&self) -> MlResult<&[String]> {
+ if let Some(entries) = self.entries.get() {
+ return Ok(entries);
+ }
+ let loaded = load_dictionary(Path::new(&self.path), REC_VOCABULARY_SIZE)?;
+ Ok(self.entries.get_or_init(|| loaded))
+ }
+}
+
+pub(crate) struct TextRecognizer {
+ session: Mutex<OnnxSession>,
+ dictionary: LazyDictionary,
+}
+
+impl TextRecognizer {
+ pub(crate) fn new(model_path: &str, dictionary_path: &str) -> Self {
+ Self {
+ session: Mutex::new(
+ OnnxSession::new(model_path, MODEL_NAMESPACE, ExecutionMode::CpuAccelerated)
+ .with_unvalidated_acceleration(),
+ ),
+ dictionary: LazyDictionary::new(dictionary_path),
+ }
+ }
+
+ pub(crate) fn recognize(
+ &self,
+ crops: &[&ImageU8],
+ request: &RequestGuard<'_>,
+ ) -> Result<Vec<Recognition>, OcrError> {
+ if crops.is_empty() {
+ return Ok(Vec::new());
+ }
+ let dictionary = self.dictionary.entries()?;
+ recognize_in_batches(crops, request, |batch| self.infer_batch(batch, dictionary))
+ }
+
+ fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {
+ let layout = BatchLayout::new(batch);
+ let input = PreparedF32Input::new(layout.tensor(batch)?);
+ let count = batch.len();
+ let input_shape = [
+ count as i64,
+ 3,
+ i64::from(REC_HEIGHT),
+ i64::from(layout.target_width),
+ ];
+ let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
+ let (output, _usage) = session.run(|session| {
+ let (shape, values) = run_f32(session, &input, input_shape)?;
+ SequenceOutput::new(&shape, values, count, dictionary.len())
+ .map_err(SessionRunError::from)
+ })?;
+ Ok(layout
+ .content_widths
+ .iter()
+ .zip(output.sequences())
+ .map(|(&content_width, logits)| {
+ ctc_decode(
+ logits,
+ output.vocabulary,
+ dictionary,
+ layout.padding_scale(content_width),
+ )
+ })
+ .collect())
+ }
+}
+
+fn recognize_in_batches(
+ crops: &[&ImageU8],
+ request: &RequestGuard<'_>,
+ mut infer_batch: impl FnMut(&[&ImageU8]) -> MlResult<Vec<Recognition>>,
+) -> Result<Vec<Recognition>, OcrError> {
+ let mut results: Vec<Option<Recognition>> = vec![None; crops.len()];
+ for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {
+ request.check()?;
+ let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();
+ let recognized = infer_batch(&batch)?;
+ if recognized.len() != batch.len() {
+ return Err(MlError::CorruptModel(format!(
+ "text recognizer decoded {} crops out of {}",
+ recognized.len(),
+ batch.len()
+ ))
+ .into());
+ }
+ for (&index, recognition) in indices.iter().zip(recognized) {
+ results[index] = Some(recognition);
+ }
+ }
+ results
+ .into_iter()
+ .collect::<Option<Vec<_>>>()
+ .ok_or_else(|| MlError::CorruptModel("text recognizer skipped a crop".to_string()).into())
+}
+
+fn ascending_aspect_order(crops: &[&ImageU8]) -> Vec<usize> {
+ let mut order: Vec<usize> = (0..crops.len()).collect();
+ order.sort_by(|&a, &b| aspect(crops[a]).total_cmp(&aspect(crops[b])));
+ order
+}
+
+fn aspect(crop: &ImageU8) -> f64 {
+ f64::from(crop.width) / f64::from(crop.height)
+}
+
+struct BatchLayout {
+ target_width: i32,
+ content_widths: Vec<i32>,
+}
+
+impl BatchLayout {
+ fn new(batch: &[&ImageU8]) -> Self {
+ let base_ratio = f64::from(REC_BASE_WIDTH) / f64::from(REC_HEIGHT);
+ let max_wh_ratio = batch
+ .iter()
+ .map(|crop| aspect(crop))
+ .fold(base_ratio, f64::max);
+ let target_width = target_width(max_wh_ratio);
+ let content_widths = batch
+ .iter()
+ .map(|crop| content_width(crop, target_width))
+ .collect();
+ Self {
+ target_width,
+ content_widths,
+ }
+ }
+
+ fn padding_scale(&self, content_width: i32) -> f32 {
+ self.target_width as f32 / content_width as f32
+ }
+
+ fn tensor(&self, batch: &[&ImageU8]) -> MlResult<Vec<f32>> {
+ let plane = (self.target_width * REC_HEIGHT) as usize;
+ let mut tensor = vec![0.0f32; batch.len() * 3 * plane];
+ let slots = tensor.chunks_exact_mut(3 * plane);
+ for ((crop, &content_width), slot) in batch.iter().zip(&self.content_widths).zip(slots) {
+ let resized = cv::resize_u8(crop, content_width, REC_HEIGHT, cv::Interp::Bilinear)
+ .map_err(MlError::Preprocess)?;
+ write_bgr_planes(
+ &resized,
+ slot,
+ self.target_width as usize,
+ BgrNormalization::CENTERED,
+ )?;
+ }
+ Ok(tensor)
+ }
+}
+
+fn target_width(max_wh_ratio: f64) -> i32 {
+ ((f64::from(REC_HEIGHT) * max_wh_ratio).trunc() as i32).max(1)
+}
+
+fn content_width(crop: &ImageU8, target_width: i32) -> i32 {
+ let scaled = (f64::from(REC_HEIGHT * crop.width) / f64::from(crop.height)).ceil() as i32;
+ scaled.min(target_width)
+}
+
+#[derive(Debug)]
+struct SequenceOutput {
+ steps: usize,
+ vocabulary: usize,
+ values: Vec<f32>,
+}
+
+impl SequenceOutput {
+ fn new(shape: &[i64], values: Vec<f32>, count: usize, vocabulary: usize) -> MlResult<Self> {
+ match *shape {
+ [n, steps, v]
+ if n == count as i64
+ && steps > 0
+ && v == vocabulary as i64
+ && values.len() == count * steps as usize * vocabulary =>
+ {
+ Ok(Self {
+ steps: steps as usize,
+ vocabulary,
+ values,
+ })
+ }
+ _ => Err(MlError::CorruptModel(format!(
+ "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, {vocabulary}]",
+ values.len()
+ ))),
+ }
+ }
+
+ fn sequences(&self) -> impl Iterator<Item = &[f32]> {
+ self.values.chunks_exact(self.steps * self.vocabulary)
+ }
+}
+
+fn ctc_decode(
+ logits: &[f32],
+ vocabulary: usize,
+ dictionary: &[String],
+ padding_scale: f32,
+) -> Recognition {
+ let best = best_per_step(logits, vocabulary);
+ let spans: Vec<CharacterSpan> = character_runs(&best)
+ .iter()
+ .filter_map(|run| run.span(best.len(), dictionary, padding_scale))
+ .collect();
+ Recognition {
+ text: spans.iter().map(|span| span.text.as_str()).collect(),
+ confidence: mean_confidence(&spans),
+ spans,
+ }
+}
+
+#[derive(Clone, Copy)]
+struct StepBest {
+ index: usize,
+ probability: f32,
+}
+
+fn best_per_step(logits: &[f32], vocabulary: usize) -> Vec<StepBest> {
+ if vocabulary == 0 {
+ return Vec::new();
+ }
+ logits.chunks_exact(vocabulary).map(argmax).collect()
+}
+
+fn argmax(step: &[f32]) -> StepBest {
+ let mut best = StepBest {
+ index: 0,
+ probability: f32::NEG_INFINITY,
+ };
+ for (index, &probability) in step.iter().enumerate() {
+ if probability > best.probability {
+ best = StepBest { index, probability };
+ }
+ }
+ best
+}
+
+struct CharacterRun {
+ index: usize,
+ start: usize,
+ end: usize,
+ confidence: f32,
+}
+
+fn character_runs(best: &[StepBest]) -> Vec<CharacterRun> {
+ let mut runs = Vec::new();
+ let mut start = 0;
+ while start < best.len() {
+ let index = best[start].index;
+ let end = start
+ + best[start..]
+ .iter()
+ .take_while(|b| b.index == index)
+ .count();
+ if index != BLANK_INDEX {
+ let total: f32 = best[start..end].iter().map(|b| b.probability).sum();
+ runs.push(CharacterRun {
+ index,
+ start,
+ end,
+ confidence: total / (end - start) as f32,
+ });
+ }
+ start = end;
+ }
+ runs
+}
+
+impl CharacterRun {
+ fn span(
+ &self,
+ steps: usize,
+ dictionary: &[String],
+ padding_scale: f32,
+ ) -> Option<CharacterSpan> {
+ let text = dictionary.get(self.index)?.clone();
+ let (start, end) = span_bounds(self.start, self.end, steps, padding_scale);
+ Some(CharacterSpan {
+ text,
+ confidence: self.confidence,
+ start,
+ end,
+ })
+ }
+}
+
+fn span_bounds(run_start: usize, run_end: usize, steps: usize, scale: f32) -> (f32, f32) {
+ let steps = steps as f32;
+ let min_span = (scale / steps).max(MIN_SPAN);
+ let mut start = (run_start as f32 / steps * scale).clamp(0.0, 1.0);
+ let mut end = (run_end as f32 / steps * scale).clamp(start, 1.0);
+ if end - start < min_span {
+ end = (start + min_span).min(1.0);
+ if end - start < min_span {
+ start = (end - min_span).max(0.0);
+ }
+ }
+ (start, end)
+}
+
+fn mean_confidence(spans: &[CharacterSpan]) -> f32 {
+ if spans.is_empty() {
+ return 0.0;
+ }
+ spans.iter().map(|span| span.confidence).sum::<f32>() / spans.len() as f32
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::ocr::cancel::RequestRegistry;
+
+ const TOLERANCE: f32 = 1e-6;
+
+ fn dictionary() -> Vec<String> {
+ ["blank", "a", "b", "c"].map(str::to_string).to_vec()
+ }
+
+ fn logits(vocabulary: usize, steps: &[(usize, f32)]) -> Vec<f32> {
+ steps
+ .iter()
+ .flat_map(|&(index, probability)| {
+ let mut row = vec![0.0f32; vocabulary];
+ row[index] = probability;
+ row
+ })
+ .collect()
+ }
+
+ fn solid(width: i32, height: i32, rgb: [u8; 3]) -> ImageU8 {
+ let data = std::iter::repeat_n(rgb, (width * height) as usize)
+ .flatten()
+ .collect();
+ ImageU8::new(width, height, 3, data).unwrap()
+ }
+
+ fn assert_close(actual: f32, expected: f32) {
+ assert!(
+ (actual - expected).abs() <= TOLERANCE,
+ "{actual} != {expected}"
+ );
+ }
+
+ fn assert_span(span: &CharacterSpan, text: &str, confidence: f32, start: f32, end: f32) {
+ assert_eq!(span.text, text);
+ assert_close(span.confidence, confidence);
+ assert_close(span.start, start);
+ assert_close(span.end, end);
+ }
+
+ #[test]
+ fn blank_only_sequence_decodes_to_nothing() {
+ let recognition = ctc_decode(
+ &logits(4, &[(0, 0.9), (0, 0.8), (0, 0.7)]),
+ 4,
+ &dictionary(),
+ 1.0,
+ );
+ assert_eq!(recognition, Recognition::default());
+ }
+
+ #[test]
+ fn repeats_separated_by_a_blank_produce_two_characters() {
+ let steps = [(1, 0.9), (1, 0.9), (0, 0.9), (1, 0.6)];
+ let recognition = ctc_decode(&logits(4, &steps), 4, &dictionary(), 1.0);
+ assert_eq!(recognition.text, "aa");
+ assert_eq!(recognition.spans.len(), 2);
+ assert_span(&recognition.spans[0], "a", 0.9, 0.0, 0.5);
+ assert_span(&recognition.spans[1], "a", 0.6, 0.75, 1.0);
+ assert_close(recognition.confidence, 0.75);
+ }
+
+ #[test]
+ fn adjacent_repeats_collapse_into_one_run_with_the_mean_probability() {
+ let steps = [(1, 0.9), (1, 0.7), (2, 0.5), (0, 0.9), (3, 0.3), (3, 0.5)];
+ let recognition = ctc_decode(&logits(4, &steps), 4, &dictionary(), 1.0);
+ assert_eq!(recognition.text, "abc");
+ assert_span(&recognition.spans[0], "a", 0.8, 0.0, 2.0 / 6.0);
+ assert_span(&recognition.spans[1], "b", 0.5, 2.0 / 6.0, 3.0 / 6.0);
+ assert_span(&recognition.spans[2], "c", 0.4, 4.0 / 6.0, 1.0);
+ assert_close(recognition.confidence, (0.8 + 0.5 + 0.4) / 3.0);
+ }
+
+ #[test]
+ fn argmax_picks_the_first_of_equal_maxima() {
+ let best = argmax(&[0.2, 0.5, 0.5, 0.1]);
+ assert_eq!(best.index, 1);
+ assert_close(best.probability, 0.5);
+ }
+
+ #[test]
+ fn padded_spans_are_stretched_then_clamped_with_the_minimum_span() {
+ let steps = [(1, 1.0), (0, 1.0), (0, 1.0), (2, 1.0)];
+ let recognition = ctc_decode(&logits(4, &steps), 4, &dictionary(), 2.0);
+ assert_eq!(recognition.text, "ab");
+ assert_span(&recognition.spans[0], "a", 1.0, 0.0, 0.5);
+ assert_span(&recognition.spans[1], "b", 1.0, 0.5, 1.0);
+ }
+
+ #[test]
+ fn minimum_span_never_drops_below_one_thousandth() {
+ let mut steps = vec![(0usize, 1.0f32); 2000];
+ steps[0] = (1, 1.0);
+ let recognition = ctc_decode(
+ &logits(2, &steps),
+ 2,
+ &["blank".to_string(), "a".to_string()],
+ 1.0,
+ );
+ assert_span(&recognition.spans[0], "a", 1.0, 0.0, 1e-3);
+ }
+
+ #[test]
+ fn indices_outside_the_dictionary_are_skipped() {
+ let recognition = ctc_decode(&logits(5, &[(4, 0.9), (1, 0.8)]), 5, &dictionary(), 1.0);
+ assert_eq!(recognition.text, "a");
+ assert_eq!(recognition.spans.len(), 1);
+ assert_close(recognition.confidence, 0.8);
+ }
+
+ #[test]
+ fn target_width_truncates_and_content_width_ceils() {
+ assert_eq!(
+ target_width(f64::from(REC_BASE_WIDTH) / f64::from(REC_HEIGHT)),
+ 320
+ );
+ assert_eq!(target_width(7.3), 350);
+ assert_eq!(target_width(0.001), 1);
+ assert_eq!(content_width(&solid(7, 3, [0; 3]), 400), 112);
+ assert_eq!(content_width(&solid(1, 4096, [0; 3]), 320), 1);
+ assert_eq!(content_width(&solid(1000, 48, [0; 3]), 320), 320);
+ }
+
+ #[test]
+ fn batch_layout_uses_the_widest_aspect_but_never_less_than_the_base_width() {
+ let crops = [
+ solid(100, 48, [0; 3]),
+ solid(400, 48, [0; 3]),
+ solid(7, 3, [0; 3]),
+ ];
+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());
+ assert_eq!(layout.target_width, 400);
+ assert_eq!(layout.content_widths, [100, 400, 112]);
+ assert_close(layout.padding_scale(100), 4.0);
+
+ let narrow = [solid(10, 48, [0; 3])];
+ let layout = BatchLayout::new(&narrow.iter().collect::<Vec<_>>());
+ assert_eq!(layout.target_width, 320);
+ assert_eq!(layout.content_widths, [10]);
+ }
+
+ #[test]
+ fn batch_tensor_left_aligns_each_crop_and_zero_pads_the_rest() {
+ let crops = [solid(100, 48, [10, 20, 30]), solid(400, 48, [0; 3])];
+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());
+ let tensor = layout.tensor(&crops.iter().collect::<Vec<_>>()).unwrap();
+ let plane = 48 * 400;
+ assert_eq!(tensor.len(), 2 * 3 * plane);
+ let centered = |value: f32| (value / 255.0 - 0.5) / 0.5;
+ for (channel, expected) in [centered(30.0), centered(20.0), centered(10.0)]
+ .into_iter()
+ .enumerate()
+ {
+ let row = &tensor[channel * plane + 47 * 400..channel * plane + 48 * 400];
+ assert!(
+ row[..100]
+ .iter()
+ .all(|&v| (v - expected).abs() <= TOLERANCE),
+ "channel {channel}"
+ );
+ assert!(row[100..].iter().all(|&v| v == 0.0), "channel {channel}");
+ }
+ }
+
+ #[test]
+ fn sequence_output_requires_the_dictionary_vocabulary() {
+ let ok = SequenceOutput::new(&[1, 2, 4], vec![0.0; 8], 1, 4).unwrap();
+ assert_eq!((ok.steps, ok.vocabulary), (2, 4));
+ assert_eq!(ok.sequences().count(), 1);
+ for shape in [[1, 2, 5], [2, 2, 4], [1, 0, 4]] {
+ let error = SequenceOutput::new(&shape, vec![0.0; 8], 1, 4).unwrap_err();
+ assert!(matches!(error, MlError::CorruptModel(_)), "{error}");
+ }
+ let short = SequenceOutput::new(&[1, 2, 4], vec![0.0; 7], 1, 4).unwrap_err();
+ assert!(matches!(short, MlError::CorruptModel(_)), "{short}");
+ }
+
+ #[test]
+ fn crops_are_batched_by_ascending_aspect_and_restored_to_input_order() {
+ let widths = [80, 10, 60, 30, 70, 20, 50, 40];
+ let crops: Vec<ImageU8> = widths.iter().map(|&w| solid(w, 10, [0; 3])).collect();
+ let registry = RequestRegistry::default();
+ let request = registry.begin(None);
+ let mut batches = Vec::new();
+
+ let results = recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch| {
+ batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());
+ Ok(batch
+ .iter()
+ .map(|crop| Recognition {
+ text: crop.width.to_string(),
+ ..Recognition::default()
+ })
+ .collect())
+ })
+ .unwrap();
+
+ assert_eq!(batches, [vec![10, 20, 30, 40, 50, 60], vec![70, 80]]);
+ let texts: Vec<String> = results.into_iter().map(|r| r.text).collect();
+ let expected: Vec<String> = widths.iter().map(|w| w.to_string()).collect();
+ assert_eq!(texts, expected);
+ }
+
+ #[test]
+ fn a_cancelled_request_stops_before_the_first_batch() {
+ let crop = solid(4, 4, [0; 3]);
+ let registry = RequestRegistry::default();
+ let request = registry.begin(Some("cancelled"));
+ registry.cancel("cancelled");
+
+ let error = recognize_in_batches(&[&crop], &request, |_| unreachable!()).unwrap_err();
+
+ assert!(matches!(error, OcrError::Cancelled), "{error}");
+ }
+
+ #[test]
+ fn a_short_result_list_is_reported_as_a_corrupt_model() {
+ let crops = [solid(4, 4, [0; 3]), solid(4, 4, [0; 3])];
+ let registry = RequestRegistry::default();
+ let request = registry.begin(None);
+
+ let error = recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |_| {
+ Ok(vec![Recognition::default()])
+ })
+ .unwrap_err();
+
+ assert!(
+ matches!(error, OcrError::Ml(MlError::CorruptModel(_))),
+ "{error}"
+ );
+ }
+}
diff --git a/rust/crates/ml/src/ocr/source.rs b/rust/crates/ml/src/ocr/source.rs
index 240ca72bd0..9bb04cea40 100644
--- a/rust/crates/ml/src/ocr/source.rs
+++ b/rust/crates/ml/src/ocr/source.rs
@@ -13,6 +13,11 @@ pub(crate) struct WorkingSizeCap {
max_pixels: u32,
}
+pub(crate) const FULL_TEXT_CAP: WorkingSizeCap = WorkingSizeCap {
+ max_side: 4096,
+ max_pixels: 12_000_000,
+};
+
pub(crate) const REGIONS_CAP: WorkingSizeCap = WorkingSizeCap {
max_side: 1280,
max_pixels: 2_000_000,
@@ -87,11 +92,6 @@ mod tests {
use super::*;
- const FULL_TEXT_CAP: WorkingSizeCap = WorkingSizeCap {
- max_side: 4096,
- max_pixels: 12_000_000,
- };
-
fn write_png(dir: &Path, name: &str, width: u32, height: u32) -> String {
let path = dir.join(name);
let writer = BufWriter::new(File::create(&path).unwrap());