π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Add OCR text recognition
laurenspriem86ead8087d20
diff --git a/rust/crates/ml/examples/ocr.rs b/rust/crates/ml/examples/ocr.rsindex 4ae088696e..849c434776 100644--- a/rust/crates/ml/examples/ocr.rs+++ b/rust/crates/ml/examples/ocr.rs@@ -9,29 +9,58 @@ use anyhow::{Context, Result, bail};use ente_assets::AssetStore;use ente_image::decode::decode_image_from_path;use ente_ml::ocr::{- CropDebug, DetectRegionsRequest, OcrEngine, OcrModelPaths, Orientation, Point, ProbabilityMap,- RegionDetectionDebug, TextRegion, assets,+ CropDebug, DetectRegionsRequest, DetectTextRequest, OcrEngine, OcrModelPaths, Orientation,+ Point, ProbabilityMap, RegionDetectionDebug, TextBlock, TextDetectionDebug,+ TextDetectionResult, TextRegion, assets,};use image::{GrayImage, Rgb, RgbImage};use imageproc::drawing::draw_line_segment_mut;use support::ml_indexing::{asset_cache_dir, load_onnx_runtime, run_with_large_stack};-const OVERLAY_COLOR: Rgb<u8> = Rgb([255, 0, 0]);+const BLOCK_COLOR: Rgb<u8> = Rgb([255, 0, 0]);+const CHARACTER_COLOR: Rgb<u8> = Rgb([0, 0, 255]);struct Options {+ regions: bool,+ all_confidences: bool,dump_dir: Option<PathBuf>,images: Vec<String>,}+struct OcrLogger;++impl log::Log for OcrLogger {+ fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {+ metadata.level() <= log::Level::Info && metadata.target().starts_with("ente_ml::ocr")+ }++ fn log(&self, record: &log::Record<'_>) {+ if self.enabled(record.metadata()) {+ eprintln!("{}", record.args());+ }+ }++ fn flush(&self) {}+}++static LOGGER: OcrLogger = OcrLogger;+#[tokio::main(flavor = "current_thread")]async fn main() -> Result<()> {let options = parse_args(std::env::args().skip(1))?;+ install_logger();load_onnx_runtime().await?;let store = AssetStore::new(asset_cache_dir()?);let paths = assets::ensure_models(&store).await?;run_with_large_stack("ocr", move || run(options, paths))}+fn install_logger() {+ if log::set_logger(&LOGGER).is_ok() {+ log::set_max_level(log::LevelFilter::Info);+ }+}+fn usage() -> &'static str {"usage: cargo run -p ente-ml --example ocr -- [--regions] [--all-confidences] \[--dump-dir DIR] <image>..."@@ -39,12 +68,15 @@ fn usage() -> &'static str {fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Options> {let mut options = Options {+ regions: false,+ all_confidences: false,dump_dir: None,images: Vec::new(),};while let Some(arg) = args.next() {match arg.as_str() {- "--regions" | "--all-confidences" => {}+ "--regions" => options.regions = true,+ "--all-confidences" => options.all_confidences = true,"--dump-dir" => {let dir = args.next().context("--dump-dir needs a directory")?;options.dump_dir = Some(PathBuf::from(dir));@@ -63,12 +95,19 @@ fn run(options: Options, paths: OcrModelPaths) -> Result<()> {let engine = OcrEngine::new(paths)?;let mut failures = 0usize;for image_path in &options.images {- match detect_regions(&engine, image_path, options.dump_dir.as_deref()) {- Ok(()) => {}- Err(error) => {- failures += 1;- eprintln!("{image_path}: {error:#}");- }+ let outcome = if options.regions {+ detect_regions(&engine, image_path, options.dump_dir.as_deref())+ } else {+ detect_text(+ &engine,+ image_path,+ options.all_confidences,+ options.dump_dir.as_deref(),+ )+ };+ if let Err(error) = outcome {+ failures += 1;+ eprintln!("{image_path}: {error:#}");}}if failures > 0 {@@ -77,6 +116,57 @@ fn run(options: Options, paths: OcrModelPaths) -> Result<()> {Ok(())}+fn detect_text(+ engine: &OcrEngine,+ image_path: &str,+ all_confidences: bool,+ dump_dir: Option<&Path>,+) -> Result<()> {+ let request = DetectTextRequest {+ image_path: image_path.to_string(),+ include_all_confidence_scores: all_confidences,+ request_id: None,+ };+ let started = Instant::now();+ match dump_dir {+ None => {+ let result = engine.detect_text(&request)?;+ print_text_result(image_path, &result, None, started.elapsed().as_millis());+ }+ Some(dir) => {+ let debug = engine.detect_text_debug(&request)?;+ print_text_result(+ image_path,+ &debug.result,+ Some((debug.working_width, debug.working_height)),+ started.elapsed().as_millis(),+ );+ let target = dump_target(dir, image_path)?;+ dump_text(&target, image_path, &debug)?;+ }+ }+ Ok(())+}++fn print_text_result(+ image_path: &str,+ result: &TextDetectionResult,+ working: Option<(u32, u32)>,+ elapsed_ms: u128,+) {+ let working = working+ .map(|(width, height)| format!(", working {width}x{height}"))+ .unwrap_or_default();+ println!(+ "{image_path}: decoded {}x{}{working}, {elapsed_ms}ms",+ result.image_width, result.image_height+ );+ for block in &result.blocks {+ println!(" {}", format_block(block));+ }+ println!(" blocks: {}", result.blocks.len());+}+fn detect_regions(engine: &OcrEngine, image_path: &str, dump_dir: Option<&Path>) -> Result<()> {let started = Instant::now();let debug = engine.detect_text_regions_debug(&DetectRegionsRequest {@@ -100,20 +190,34 @@ fn detect_regions(engine: &OcrEngine, image_path: &str, dump_dir: Option<&Path>)}if let Some(dir) = dump_dir {let target = dump_target(dir, image_path)?;- dump(&target, image_path, &debug)?;- dump_crops(engine, image_path, &target)?;+ dump_regions(&target, image_path, &debug)?;}Ok(())}+fn format_block(block: &TextBlock) -> String {+ format!(+ "[{:.3}] {} {}",+ block.confidence,+ block.text,+ format_points(&block.points)+ )+}+fn format_region(region: &TextRegion) -> String {- let corners = region- .points+ format!(+ "[{:.3}] {}",+ region.confidence,+ format_points(®ion.points)+ )+}++fn format_points(points: &[Point; 4]) -> String {+ points.iter().map(|point| format!("({:.0},{:.0})", point.x, point.y)).collect::<Vec<_>>()- .join(" ");- format!("[{:.3}] {corners}", region.confidence)+ .join(" ")}fn dump_target(dir: &Path, image_path: &str) -> Result<PathBuf> {@@ -126,38 +230,45 @@ fn dump_target(dir: &Path, image_path: &str) -> Result<PathBuf> {Ok(target)}-fn dump(target: &Path, image_path: &str, debug: &RegionDetectionDebug) -> Result<()> {+fn dump_regions(target: &Path, image_path: &str, debug: &RegionDetectionDebug) -> Result<()> {probability_map_image(&debug.probability_map)?.save(target.join("probmap.png")).context("write probmap.png")?;- overlay_image(image_path, &debug.result.regions)?+ let mut canvas = decoded_canvas(image_path)?;+ let thickness = line_thickness(&canvas);+ for region in &debug.result.regions {+ draw_quad(&mut canvas, ®ion.points, thickness, BLOCK_COLOR);+ }+ canvas.save(target.join("overlay.png")).context("write overlay.png")?;println!(" wrote {}", target.display());Ok(())}-fn dump_crops(engine: &OcrEngine, image_path: &str, target: &Path) -> Result<()> {- let debug = engine.detect_and_crop_debug(&DetectRegionsRequest {- image_path: image_path.to_string(),- request_id: None,- })?;+fn dump_text(target: &Path, image_path: &str, debug: &TextDetectionDebug) -> Result<()> {+ probability_map_image(&debug.probability_map)?+ .save(target.join("probmap.png"))+ .context("write probmap.png")?;+ block_overlay(image_path, &debug.result.blocks)?+ .save(target.join("overlay.png"))+ .context("write overlay.png")?;+ dump_crops(target, &debug.crops)?;+ println!(" wrote {}", target.display());+ Ok(())+}++fn dump_crops(target: &Path, crops: &[CropDebug]) -> Result<()> {let crops_dir = target.join("crops");std::fs::create_dir_all(&crops_dir).with_context(|| format!("create {}", crops_dir.display()))?;- for (index, crop) in debug.crops.iter().enumerate() {+ for (index, crop) in crops.iter().enumerate() {println!(" {}", format_crop(index, crop));let name = crop_file_name(index, crop);crop_image(crop)?.save(crops_dir.join(&name)).with_context(|| format!("write {name}"))?;}- println!(- " wrote {} crops for {} candidates to {}",- debug.crops.len(),- debug.candidates.len(),- crops_dir.display()- );Ok(())}@@ -171,20 +282,21 @@ fn orientation_label(orientation: Orientation) -> &'static str {fn format_crop(index: usize, crop: &CropDebug) -> String {let rotated = if crop.rotated { " rotated" } else { "" };format!(- "{index:03} {}x{} {} p180={:.3}{rotated}",+ "{index:03} {}x{} {}{rotated} [{:.3}] {}",crop.width,crop.height,orientation_label(crop.orientation),- crop.p180+ crop.confidence,+ crop.text)}fn crop_file_name(index: usize, crop: &CropDebug) -> String {let rotated = if crop.rotated { "_rot180" } else { "" };format!(- "{index:03}_{}_{:.2}{rotated}.png",+ "{index:03}_{}{rotated}_{:.2}.png",orientation_label(crop.orientation),- crop.p180+ crop.confidence)}@@ -203,22 +315,33 @@ fn probability_map_image(map: &ProbabilityMap) -> Result<GrayImage> {.context("probability map size does not match its values")}-fn overlay_image(image_path: &str, regions: &[TextRegion]) -> Result<RgbImage> {+fn decoded_canvas(image_path: &str) -> Result<RgbImage> {let decoded = decode_image_from_path(image_path)?;- let mut canvas = RgbImage::from_raw(+ RgbImage::from_raw(decoded.dimensions.width,decoded.dimensions.height,decoded.rgb,)- .context("decoded image size does not match its buffer")?;- let thickness = (canvas.width().max(canvas.height()) / 800).clamp(1, 6);- for region in regions {- draw_quad(&mut canvas, ®ion.points, thickness);+ .context("decoded image size does not match its buffer")+}++fn line_thickness(canvas: &RgbImage) -> u32 {+ (canvas.width().max(canvas.height()) / 800).clamp(1, 6)+}++fn block_overlay(image_path: &str, blocks: &[TextBlock]) -> Result<RgbImage> {+ let mut canvas = decoded_canvas(image_path)?;+ let thickness = line_thickness(&canvas);+ for character in blocks.iter().flat_map(|block| &block.characters) {+ draw_quad(&mut canvas, &character.points, thickness, CHARACTER_COLOR);+ }+ for block in blocks {+ draw_quad(&mut canvas, &block.points, thickness, BLOCK_COLOR);}Ok(canvas)}-fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32) {+fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32, color: Rgb<u8>) {for index in 0..4 {let start = points[index];let end = points[(index + 1) % 4];@@ -228,13 +351,13 @@ fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32) {canvas,(start.x + shift, start.y),(end.x + shift, end.y),- OVERLAY_COLOR,+ color,);draw_line_segment_mut(canvas,(start.x, start.y + shift),(end.x, end.y + shift),- OVERLAY_COLOR,+ color,);}}diff --git a/rust/crates/ml/src/ocr/characters.rs b/rust/crates/ml/src/ocr/characters.rsnew file mode 100644index 0000000000..8edc0509be--- /dev/null+++ b/rust/crates/ml/src/ocr/characters.rs@@ -0,0 +1,181 @@+use super::recognize::CharacterSpan;+use super::{CharacterBox, Orientation, Point};++const MIN_SPAN_LENGTH: f32 = 1e-4;++pub(crate) fn character_boxes(+ quad: &[Point; 4],+ spans: &[CharacterSpan],+ orientation: Orientation,+ rotated: bool,+) -> Vec<CharacterBox> {+ spans+ .iter()+ .filter_map(|span| {+ let (start, end) = span_range(span, rotated)?;+ Some(CharacterBox {+ text: span.text.clone(),+ confidence: span.confidence,+ points: span_polygon(quad, orientation, start, end),+ })+ })+ .collect()+}++fn span_range(span: &CharacterSpan, rotated: bool) -> Option<(f32, f32)> {+ let (start, end) = if rotated {+ (1.0 - span.end, 1.0 - span.start)+ } else {+ (span.start, span.end)+ };+ let start = start.clamp(0.0, 1.0);+ let end = end.clamp(start, 1.0);+ (end - start > MIN_SPAN_LENGTH).then_some((start, end))+}++fn span_polygon(quad: &[Point; 4], orientation: Orientation, start: f32, end: f32) -> [Point; 4] {+ let [tl, tr, br, bl] = *quad;+ match orientation {+ Orientation::Horizontal => [+ lerp(tl, tr, start),+ lerp(tl, tr, end),+ lerp(bl, br, end),+ lerp(bl, br, start),+ ],+ Orientation::Vertical => [+ lerp(tl, bl, start),+ lerp(tr, br, start),+ lerp(tr, br, end),+ lerp(tl, bl, end),+ ],+ }+}++fn lerp(from: Point, to: Point, ratio: f32) -> Point {+ from + (to - from) * ratio+}++#[cfg(test)]+mod tests {+ use super::*;++ fn span(text: &str, start: f32, end: f32) -> CharacterSpan {+ CharacterSpan {+ text: text.to_string(),+ confidence: 0.9,+ start,+ end,+ }+ }++ fn quad(points: [(f32, f32); 4]) -> [Point; 4] {+ points.map(|(x, y)| Point::new(x, y))+ }++ fn wide_quad() -> [Point; 4] {+ quad([(0.0, 0.0), (100.0, 0.0), (100.0, 10.0), (0.0, 10.0)])+ }++ fn tall_quad() -> [Point; 4] {+ quad([(0.0, 0.0), (10.0, 0.0), (10.0, 100.0), (0.0, 100.0)])+ }++ #[test]+ fn horizontal_spans_interpolate_along_the_top_and_bottom_edges() {+ let boxes = character_boxes(+ &wide_quad(),+ &[span("a", 0.25, 0.5), span("b", 0.5, 1.0)],+ Orientation::Horizontal,+ false,+ );+ assert_eq!(boxes.len(), 2);+ assert_eq!(boxes[0].text, "a");+ assert_eq!(boxes[0].confidence, 0.9);+ assert_eq!(+ boxes[0].points,+ quad([(25.0, 0.0), (50.0, 0.0), (50.0, 10.0), (25.0, 10.0)])+ );+ assert_eq!(+ boxes[1].points,+ quad([(50.0, 0.0), (100.0, 0.0), (100.0, 10.0), (50.0, 10.0)])+ );+ }++ #[test]+ fn sheared_quads_interpolate_each_edge_separately() {+ let sheared = quad([(10.0, 10.0), (50.0, 30.0), (40.0, 50.0), (0.0, 30.0)]);+ let boxes = character_boxes(+ &sheared,+ &[span("a", 0.5, 1.0)],+ Orientation::Horizontal,+ false,+ );+ assert_eq!(+ boxes[0].points,+ quad([(30.0, 20.0), (50.0, 30.0), (40.0, 50.0), (20.0, 40.0)])+ );+ }++ #[test]+ fn vertical_spans_interpolate_along_the_left_and_right_edges() {+ let boxes = character_boxes(+ &tall_quad(),+ &[span("a", 0.25, 0.5)],+ Orientation::Vertical,+ false,+ );+ assert_eq!(+ boxes[0].points,+ quad([(0.0, 25.0), (10.0, 25.0), (10.0, 50.0), (0.0, 50.0)])+ );+ }++ #[test]+ fn rotated_horizontal_spans_are_mirrored() {+ let boxes = character_boxes(+ &wide_quad(),+ &[span("a", 0.25, 0.5)],+ Orientation::Horizontal,+ true,+ );+ assert_eq!(+ boxes[0].points,+ quad([(50.0, 0.0), (75.0, 0.0), (75.0, 10.0), (50.0, 10.0)])+ );+ }++ #[test]+ fn rotated_vertical_spans_are_mirrored_bottom_to_top() {+ let boxes = character_boxes(+ &tall_quad(),+ &[span("a", 0.0, 0.25)],+ Orientation::Vertical,+ true,+ );+ assert_eq!(+ boxes[0].points,+ quad([(0.0, 75.0), (10.0, 75.0), (10.0, 100.0), (0.0, 100.0)])+ );+ }++ #[test]+ fn empty_and_out_of_range_spans_are_dropped() {+ let boxes = character_boxes(+ &wide_quad(),+ &[+ span("a", 0.5, 0.5),+ span("b", 0.7, 0.6),+ span("c", 1.2, 1.5),+ span("d", -0.5, 0.1),+ ],+ Orientation::Horizontal,+ false,+ );+ assert_eq!(boxes.len(), 1);+ assert_eq!(boxes[0].text, "d");+ assert_eq!(+ boxes[0].points,+ quad([(0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0)])+ );+ }+}diff --git a/rust/crates/ml/src/ocr/dictionary.rs b/rust/crates/ml/src/ocr/dictionary.rsindex 295be010f0..b0da00b2b8 100644--- a/rust/crates/ml/src/ocr/dictionary.rs+++ b/rust/crates/ml/src/ocr/dictionary.rs@@ -5,7 +5,7 @@ use crate::error::{MlError, MlResult};const BLANK_TOKEN: &str = "blank";const SPACE_TOKEN: &str = " ";-pub fn load_dictionary(path: &Path, expected_size: usize) -> MlResult<Vec<String>> {+pub(crate) fn load_dictionary(path: &Path, expected_size: usize) -> MlResult<Vec<String>> {let contents = std::fs::read_to_string(path).map_err(|error| {MlError::CorruptModel(format!("cannot read OCR dictionary {}: {error}",diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex fb6ae9bfe4..8aac580be1 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -1,34 +1,41 @@pub mod assets;mod cancel;+mod characters;mod classify;mod crop;mod detect;mod dictionary;mod geometry;+mod recognize;mod source;mod tensor;pub use crop::Orientation;pub use detect::ProbabilityMap;-pub use dictionary::load_dictionary;pub use geometry::Point;use std::time::Instant;use thiserror::Error;-use cancel::RequestRegistry;-use classify::{AngleClassifier, AngleDecision};-use crop::crop_text;+use cancel::{RequestGuard, RequestRegistry};+use characters::character_boxes;+use classify::AngleClassifier;+use crop::{TextCrop, crop_text};use detect::{DetectionCandidate, TextDetector};use geometry::scale_points;-use source::{REGIONS_CAP, SourceImage, load_source};+use recognize::{Recognition, TextRecognizer};+use source::{FULL_TEXT_CAP, REGIONS_CAP, SourceImage, load_source};+use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};const REGION_MIN_SCORE: f32 = 0.5;const MAX_REGIONS: usize = 1000;+const MIN_BLOCK_CONFIDENCE: f32 = 0.8;+const MIN_BLOCK_CONFIDENCE_WITH_ALL_SCORES: f32 = 0.5;+const RETRY_BELOW_CONFIDENCE: f32 = 0.65;#[derive(Clone, Debug, PartialEq, Eq)]pub struct OcrModelPaths {@@ -101,15 +108,16 @@ pub struct CropDebug {pub rgb: Vec<u8>,pub orientation: Orientation,pub rotated: bool,- pub p0: f32,- pub p180: f32,+ pub text: String,+ pub confidence: f32,}#[derive(Clone, Debug, PartialEq)]-pub struct CropDetectionDebug {- pub image_width: u32,- pub image_height: u32,- pub candidates: Vec<TextRegion>,+pub struct TextDetectionDebug {+ pub result: TextDetectionResult,+ pub working_width: u32,+ pub working_height: u32,+ pub probability_map: ProbabilityMap,pub crops: Vec<CropDebug>,}@@ -126,6 +134,8 @@ pub enum OcrError {pub struct OcrEngine {detector: TextDetector,classifier: Option<AngleClassifier>,+ recognizer: Option<TextRecognizer>,+ missing_text_model_paths: Vec<&'static str>,requests: RequestRegistry,}@@ -138,67 +148,98 @@ impl OcrEngine {}let classifier =(!paths.classification.is_empty()).then(|| AngleClassifier::new(&paths.classification));+ let recognizer = (!paths.recognition.is_empty())+ .then(|| TextRecognizer::new(&paths.recognition, &paths.dictionary));Ok(Self {detector: TextDetector::new(&paths.detection),classifier,+ recognizer,+ missing_text_model_paths: missing_text_model_paths(&paths),requests: RequestRegistry::default(),})}- fn classifier(&self) -> MlResult<&AngleClassifier> {- self.classifier.as_ref().ok_or_else(|| {- MlError::InvalidRequest("OCR classification model path is empty".to_string())- })+ fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {+ match (&self.classifier, &self.recognizer) {+ (Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {+ Ok((classifier, recognizer))+ }+ _ => Err(MlError::InvalidRequest(format!(+ "OCR model paths are empty: {}",+ self.missing_text_model_paths.join(", ")+ ))),+ }}- pub fn detect_text_regions(- &self,- req: &DetectRegionsRequest,- ) -> Result<TextRegionDetectionResult, OcrError> {- self.detect_text_regions_debug(req)- .map(|debug| debug.result)+ pub fn detect_text(&self, req: &DetectTextRequest) -> Result<TextDetectionResult, OcrError> {+ self.run_text_pipeline(req).map(TextPipeline::into_result)}- pub fn detect_text_regions_debug(+ pub fn detect_text_debug(&self,- req: &DetectRegionsRequest,- ) -> Result<RegionDetectionDebug, OcrError> {+ req: &DetectTextRequest,+ ) -> Result<TextDetectionDebug, OcrError> {+ self.run_text_pipeline(req).map(TextPipeline::into_debug)+ }++ fn run_text_pipeline(&self, req: &DetectTextRequest) -> Result<TextPipeline, OcrError> {+ let (classifier, recognizer) = self.text_models()?;let started = Instant::now();let request = self.requests.begin(req.request_id.as_deref());- let source = load_source(&req.image_path, REGIONS_CAP)?;+ let source = load_source(&req.image_path, FULL_TEXT_CAP)?;request.check()?;let detection_started = Instant::now();let detection = self.detector.detect(&source.working)?;let detection_ms = detection_started.elapsed().as_millis();request.check()?;- let regions = regions_in_decoded_pixels(&detection.candidates, &source);+ let crops = crop_candidates(&source.working, &detection.candidates)?;+ request.check()?;+ let recognition_started = Instant::now();+ let mut recognized = recognize_crops(recognizer, crops, &request)?;+ request.check()?;+ let retries = retry_low_confidence(classifier, recognizer, &mut recognized, &request)?;+ let recognition_ms = recognition_started.elapsed().as_millis();+ request.check()?;+ let blocks = text_blocks(+ &detection.candidates,+ &recognized,+ &source,+ min_block_confidence(req.include_all_confidence_scores),+ );log::info!(- "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",+ "ocr detect_text: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, rec {} crops in {recognition_ms}ms, cls retries {retries}, kept {}, total {}ms",source.decoded_width,source.decoded_height,source.working.width,source.working.height,detection.candidates.len(),- regions.len(),+ recognized.len(),+ blocks.len(),started.elapsed().as_millis());- Ok(RegionDetectionDebug {- result: TextRegionDetectionResult {- regions,- image_width: source.decoded_width,- image_height: source.decoded_height,- },+ Ok(TextPipeline {+ image_width: source.decoded_width,+ image_height: source.decoded_height,working_width: source.working.width as u32,working_height: source.working.height as u32,probability_map: detection.probability_map,+ crops: recognized,+ blocks,})}- pub fn detect_and_crop_debug(+ pub fn detect_text_regions(&self,req: &DetectRegionsRequest,- ) -> Result<CropDetectionDebug, OcrError> {- let classifier = self.classifier()?;+ ) -> Result<TextRegionDetectionResult, OcrError> {+ self.detect_text_regions_debug(req)+ .map(|debug| debug.result)+ }++ pub fn detect_text_regions_debug(+ &self,+ req: &DetectRegionsRequest,+ ) -> Result<RegionDetectionDebug, OcrError> {let started = Instant::now();let request = self.requests.begin(req.request_id.as_deref());let source = load_source(&req.image_path, REGIONS_CAP)?;@@ -207,29 +248,26 @@ impl OcrEngine {let detection = self.detector.detect(&source.working)?;let detection_ms = detection_started.elapsed().as_millis();request.check()?;- let (mut images, orientations) = crop_candidates(&source.working, &detection.candidates)?;- request.check()?;- let classification_started = Instant::now();- let decisions = classifier.classify(&mut images)?;- let classification_ms = classification_started.elapsed().as_millis();- request.check()?;- let crops = crop_debug_entries(images, orientations, &decisions);+ let regions = regions_in_decoded_pixels(&detection.candidates, &source);log::info!(- "ocr detect_and_crop_debug: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, cls {} crops in {classification_ms}ms, rotated {}, total {}ms",+ "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",source.decoded_width,source.decoded_height,source.working.width,source.working.height,detection.candidates.len(),- crops.len(),- decisions.iter().filter(|d| d.rotated).count(),+ regions.len(),started.elapsed().as_millis());- Ok(CropDetectionDebug {- image_width: source.decoded_width,- image_height: source.decoded_height,- candidates: candidates_in_decoded_pixels(&detection.candidates, &source),- crops,+ Ok(RegionDetectionDebug {+ result: TextRegionDetectionResult {+ regions,+ image_width: source.decoded_width,+ image_height: source.decoded_height,+ },+ working_width: source.working.width as u32,+ working_height: source.working.height as u32,+ probability_map: detection.probability_map,})}@@ -238,71 +276,293 @@ impl OcrEngine {}}+fn missing_text_model_paths(paths: &OcrModelPaths) -> Vec<&'static str> {+ [+ ("classification", &paths.classification),+ ("recognition", &paths.recognition),+ ("dictionary", &paths.dictionary),+ ]+ .into_iter()+ .filter(|(_, path)| path.is_empty())+ .map(|(name, _)| name)+ .collect()+}++fn min_block_confidence(include_all_confidence_scores: bool) -> f32 {+ if include_all_confidence_scores {+ MIN_BLOCK_CONFIDENCE_WITH_ALL_SCORES+ } else {+ MIN_BLOCK_CONFIDENCE+ }+}++struct TextPipeline {+ image_width: u32,+ image_height: u32,+ working_width: u32,+ working_height: u32,+ probability_map: ProbabilityMap,+ crops: Vec<RecognizedCrop>,+ blocks: Vec<TextBlock>,+}++impl TextPipeline {+ fn into_result(self) -> TextDetectionResult {+ TextDetectionResult {+ blocks: self.blocks,+ image_width: self.image_width,+ image_height: self.image_height,+ }+ }++ fn into_debug(self) -> TextDetectionDebug {+ let crops = self+ .crops+ .into_iter()+ .map(RecognizedCrop::into_debug)+ .collect();+ TextDetectionDebug {+ result: TextDetectionResult {+ blocks: self.blocks,+ image_width: self.image_width,+ image_height: self.image_height,+ },+ working_width: self.working_width,+ working_height: self.working_height,+ probability_map: self.probability_map,+ crops,+ }+ }+}++struct RecognizedCrop {+ image: ImageU8,+ orientation: Orientation,+ rotated: bool,+ recognition: Recognition,+}++impl RecognizedCrop {+ fn upright(crop: TextCrop, recognition: Recognition) -> Self {+ Self {+ image: crop.image,+ orientation: crop.orientation,+ rotated: false,+ recognition,+ }+ }++ fn prefer_rotated(&mut self, image: ImageU8, recognition: Recognition) {+ if recognition.confidence > self.recognition.confidence {+ self.image = image;+ self.rotated = true;+ self.recognition = recognition;+ }+ }++ fn needs_retry(&self) -> bool {+ self.orientation == Orientation::Horizontal+ && self.recognition.confidence < RETRY_BELOW_CONFIDENCE+ }++ fn into_debug(self) -> CropDebug {+ CropDebug {+ width: self.image.width as u32,+ height: self.image.height as u32,+ rgb: self.image.data,+ orientation: self.orientation,+ rotated: self.rotated,+ text: self.recognition.text,+ confidence: self.recognition.confidence,+ }+ }+}+fn crop_candidates(working: &ImageU8,candidates: &[DetectionCandidate],-) -> MlResult<(Vec<ImageU8>, Vec<Orientation>)> {+) -> MlResult<Vec<TextCrop>> {candidates.iter()- .map(|candidate| {- crop_text(working, &candidate.points).map(|crop| (crop.image, crop.orientation))- })+ .map(|candidate| crop_text(working, &candidate.points)).collect()}-fn crop_debug_entries(- images: Vec<ImageU8>,- orientations: Vec<Orientation>,- decisions: &[AngleDecision],-) -> Vec<CropDebug> {- images+fn recognize_crops(+ recognizer: &TextRecognizer,+ crops: Vec<TextCrop>,+ request: &RequestGuard<'_>,+) -> Result<Vec<RecognizedCrop>, OcrError> {+ let flipped = crops+ .iter()+ .map(flipped_if_vertical)+ .collect::<MlResult<Vec<_>>>()?;+ let inputs: Vec<&ImageU8> = crops+ .iter()+ .zip(&flipped)+ .flat_map(|(crop, flipped)| std::iter::once(&crop.image).chain(flipped))+ .collect();+ let mut results = recognizer.recognize(&inputs, request)?.into_iter();+ crops.into_iter()- .zip(orientations)- .zip(decisions)- .map(|((image, orientation), decision)| CropDebug {- width: image.width as u32,- height: image.height as u32,- rgb: image.data,- orientation,- rotated: decision.rotated,- p0: decision.scores.p0,- p180: decision.scores.p180,+ .zip(flipped)+ .map(|(crop, flipped)| {+ let mut recognized = RecognizedCrop::upright(crop, next_recognition(&mut results)?);+ if let Some(image) = flipped {+ recognized.prefer_rotated(image, next_recognition(&mut results)?);+ }+ Ok(recognized)}).collect()}-fn regions_in_decoded_pixels(+fn flipped_if_vertical(crop: &TextCrop) -> MlResult<Option<ImageU8>> {+ match crop.orientation {+ Orientation::Vertical => cv::rotate_u8(&crop.image, 180)+ .map(Some)+ .map_err(MlError::Preprocess),+ Orientation::Horizontal => Ok(None),+ }+}++fn next_recognition(results: &mut impl Iterator<Item = Recognition>) -> MlResult<Recognition> {+ results.next().ok_or_else(|| {+ MlError::CorruptModel("text recognizer returned fewer results than crops".to_string())+ })+}++fn retry_low_confidence(+ classifier: &AngleClassifier,+ recognizer: &TextRecognizer,+ crops: &mut [RecognizedCrop],+ request: &RequestGuard<'_>,+) -> Result<usize, OcrError> {+ let indices: Vec<usize> = crops+ .iter()+ .enumerate()+ .filter(|(_, crop)| crop.needs_retry())+ .map(|(index, _)| index)+ .collect();+ if indices.is_empty() {+ return Ok(0);+ }+ let mut images: Vec<ImageU8> = indices+ .iter()+ .map(|&index| crops[index].image.clone())+ .collect();+ let decisions = classifier.classify(&mut images)?;+ request.check()?;+ let flipped: Vec<(usize, ImageU8)> = indices+ .iter()+ .copied()+ .zip(images)+ .zip(decisions)+ .filter(|(_, decision)| decision.rotated)+ .map(|(flipped, _)| flipped)+ .collect();+ let inputs: Vec<&ImageU8> = flipped.iter().map(|(_, image)| image).collect();+ let results = recognizer.recognize(&inputs, request)?;+ for ((index, image), retry) in flipped.into_iter().zip(results) {+ crops[index].prefer_rotated(image, retry);+ }+ Ok(indices.len())+}++fn text_blocks(candidates: &[DetectionCandidate],+ crops: &[RecognizedCrop],source: &SourceImage,-) -> Vec<TextRegion> {- let to_region = decoded_region_of(source);+ min_confidence: f32,+) -> Vec<TextBlock> {+ let (scale_x, scale_y) = decoded_scale(source);candidates.iter()- .filter(|candidate| candidate.score >= REGION_MIN_SCORE)- .take(MAX_REGIONS)- .map(to_region)+ .zip(crops)+ .filter(|(_, crop)| crop.recognition.confidence >= min_confidence)+ .map(|(candidate, crop)| TextBlock {+ text: crop.recognition.text.clone(),+ confidence: crop.recognition.confidence,+ points: scale_points(&candidate.points, scale_x, scale_y),+ characters: character_boxes(+ &candidate.points,+ &crop.recognition.spans,+ crop.orientation,+ crop.rotated,+ )+ .into_iter()+ .map(|character| CharacterBox {+ points: scale_points(&character.points, scale_x, scale_y),+ ..character+ })+ .collect(),+ }).collect()}-fn candidates_in_decoded_pixels(+fn regions_in_decoded_pixels(candidates: &[DetectionCandidate],source: &SourceImage,) -> Vec<TextRegion> {- candidates.iter().map(decoded_region_of(source)).collect()+ let (scale_x, scale_y) = decoded_scale(source);+ candidates+ .iter()+ .filter(|candidate| candidate.score >= REGION_MIN_SCORE)+ .take(MAX_REGIONS)+ .map(|candidate| TextRegion {+ confidence: candidate.score,+ points: scale_points(&candidate.points, scale_x, scale_y),+ })+ .collect()}-fn decoded_region_of(source: &SourceImage) -> impl Fn(&DetectionCandidate) -> TextRegion {- let scale_x = source.decoded_width as f32 / source.working.width as f32;- let scale_y = source.decoded_height as f32 / source.working.height as f32;- move |candidate| TextRegion {- confidence: candidate.score,- points: scale_points(&candidate.points, scale_x, scale_y),- }+fn decoded_scale(source: &SourceImage) -> (f32, f32) {+ (+ source.decoded_width as f32 / source.working.width as f32,+ source.decoded_height as f32 / source.working.height as f32,+ )}#[cfg(test)]mod tests {use super::*;+ use recognize::CharacterSpan;++ fn detector_only_engine() -> OcrEngine {+ OcrEngine::new(OcrModelPaths {+ detection: "missing/det.onnx".to_string(),+ classification: String::new(),+ recognition: String::new(),+ dictionary: String::new(),+ })+ .unwrap()+ }++ fn quad(x: f32, y: f32) -> [Point; 4] {+ [+ Point::new(x, y),+ Point::new(x + 10.0, y),+ Point::new(x + 10.0, y + 5.0),+ Point::new(x, y + 5.0),+ ]+ }++ fn recognized(text: &str, confidence: f32, rotated: bool) -> RecognizedCrop {+ RecognizedCrop {+ image: ImageU8::zeros(2, 1, 3).unwrap(),+ orientation: Orientation::Horizontal,+ rotated,+ recognition: Recognition {+ text: text.to_string(),+ confidence,+ spans: vec![CharacterSpan {+ text: text.to_string(),+ confidence,+ start: 0.0,+ end: 0.5,+ }],+ },+ }+ }#[test]fn engine_requires_a_detection_model_path() {@@ -319,31 +579,47 @@ mod tests {#[test]fn engine_accepts_a_detector_only_path_set_without_loading_it() {- let engine = OcrEngine::new(OcrModelPaths {- detection: "missing/det.onnx".to_string(),- classification: String::new(),- recognition: String::new(),- dictionary: String::new(),- });- assert!(engine.is_ok());+ let engine = detector_only_engine();+ assert_eq!(+ engine.missing_text_model_paths,+ ["classification", "recognition", "dictionary"]+ );}#[test]fn missing_image_is_reported_before_the_detector_is_touched() {+ let error = detector_only_engine()+ .detect_text_regions(&DetectRegionsRequest {+ image_path: "missing/image.jpg".to_string(),+ request_id: Some("r1".to_string()),+ })+ .unwrap_err();+ assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");+ }++ #[test]+ fn detect_text_names_the_missing_model_paths_before_reading_the_image() {let engine = OcrEngine::new(OcrModelPaths {detection: "missing/det.onnx".to_string(),- classification: String::new(),+ classification: "missing/cls.onnx".to_string(),recognition: String::new(),dictionary: String::new(),}).unwrap();let error = engine- .detect_text_regions(&DetectRegionsRequest {+ .detect_text(&DetectTextRequest {image_path: "missing/image.jpg".to_string(),- request_id: Some("r1".to_string()),+ include_all_confidence_scores: false,+ request_id: None,}).unwrap_err();- assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");+ let OcrError::Ml(MlError::InvalidRequest(message)) = error else {+ panic!("{error}");+ };+ assert_eq!(+ message,+ "OCR model paths are empty: recognition, dictionary"+ );}#[test]@@ -351,15 +627,7 @@ mod tests {let source = SourceImage {decoded_width: 2000,decoded_height: 1000,- working: crate::cv::image::ImageU8::zeros(1000, 500, 3).unwrap(),- };- let quad = |x: f32, y: f32| {- [- Point::new(x, y),- Point::new(x + 10.0, y),- Point::new(x + 10.0, y + 5.0),- Point::new(x, y + 5.0),- ]+ working: ImageU8::zeros(1000, 500, 3).unwrap(),};let candidates = vec![DetectionCandidate {@@ -379,4 +647,72 @@ mod tests {assert_eq!(regions[0].points[0], Point::new(200.0, 100.0));assert_eq!(regions[0].points[2], Point::new(220.0, 110.0));}++ #[test]+ fn text_blocks_filter_by_confidence_and_scale_blocks_and_characters() {+ let source = SourceImage {+ decoded_width: 2000,+ decoded_height: 1000,+ working: ImageU8::zeros(1000, 500, 3).unwrap(),+ };+ let candidates = vec![+ DetectionCandidate {+ points: quad(100.0, 50.0),+ score: 0.9,+ },+ DetectionCandidate {+ points: quad(300.0, 50.0),+ score: 0.9,+ },+ DetectionCandidate {+ points: quad(500.0, 50.0),+ score: 0.9,+ },+ ];+ let crops = vec![+ recognized("hi", 0.95, false),+ recognized("lo", 0.6, false),+ recognized("flip", 0.85, true),+ ];++ let blocks = text_blocks(&candidates, &crops, &source, MIN_BLOCK_CONFIDENCE);++ assert_eq!(blocks.len(), 2);+ assert_eq!(blocks[0].text, "hi");+ assert_eq!(blocks[0].points[0], Point::new(200.0, 100.0));+ assert_eq!(blocks[0].points[2], Point::new(220.0, 110.0));+ assert_eq!(blocks[0].characters.len(), 1);+ assert_eq!(blocks[0].characters[0].text, "hi");+ assert_eq!(blocks[0].characters[0].points[0], Point::new(200.0, 100.0));+ assert_eq!(blocks[0].characters[0].points[1], Point::new(210.0, 100.0));+ assert_eq!(blocks[1].text, "flip");+ assert_eq!(blocks[1].characters[0].points[0], Point::new(1010.0, 100.0));+ assert_eq!(blocks[1].characters[0].points[1], Point::new(1020.0, 100.0));++ let lenient = text_blocks(&candidates, &crops, &source, min_block_confidence(true));+ assert_eq!(lenient.len(), 3);+ }++ #[test]+ fn a_rotated_result_replaces_the_upright_one_only_when_more_confident() {+ let mut crop = recognized("up", 0.7, false);+ let same = recognized("same", 0.7, false);+ crop.prefer_rotated(same.image, same.recognition);+ assert!(!crop.rotated);+ assert_eq!(crop.recognition.text, "up");++ let better = recognized("down", 0.9, false);+ crop.prefer_rotated(better.image, better.recognition);+ assert!(crop.rotated);+ assert_eq!(crop.recognition.text, "down");+ }++ #[test]+ fn only_low_confidence_horizontal_crops_are_retried() {+ assert!(recognized("a", 0.64, false).needs_retry());+ assert!(!recognized("a", 0.65, false).needs_retry());+ let mut vertical = recognized("a", 0.1, false);+ vertical.orientation = Orientation::Vertical;+ assert!(!vertical.needs_retry());+ }}diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsnew file mode 100644index 0000000000..333d66b758--- /dev/null+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -0,0 +1,592 @@+use std::path::Path;+use std::sync::{Mutex, OnceLock, PoisonError};++use super::OcrError;+use super::cancel::RequestGuard;+use super::dictionary::load_dictionary;+use super::tensor::{BgrNormalization, write_bgr_planes};+use crate::cv;+use crate::cv::image::ImageU8;+use crate::error::{MlError, MlResult};+use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};++const MODEL_NAMESPACE: &str = "ocr-recognition";+const REC_VOCABULARY_SIZE: usize = 18385;+const REC_HEIGHT: i32 = 48;+const REC_BASE_WIDTH: i32 = 320;+const REC_BATCH_SIZE: usize = 6;+const BLANK_INDEX: usize = 0;+const MIN_SPAN: f32 = 1e-3;++#[derive(Clone, Debug, PartialEq)]+pub(crate) struct CharacterSpan {+ pub(crate) text: String,+ pub(crate) confidence: f32,+ pub(crate) start: f32,+ pub(crate) end: f32,+}++#[derive(Clone, Debug, Default, PartialEq)]+pub(crate) struct Recognition {+ pub(crate) text: String,+ pub(crate) confidence: f32,+ pub(crate) spans: Vec<CharacterSpan>,+}++struct LazyDictionary {+ path: String,+ entries: OnceLock<Vec<String>>,+}++impl LazyDictionary {+ fn new(path: &str) -> Self {+ Self {+ path: path.to_string(),+ entries: OnceLock::new(),+ }+ }++ fn entries(&self) -> MlResult<&[String]> {+ if let Some(entries) = self.entries.get() {+ return Ok(entries);+ }+ let loaded = load_dictionary(Path::new(&self.path), REC_VOCABULARY_SIZE)?;+ Ok(self.entries.get_or_init(|| loaded))+ }+}++pub(crate) struct TextRecognizer {+ session: Mutex<OnnxSession>,+ dictionary: LazyDictionary,+}++impl TextRecognizer {+ pub(crate) fn new(model_path: &str, dictionary_path: &str) -> Self {+ Self {+ session: Mutex::new(+ OnnxSession::new(model_path, MODEL_NAMESPACE, ExecutionMode::CpuAccelerated)+ .with_unvalidated_acceleration(),+ ),+ dictionary: LazyDictionary::new(dictionary_path),+ }+ }++ pub(crate) fn recognize(+ &self,+ crops: &[&ImageU8],+ request: &RequestGuard<'_>,+ ) -> Result<Vec<Recognition>, OcrError> {+ if crops.is_empty() {+ return Ok(Vec::new());+ }+ let dictionary = self.dictionary.entries()?;+ recognize_in_batches(crops, request, |batch| self.infer_batch(batch, dictionary))+ }++ fn infer_batch(&self, batch: &[&ImageU8], dictionary: &[String]) -> MlResult<Vec<Recognition>> {+ let layout = BatchLayout::new(batch);+ let input = PreparedF32Input::new(layout.tensor(batch)?);+ let count = batch.len();+ let input_shape = [+ count as i64,+ 3,+ i64::from(REC_HEIGHT),+ i64::from(layout.target_width),+ ];+ let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);+ let (output, _usage) = session.run(|session| {+ let (shape, values) = run_f32(session, &input, input_shape)?;+ SequenceOutput::new(&shape, values, count, dictionary.len())+ .map_err(SessionRunError::from)+ })?;+ Ok(layout+ .content_widths+ .iter()+ .zip(output.sequences())+ .map(|(&content_width, logits)| {+ ctc_decode(+ logits,+ output.vocabulary,+ dictionary,+ layout.padding_scale(content_width),+ )+ })+ .collect())+ }+}++fn recognize_in_batches(+ crops: &[&ImageU8],+ request: &RequestGuard<'_>,+ mut infer_batch: impl FnMut(&[&ImageU8]) -> MlResult<Vec<Recognition>>,+) -> Result<Vec<Recognition>, OcrError> {+ let mut results: Vec<Option<Recognition>> = vec![None; crops.len()];+ for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {+ request.check()?;+ let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();+ let recognized = infer_batch(&batch)?;+ if recognized.len() != batch.len() {+ return Err(MlError::CorruptModel(format!(+ "text recognizer decoded {} crops out of {}",+ recognized.len(),+ batch.len()+ ))+ .into());+ }+ for (&index, recognition) in indices.iter().zip(recognized) {+ results[index] = Some(recognition);+ }+ }+ results+ .into_iter()+ .collect::<Option<Vec<_>>>()+ .ok_or_else(|| MlError::CorruptModel("text recognizer skipped a crop".to_string()).into())+}++fn ascending_aspect_order(crops: &[&ImageU8]) -> Vec<usize> {+ let mut order: Vec<usize> = (0..crops.len()).collect();+ order.sort_by(|&a, &b| aspect(crops[a]).total_cmp(&aspect(crops[b])));+ order+}++fn aspect(crop: &ImageU8) -> f64 {+ f64::from(crop.width) / f64::from(crop.height)+}++struct BatchLayout {+ target_width: i32,+ content_widths: Vec<i32>,+}++impl BatchLayout {+ fn new(batch: &[&ImageU8]) -> Self {+ let base_ratio = f64::from(REC_BASE_WIDTH) / f64::from(REC_HEIGHT);+ let max_wh_ratio = batch+ .iter()+ .map(|crop| aspect(crop))+ .fold(base_ratio, f64::max);+ let target_width = target_width(max_wh_ratio);+ let content_widths = batch+ .iter()+ .map(|crop| content_width(crop, target_width))+ .collect();+ Self {+ target_width,+ content_widths,+ }+ }++ fn padding_scale(&self, content_width: i32) -> f32 {+ self.target_width as f32 / content_width as f32+ }++ fn tensor(&self, batch: &[&ImageU8]) -> MlResult<Vec<f32>> {+ let plane = (self.target_width * REC_HEIGHT) as usize;+ let mut tensor = vec![0.0f32; batch.len() * 3 * plane];+ let slots = tensor.chunks_exact_mut(3 * plane);+ for ((crop, &content_width), slot) in batch.iter().zip(&self.content_widths).zip(slots) {+ let resized = cv::resize_u8(crop, content_width, REC_HEIGHT, cv::Interp::Bilinear)+ .map_err(MlError::Preprocess)?;+ write_bgr_planes(+ &resized,+ slot,+ self.target_width as usize,+ BgrNormalization::CENTERED,+ )?;+ }+ Ok(tensor)+ }+}++fn target_width(max_wh_ratio: f64) -> i32 {+ ((f64::from(REC_HEIGHT) * max_wh_ratio).trunc() as i32).max(1)+}++fn content_width(crop: &ImageU8, target_width: i32) -> i32 {+ let scaled = (f64::from(REC_HEIGHT * crop.width) / f64::from(crop.height)).ceil() as i32;+ scaled.min(target_width)+}++#[derive(Debug)]+struct SequenceOutput {+ steps: usize,+ vocabulary: usize,+ values: Vec<f32>,+}++impl SequenceOutput {+ fn new(shape: &[i64], values: Vec<f32>, count: usize, vocabulary: usize) -> MlResult<Self> {+ match *shape {+ [n, steps, v]+ if n == count as i64+ && steps > 0+ && v == vocabulary as i64+ && values.len() == count * steps as usize * vocabulary =>+ {+ Ok(Self {+ steps: steps as usize,+ vocabulary,+ values,+ })+ }+ _ => Err(MlError::CorruptModel(format!(+ "text recognizer produced output shape {shape:?} with {} values, expected [{count}, T, {vocabulary}]",+ values.len()+ ))),+ }+ }++ fn sequences(&self) -> impl Iterator<Item = &[f32]> {+ self.values.chunks_exact(self.steps * self.vocabulary)+ }+}++fn ctc_decode(+ logits: &[f32],+ vocabulary: usize,+ dictionary: &[String],+ padding_scale: f32,+) -> Recognition {+ let best = best_per_step(logits, vocabulary);+ let spans: Vec<CharacterSpan> = character_runs(&best)+ .iter()+ .filter_map(|run| run.span(best.len(), dictionary, padding_scale))+ .collect();+ Recognition {+ text: spans.iter().map(|span| span.text.as_str()).collect(),+ confidence: mean_confidence(&spans),+ spans,+ }+}++#[derive(Clone, Copy)]+struct StepBest {+ index: usize,+ probability: f32,+}++fn best_per_step(logits: &[f32], vocabulary: usize) -> Vec<StepBest> {+ if vocabulary == 0 {+ return Vec::new();+ }+ logits.chunks_exact(vocabulary).map(argmax).collect()+}++fn argmax(step: &[f32]) -> StepBest {+ let mut best = StepBest {+ index: 0,+ probability: f32::NEG_INFINITY,+ };+ for (index, &probability) in step.iter().enumerate() {+ if probability > best.probability {+ best = StepBest { index, probability };+ }+ }+ best+}++struct CharacterRun {+ index: usize,+ start: usize,+ end: usize,+ confidence: f32,+}++fn character_runs(best: &[StepBest]) -> Vec<CharacterRun> {+ let mut runs = Vec::new();+ let mut start = 0;+ while start < best.len() {+ let index = best[start].index;+ let end = start+ + best[start..]+ .iter()+ .take_while(|b| b.index == index)+ .count();+ if index != BLANK_INDEX {+ let total: f32 = best[start..end].iter().map(|b| b.probability).sum();+ runs.push(CharacterRun {+ index,+ start,+ end,+ confidence: total / (end - start) as f32,+ });+ }+ start = end;+ }+ runs+}++impl CharacterRun {+ fn span(+ &self,+ steps: usize,+ dictionary: &[String],+ padding_scale: f32,+ ) -> Option<CharacterSpan> {+ let text = dictionary.get(self.index)?.clone();+ let (start, end) = span_bounds(self.start, self.end, steps, padding_scale);+ Some(CharacterSpan {+ text,+ confidence: self.confidence,+ start,+ end,+ })+ }+}++fn span_bounds(run_start: usize, run_end: usize, steps: usize, scale: f32) -> (f32, f32) {+ let steps = steps as f32;+ let min_span = (scale / steps).max(MIN_SPAN);+ let mut start = (run_start as f32 / steps * scale).clamp(0.0, 1.0);+ let mut end = (run_end as f32 / steps * scale).clamp(start, 1.0);+ if end - start < min_span {+ end = (start + min_span).min(1.0);+ if end - start < min_span {+ start = (end - min_span).max(0.0);+ }+ }+ (start, end)+}++fn mean_confidence(spans: &[CharacterSpan]) -> f32 {+ if spans.is_empty() {+ return 0.0;+ }+ spans.iter().map(|span| span.confidence).sum::<f32>() / spans.len() as f32+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::ocr::cancel::RequestRegistry;++ const TOLERANCE: f32 = 1e-6;++ fn dictionary() -> Vec<String> {+ ["blank", "a", "b", "c"].map(str::to_string).to_vec()+ }++ fn logits(vocabulary: usize, steps: &[(usize, f32)]) -> Vec<f32> {+ steps+ .iter()+ .flat_map(|&(index, probability)| {+ let mut row = vec![0.0f32; vocabulary];+ row[index] = probability;+ row+ })+ .collect()+ }++ fn solid(width: i32, height: i32, rgb: [u8; 3]) -> ImageU8 {+ let data = std::iter::repeat_n(rgb, (width * height) as usize)+ .flatten()+ .collect();+ ImageU8::new(width, height, 3, data).unwrap()+ }++ fn assert_close(actual: f32, expected: f32) {+ assert!(+ (actual - expected).abs() <= TOLERANCE,+ "{actual} != {expected}"+ );+ }++ fn assert_span(span: &CharacterSpan, text: &str, confidence: f32, start: f32, end: f32) {+ assert_eq!(span.text, text);+ assert_close(span.confidence, confidence);+ assert_close(span.start, start);+ assert_close(span.end, end);+ }++ #[test]+ fn blank_only_sequence_decodes_to_nothing() {+ let recognition = ctc_decode(+ &logits(4, &[(0, 0.9), (0, 0.8), (0, 0.7)]),+ 4,+ &dictionary(),+ 1.0,+ );+ assert_eq!(recognition, Recognition::default());+ }++ #[test]+ fn repeats_separated_by_a_blank_produce_two_characters() {+ let steps = [(1, 0.9), (1, 0.9), (0, 0.9), (1, 0.6)];+ let recognition = ctc_decode(&logits(4, &steps), 4, &dictionary(), 1.0);+ assert_eq!(recognition.text, "aa");+ assert_eq!(recognition.spans.len(), 2);+ assert_span(&recognition.spans[0], "a", 0.9, 0.0, 0.5);+ assert_span(&recognition.spans[1], "a", 0.6, 0.75, 1.0);+ assert_close(recognition.confidence, 0.75);+ }++ #[test]+ fn adjacent_repeats_collapse_into_one_run_with_the_mean_probability() {+ let steps = [(1, 0.9), (1, 0.7), (2, 0.5), (0, 0.9), (3, 0.3), (3, 0.5)];+ let recognition = ctc_decode(&logits(4, &steps), 4, &dictionary(), 1.0);+ assert_eq!(recognition.text, "abc");+ assert_span(&recognition.spans[0], "a", 0.8, 0.0, 2.0 / 6.0);+ assert_span(&recognition.spans[1], "b", 0.5, 2.0 / 6.0, 3.0 / 6.0);+ assert_span(&recognition.spans[2], "c", 0.4, 4.0 / 6.0, 1.0);+ assert_close(recognition.confidence, (0.8 + 0.5 + 0.4) / 3.0);+ }++ #[test]+ fn argmax_picks_the_first_of_equal_maxima() {+ let best = argmax(&[0.2, 0.5, 0.5, 0.1]);+ assert_eq!(best.index, 1);+ assert_close(best.probability, 0.5);+ }++ #[test]+ fn padded_spans_are_stretched_then_clamped_with_the_minimum_span() {+ let steps = [(1, 1.0), (0, 1.0), (0, 1.0), (2, 1.0)];+ let recognition = ctc_decode(&logits(4, &steps), 4, &dictionary(), 2.0);+ assert_eq!(recognition.text, "ab");+ assert_span(&recognition.spans[0], "a", 1.0, 0.0, 0.5);+ assert_span(&recognition.spans[1], "b", 1.0, 0.5, 1.0);+ }++ #[test]+ fn minimum_span_never_drops_below_one_thousandth() {+ let mut steps = vec![(0usize, 1.0f32); 2000];+ steps[0] = (1, 1.0);+ let recognition = ctc_decode(+ &logits(2, &steps),+ 2,+ &["blank".to_string(), "a".to_string()],+ 1.0,+ );+ assert_span(&recognition.spans[0], "a", 1.0, 0.0, 1e-3);+ }++ #[test]+ fn indices_outside_the_dictionary_are_skipped() {+ let recognition = ctc_decode(&logits(5, &[(4, 0.9), (1, 0.8)]), 5, &dictionary(), 1.0);+ assert_eq!(recognition.text, "a");+ assert_eq!(recognition.spans.len(), 1);+ assert_close(recognition.confidence, 0.8);+ }++ #[test]+ fn target_width_truncates_and_content_width_ceils() {+ assert_eq!(+ target_width(f64::from(REC_BASE_WIDTH) / f64::from(REC_HEIGHT)),+ 320+ );+ assert_eq!(target_width(7.3), 350);+ assert_eq!(target_width(0.001), 1);+ assert_eq!(content_width(&solid(7, 3, [0; 3]), 400), 112);+ assert_eq!(content_width(&solid(1, 4096, [0; 3]), 320), 1);+ assert_eq!(content_width(&solid(1000, 48, [0; 3]), 320), 320);+ }++ #[test]+ fn batch_layout_uses_the_widest_aspect_but_never_less_than_the_base_width() {+ let crops = [+ solid(100, 48, [0; 3]),+ solid(400, 48, [0; 3]),+ solid(7, 3, [0; 3]),+ ];+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());+ assert_eq!(layout.target_width, 400);+ assert_eq!(layout.content_widths, [100, 400, 112]);+ assert_close(layout.padding_scale(100), 4.0);++ let narrow = [solid(10, 48, [0; 3])];+ let layout = BatchLayout::new(&narrow.iter().collect::<Vec<_>>());+ assert_eq!(layout.target_width, 320);+ assert_eq!(layout.content_widths, [10]);+ }++ #[test]+ fn batch_tensor_left_aligns_each_crop_and_zero_pads_the_rest() {+ let crops = [solid(100, 48, [10, 20, 30]), solid(400, 48, [0; 3])];+ let layout = BatchLayout::new(&crops.iter().collect::<Vec<_>>());+ let tensor = layout.tensor(&crops.iter().collect::<Vec<_>>()).unwrap();+ let plane = 48 * 400;+ assert_eq!(tensor.len(), 2 * 3 * plane);+ let centered = |value: f32| (value / 255.0 - 0.5) / 0.5;+ for (channel, expected) in [centered(30.0), centered(20.0), centered(10.0)]+ .into_iter()+ .enumerate()+ {+ let row = &tensor[channel * plane + 47 * 400..channel * plane + 48 * 400];+ assert!(+ row[..100]+ .iter()+ .all(|&v| (v - expected).abs() <= TOLERANCE),+ "channel {channel}"+ );+ assert!(row[100..].iter().all(|&v| v == 0.0), "channel {channel}");+ }+ }++ #[test]+ fn sequence_output_requires_the_dictionary_vocabulary() {+ let ok = SequenceOutput::new(&[1, 2, 4], vec![0.0; 8], 1, 4).unwrap();+ assert_eq!((ok.steps, ok.vocabulary), (2, 4));+ assert_eq!(ok.sequences().count(), 1);+ for shape in [[1, 2, 5], [2, 2, 4], [1, 0, 4]] {+ let error = SequenceOutput::new(&shape, vec![0.0; 8], 1, 4).unwrap_err();+ assert!(matches!(error, MlError::CorruptModel(_)), "{error}");+ }+ let short = SequenceOutput::new(&[1, 2, 4], vec![0.0; 7], 1, 4).unwrap_err();+ assert!(matches!(short, MlError::CorruptModel(_)), "{short}");+ }++ #[test]+ fn crops_are_batched_by_ascending_aspect_and_restored_to_input_order() {+ let widths = [80, 10, 60, 30, 70, 20, 50, 40];+ let crops: Vec<ImageU8> = widths.iter().map(|&w| solid(w, 10, [0; 3])).collect();+ let registry = RequestRegistry::default();+ let request = registry.begin(None);+ let mut batches = Vec::new();++ let results = recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |batch| {+ batches.push(batch.iter().map(|crop| crop.width).collect::<Vec<_>>());+ Ok(batch+ .iter()+ .map(|crop| Recognition {+ text: crop.width.to_string(),+ ..Recognition::default()+ })+ .collect())+ })+ .unwrap();++ assert_eq!(batches, [vec![10, 20, 30, 40, 50, 60], vec![70, 80]]);+ let texts: Vec<String> = results.into_iter().map(|r| r.text).collect();+ let expected: Vec<String> = widths.iter().map(|w| w.to_string()).collect();+ assert_eq!(texts, expected);+ }++ #[test]+ fn a_cancelled_request_stops_before_the_first_batch() {+ let crop = solid(4, 4, [0; 3]);+ let registry = RequestRegistry::default();+ let request = registry.begin(Some("cancelled"));+ registry.cancel("cancelled");++ let error = recognize_in_batches(&[&crop], &request, |_| unreachable!()).unwrap_err();++ assert!(matches!(error, OcrError::Cancelled), "{error}");+ }++ #[test]+ fn a_short_result_list_is_reported_as_a_corrupt_model() {+ let crops = [solid(4, 4, [0; 3]), solid(4, 4, [0; 3])];+ let registry = RequestRegistry::default();+ let request = registry.begin(None);++ let error = recognize_in_batches(&crops.iter().collect::<Vec<_>>(), &request, |_| {+ Ok(vec![Recognition::default()])+ })+ .unwrap_err();++ assert!(+ matches!(error, OcrError::Ml(MlError::CorruptModel(_))),+ "{error}"+ );+ }+}diff --git a/rust/crates/ml/src/ocr/source.rs b/rust/crates/ml/src/ocr/source.rsindex 240ca72bd0..9bb04cea40 100644--- a/rust/crates/ml/src/ocr/source.rs+++ b/rust/crates/ml/src/ocr/source.rs@@ -13,6 +13,11 @@ pub(crate) struct WorkingSizeCap {max_pixels: u32,}+pub(crate) const FULL_TEXT_CAP: WorkingSizeCap = WorkingSizeCap {+ max_side: 4096,+ max_pixels: 12_000_000,+};+pub(crate) const REGIONS_CAP: WorkingSizeCap = WorkingSizeCap {max_side: 1280,max_pixels: 2_000_000,@@ -87,11 +92,6 @@ mod tests {use super::*;- const FULL_TEXT_CAP: WorkingSizeCap = WorkingSizeCap {- max_side: 4096,- max_pixels: 12_000_000,- };-fn write_png(dir: &Path, name: &str, width: u32, height: u32) -> String {let path = dir.join(name);let writer = BufWriter::new(File::create(&path).unwrap());