Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

24.2 KB
Raw Blame
pub mod assets;
mod characters;
mod classify;
mod context;
mod crop;
mod detect;
mod dictionary;
mod geometry;
mod recognize;
mod session;
mod source;
mod tensor;

pub use crop::Orientation;
pub use detect::ProbabilityMap;
pub use geometry::Point;

use std::time::Instant;

use thiserror::Error;

use characters::character_boxes;
use classify::AngleClassifier;
use crop::{TextCrop, crop_text};
use detect::{DetectionCandidate, TextDetector};
use geometry::scale_points;
use recognize::{Recognition, TextRecognizer};
use source::{FULL_TEXT_CAP, REGIONS_CAP, SourceImage, load_source};

use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};

const REGION_MIN_SCORE: f32 = 0.5;
const MAX_REGIONS: usize = 1000;
const MIN_BLOCK_CONFIDENCE: f32 = 0.8;
const MIN_BLOCK_CONFIDENCE_WITH_ALL_SCORES: f32 = 0.5;
const RETRY_BELOW_CONFIDENCE: f32 = 0.65;

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OcrModelPaths {
    pub detection: String,
    pub classification: String,
    pub recognition: String,
    pub dictionary: String,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DetectTextRequest {
    pub image_path: String,
    pub include_all_confidence_scores: bool,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DetectRegionsRequest {
    pub image_path: String,
}

#[derive(Clone, Debug, PartialEq)]
pub struct TextBlock {
    pub text: String,
    pub confidence: f32,
    pub points: [Point; 4],
    pub characters: Vec<CharacterBox>,
}

#[derive(Clone, Debug, PartialEq)]
pub struct CharacterBox {
    pub text: String,
    pub confidence: f32,
    pub points: [Point; 4],
}

#[derive(Clone, Debug, PartialEq)]
pub struct TextDetectionResult {
    pub blocks: Vec<TextBlock>,
    pub image_width: u32,
    pub image_height: u32,
}

#[derive(Clone, Debug, PartialEq)]
pub struct TextRegion {
    pub confidence: f32,
    pub points: [Point; 4],
}

#[derive(Clone, Debug, PartialEq)]
pub struct TextRegionDetectionResult {
    pub regions: Vec<TextRegion>,
    pub image_width: u32,
    pub image_height: u32,
}

#[derive(Clone, Debug, PartialEq)]
pub struct RegionDetectionDebug {
    pub result: TextRegionDetectionResult,
    pub working_width: u32,
    pub working_height: u32,
    pub probability_map: ProbabilityMap,
}

#[derive(Clone, Debug, PartialEq)]
pub struct CropDebug {
    pub width: u32,
    pub height: u32,
    pub rgb: Vec<u8>,
    pub orientation: Orientation,
    pub rotated: bool,
    pub text: String,
    pub confidence: f32,
}

#[derive(Clone, Debug, PartialEq)]
pub struct TextDetectionDebug {
    pub result: TextDetectionResult,
    pub working_width: u32,
    pub working_height: u32,
    pub probability_map: ProbabilityMap,
    pub crops: Vec<CropDebug>,
}

#[derive(Debug, Error)]
pub enum OcrError {
    #[error("image not found: {0}")]
    ImageNotFound(String),
    #[error(transparent)]
    Ml(#[from] MlError),
}

pub struct OcrEngine {
    detector: TextDetector,
    classifier: Option<AngleClassifier>,
    recognizer: Option<TextRecognizer>,
    missing_text_model_paths: Vec<&'static str>,
}

impl OcrEngine {
    pub fn new(paths: OcrModelPaths) -> MlResult<Self> {
        if paths.detection.is_empty() {
            return Err(MlError::InvalidRequest(
                "OCR detection model path is empty".to_string(),
            ));
        }
        let classifier =
            (!paths.classification.is_empty()).then(|| AngleClassifier::new(&paths.classification));
        let recognizer = (!paths.recognition.is_empty())
            .then(|| TextRecognizer::new(&paths.recognition, &paths.dictionary));
        Ok(Self {
            detector: TextDetector::new(&paths.detection),
            classifier,
            recognizer,
            missing_text_model_paths: missing_text_model_paths(&paths),
        })
    }

    pub fn load_models(&self) -> MlResult<()> {
        self.detector.load()?;
        if let Some(classifier) = &self.classifier {
            classifier.load()?;
        }
        if let Some(recognizer) = &self.recognizer {
            recognizer.load()?;
        }
        Ok(())
    }

    pub fn unload_models(&self) {
        self.detector.unload();
        if let Some(classifier) = &self.classifier {
            classifier.unload();
        }
        if let Some(recognizer) = &self.recognizer {
            recognizer.unload();
        }
    }

    fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {
        match (&self.classifier, &self.recognizer) {
            (Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {
                Ok((classifier, recognizer))
            }
            _ => Err(MlError::InvalidRequest(format!(
                "OCR model paths are empty: {}",
                self.missing_text_model_paths.join(", ")
            ))),
        }
    }

    pub fn detect_text(&self, req: &DetectTextRequest) -> Result<TextDetectionResult, OcrError> {
        self.run_text_pipeline(req).map(TextPipeline::into_result)
    }

    pub fn detect_text_debug(
        &self,
        req: &DetectTextRequest,
    ) -> Result<TextDetectionDebug, OcrError> {
        self.run_text_pipeline(req).map(TextPipeline::into_debug)
    }

    fn run_text_pipeline(&self, req: &DetectTextRequest) -> Result<TextPipeline, OcrError> {
        let (classifier, recognizer) = self.text_models()?;
        let started = Instant::now();
        let mut source = load_source(&req.image_path, FULL_TEXT_CAP)?;
        let detection_started = Instant::now();
        let detection = self.detector.detect(&source.working)?;
        let detection_ms = detection_started.elapsed().as_millis();
        let crops = crop_candidates(&mut source.working, &detection.candidates)?;
        let recognition_started = Instant::now();
        let mut recognized = recognize_crops(recognizer, crops)?;
        let retries = retry_low_confidence(classifier, recognizer, &mut recognized)?;
        let recognition_ms = recognition_started.elapsed().as_millis();
        let blocks = text_blocks(
            &detection.candidates,
            &recognized,
            &source,
            min_block_confidence(req.include_all_confidence_scores),
        );
        log::info!(
            "ocr detect_text: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, rec {} crops in {recognition_ms}ms, cls retries {retries}, kept {}, total {}ms",
            source.decoded_width,
            source.decoded_height,
            source.working.width,
            source.working.height,
            detection.candidates.len(),
            recognized.len(),
            blocks.len(),
            started.elapsed().as_millis()
        );
        Ok(TextPipeline {
            image_width: source.decoded_width,
            image_height: source.decoded_height,
            working_width: source.working.width as u32,
            working_height: source.working.height as u32,
            probability_map: detection.probability_map,
            crops: recognized,
            blocks,
        })
    }

    pub fn detect_text_regions(
        &self,
        req: &DetectRegionsRequest,
    ) -> Result<TextRegionDetectionResult, OcrError> {
        self.detect_text_regions_debug(req)
            .map(|debug| debug.result)
    }

    pub fn detect_text_regions_debug(
        &self,
        req: &DetectRegionsRequest,
    ) -> Result<RegionDetectionDebug, OcrError> {
        let started = Instant::now();
        let source = load_source(&req.image_path, REGIONS_CAP)?;
        let detection_started = Instant::now();
        let detection = self.detector.detect(&source.working)?;
        let detection_ms = detection_started.elapsed().as_millis();
        let regions = regions_in_decoded_pixels(&detection.candidates, &source);
        log::info!(
            "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",
            source.decoded_width,
            source.decoded_height,
            source.working.width,
            source.working.height,
            detection.candidates.len(),
            regions.len(),
            started.elapsed().as_millis()
        );
        Ok(RegionDetectionDebug {
            result: TextRegionDetectionResult {
                regions,
                image_width: source.decoded_width,
                image_height: source.decoded_height,
            },
            working_width: source.working.width as u32,
            working_height: source.working.height as u32,
            probability_map: detection.probability_map,
        })
    }
}

fn missing_text_model_paths(paths: &OcrModelPaths) -> Vec<&'static str> {
    [
        ("classification", &paths.classification),
        ("recognition", &paths.recognition),
        ("dictionary", &paths.dictionary),
    ]
    .into_iter()
    .filter(|(_, path)| path.is_empty())
    .map(|(name, _)| name)
    .collect()
}

fn min_block_confidence(include_all_confidence_scores: bool) -> f32 {
    if include_all_confidence_scores {
        MIN_BLOCK_CONFIDENCE_WITH_ALL_SCORES
    } else {
        MIN_BLOCK_CONFIDENCE
    }
}

struct TextPipeline {
    image_width: u32,
    image_height: u32,
    working_width: u32,
    working_height: u32,
    probability_map: ProbabilityMap,
    crops: Vec<RecognizedCrop>,
    blocks: Vec<TextBlock>,
}

impl TextPipeline {
    fn into_result(self) -> TextDetectionResult {
        TextDetectionResult {
            blocks: self.blocks,
            image_width: self.image_width,
            image_height: self.image_height,
        }
    }

    fn into_debug(self) -> TextDetectionDebug {
        let crops = self
            .crops
            .into_iter()
            .map(RecognizedCrop::into_debug)
            .collect();
        TextDetectionDebug {
            result: TextDetectionResult {
                blocks: self.blocks,
                image_width: self.image_width,
                image_height: self.image_height,
            },
            working_width: self.working_width,
            working_height: self.working_height,
            probability_map: self.probability_map,
            crops,
        }
    }
}

struct RecognizedCrop {
    image: ImageU8,
    orientation: Orientation,
    rotated: bool,
    recognition: Recognition,
}

impl RecognizedCrop {
    fn upright(crop: TextCrop, recognition: Recognition) -> Self {
        Self {
            image: crop.image,
            orientation: crop.orientation,
            rotated: false,
            recognition,
        }
    }

    fn prefer_rotated(&mut self, image: ImageU8, recognition: Recognition) {
        if recognition.confidence > self.recognition.confidence {
            self.image = image;
            self.rotated = true;
            self.recognition = recognition;
        }
    }

    fn needs_retry(&self) -> bool {
        self.orientation == Orientation::Horizontal
            && self.recognition.confidence < RETRY_BELOW_CONFIDENCE
    }

    fn into_debug(self) -> CropDebug {
        CropDebug {
            width: self.image.width as u32,
            height: self.image.height as u32,
            rgb: self.image.data,
            orientation: self.orientation,
            rotated: self.rotated,
            text: self.recognition.text,
            confidence: self.recognition.confidence,
        }
    }
}

fn crop_candidates(
    working: &mut ImageU8,
    candidates: &[DetectionCandidate],
) -> MlResult<Vec<TextCrop>> {
    if candidates.is_empty() {
        return Ok(Vec::new());
    }
    let expected_len = (working.width as usize)
        .checked_mul(working.height as usize)
        .and_then(|pixels| pixels.checked_mul(3));
    if working.width <= 0
        || working.height <= 0
        || working.channels != 3
        || expected_len != Some(working.data.len())
    {
        return Err(MlError::Preprocess(
            "OCR crop source buffer mismatch".to_string(),
        ));
    }
    #[expect(
        clippy::expect_used,
        reason = "RGB dimensions and exact buffer length are checked before conversion"
    )]
    let source = image::RgbImage::from_raw(
        working.width as u32,
        working.height as u32,
        std::mem::take(&mut working.data),
    )
    .expect("validated RGB source dimensions and length");
    let prepare = || {
        candidates
            .iter()
            .map(|candidate| crop_text(&source, &candidate.points))
            .collect()
    };
    let crops = if cfg!(target_os = "android") {
        tensor::prepare_crops(prepare)
    } else {
        prepare()
    };
    working.data = source.into_raw();
    crops
}

fn recognize_crops(
    recognizer: &TextRecognizer,
    crops: Vec<TextCrop>,
) -> MlResult<Vec<RecognizedCrop>> {
    let flipped = crops
        .iter()
        .map(flipped_if_vertical)
        .collect::<MlResult<Vec<_>>>()?;
    let inputs: Vec<&ImageU8> = crops
        .iter()
        .zip(&flipped)
        .flat_map(|(crop, flipped)| std::iter::once(&crop.image).chain(flipped))
        .collect();
    let mut results = recognizer.recognize(&inputs)?.into_iter();
    crops
        .into_iter()
        .zip(flipped)
        .map(|(crop, flipped)| {
            let mut recognized = RecognizedCrop::upright(crop, next_recognition(&mut results)?);
            if let Some(image) = flipped {
                recognized.prefer_rotated(image, next_recognition(&mut results)?);
            }
            Ok(recognized)
        })
        .collect()
}

fn flipped_if_vertical(crop: &TextCrop) -> MlResult<Option<ImageU8>> {
    match crop.orientation {
        Orientation::Vertical => cv::rotate_u8(&crop.image, 180)
            .map(Some)
            .map_err(MlError::Preprocess),
        Orientation::Horizontal => Ok(None),
    }
}

fn next_recognition(results: &mut impl Iterator<Item = Recognition>) -> MlResult<Recognition> {
    results.next().ok_or_else(|| {
        MlError::CorruptModel("text recognizer returned fewer results than crops".to_string())
    })
}

fn retry_low_confidence(
    classifier: &AngleClassifier,
    recognizer: &TextRecognizer,
    crops: &mut [RecognizedCrop],
) -> MlResult<usize> {
    let indices: Vec<usize> = crops
        .iter()
        .enumerate()
        .filter(|(_, crop)| crop.needs_retry())
        .map(|(index, _)| index)
        .collect();
    if indices.is_empty() {
        return Ok(0);
    }
    let mut images: Vec<ImageU8> = indices
        .iter()
        .map(|&index| crops[index].image.clone())
        .collect();
    let decisions = classifier.classify(&mut images)?;
    let flipped: Vec<(usize, ImageU8)> = indices
        .iter()
        .copied()
        .zip(images)
        .zip(decisions)
        .filter(|(_, decision)| decision.rotated)
        .map(|(flipped, _)| flipped)
        .collect();
    let inputs: Vec<&ImageU8> = flipped.iter().map(|(_, image)| image).collect();
    let results = recognizer.recognize(&inputs)?;
    for ((index, image), retry) in flipped.into_iter().zip(results) {
        crops[index].prefer_rotated(image, retry);
    }
    Ok(indices.len())
}

fn text_blocks(
    candidates: &[DetectionCandidate],
    crops: &[RecognizedCrop],
    source: &SourceImage,
    min_confidence: f32,
) -> Vec<TextBlock> {
    let (scale_x, scale_y) = decoded_scale(source);
    candidates
        .iter()
        .zip(crops)
        .filter(|(_, crop)| crop.recognition.confidence >= min_confidence)
        .map(|(candidate, crop)| TextBlock {
            text: crop.recognition.text.clone(),
            confidence: crop.recognition.confidence,
            points: scale_points(&candidate.points, scale_x, scale_y),
            characters: character_boxes(
                &candidate.points,
                &crop.recognition.spans,
                crop.orientation,
                crop.rotated,
            )
            .into_iter()
            .map(|character| CharacterBox {
                points: scale_points(&character.points, scale_x, scale_y),
                ..character
            })
            .collect(),
        })
        .collect()
}

fn regions_in_decoded_pixels(
    candidates: &[DetectionCandidate],
    source: &SourceImage,
) -> Vec<TextRegion> {
    let (scale_x, scale_y) = decoded_scale(source);
    candidates
        .iter()
        .filter(|candidate| candidate.score >= REGION_MIN_SCORE)
        .take(MAX_REGIONS)
        .map(|candidate| TextRegion {
            confidence: candidate.score,
            points: scale_points(&candidate.points, scale_x, scale_y),
        })
        .collect()
}

fn decoded_scale(source: &SourceImage) -> (f32, f32) {
    (
        source.decoded_width as f32 / source.working.width as f32,
        source.decoded_height as f32 / source.working.height as f32,
    )
}

#[cfg(test)]
mod tests {
    use super::*;
    use recognize::CharacterSpan;

    fn detector_only_engine() -> OcrEngine {
        OcrEngine::new(OcrModelPaths {
            detection: "missing/det.onnx".to_string(),
            classification: String::new(),
            recognition: String::new(),
            dictionary: String::new(),
        })
        .unwrap()
    }

    fn quad(x: f32, y: f32) -> [Point; 4] {
        [
            Point::new(x, y),
            Point::new(x + 10.0, y),
            Point::new(x + 10.0, y + 5.0),
            Point::new(x, y + 5.0),
        ]
    }

    fn recognized(text: &str, confidence: f32, rotated: bool) -> RecognizedCrop {
        RecognizedCrop {
            image: ImageU8::zeros(2, 1, 3).unwrap(),
            orientation: Orientation::Horizontal,
            rotated,
            recognition: Recognition {
                text: text.to_string(),
                confidence,
                spans: vec![CharacterSpan {
                    text: text.to_string(),
                    confidence,
                    start: 0.0,
                    end: 0.5,
                }],
            },
        }
    }

    #[test]
    fn no_detected_regions_produce_no_crops() {
        let mut image = ImageU8::zeros(32, 32, 3).unwrap();
        assert!(crop_candidates(&mut image, &[]).unwrap().is_empty());
    }

    #[test]
    fn cropping_restores_the_source_buffer_on_success_and_error() {
        let data = (0..32 * 32 * 3).map(|i| (i * 37 % 256) as u8).collect();
        let original = ImageU8::new(32, 32, 3, data).unwrap();
        let mut image = original.clone();
        let pointer = image.data.as_ptr();
        let candidate = DetectionCandidate {
            points: quad(3.0, 4.0),
            score: 0.9,
        };
        let rgb = image::RgbImage::from_raw(32, 32, original.data.clone()).unwrap();
        let expected = crop_text(&rgb, &candidate.points).unwrap();
        let crops = crop_candidates(&mut image, std::slice::from_ref(&candidate)).unwrap();
        assert_eq!(crops[0].image, expected.image);
        assert_eq!(image, original);
        assert_eq!(image.data.as_ptr(), pointer);

        let invalid = DetectionCandidate {
            points: [Point::new(0.0, 0.0); 4],
            score: 0.9,
        };
        assert!(crop_candidates(&mut image, &[candidate, invalid]).is_err());
        assert_eq!(image, original);
        assert_eq!(image.data.as_ptr(), pointer);
    }

    #[test]
    fn cropping_rejects_invalid_source_geometry_without_taking_the_buffer() {
        let candidate = DetectionCandidate {
            points: quad(3.0, 4.0),
            score: 0.9,
        };
        for (width, height, channels) in [(33, 32, 3), (-1, 32, 3), (32, 32, 4)] {
            let mut image = ImageU8::zeros(32, 32, 3).unwrap();
            image.width = width;
            image.height = height;
            image.channels = channels;
            let original = image.clone();
            assert!(crop_candidates(&mut image, std::slice::from_ref(&candidate)).is_err());
            assert_eq!(image, original);
        }
    }

    #[test]
    fn engine_accepts_a_detector_only_path_set_without_loading_it() {
        let engine = detector_only_engine();
        assert_eq!(
            engine.missing_text_model_paths,
            ["classification", "recognition", "dictionary"]
        );
    }

    #[test]
    fn missing_image_is_reported_before_the_detector_is_touched() {
        let error = detector_only_engine()
            .detect_text_regions(&DetectRegionsRequest {
                image_path: "missing/image.jpg".to_string(),
            })
            .unwrap_err();
        assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");
    }

    #[test]
    fn detect_text_names_the_missing_model_paths_before_reading_the_image() {
        let engine = OcrEngine::new(OcrModelPaths {
            detection: "missing/det.onnx".to_string(),
            classification: "missing/cls.onnx".to_string(),
            recognition: String::new(),
            dictionary: String::new(),
        })
        .unwrap();
        let error = engine
            .detect_text(&DetectTextRequest {
                image_path: "missing/image.jpg".to_string(),
                include_all_confidence_scores: false,
            })
            .unwrap_err();
        let OcrError::Ml(MlError::InvalidRequest(message)) = error else {
            panic!("{error}");
        };
        assert_eq!(
            message,
            "OCR model paths are empty: recognition, dictionary"
        );
    }

    #[test]
    fn regions_scale_from_working_to_decoded_pixels_and_drop_low_scores() {
        let source = SourceImage {
            decoded_width: 2000,
            decoded_height: 1000,
            working: ImageU8::zeros(1000, 500, 3).unwrap(),
        };
        let candidates = vec![
            DetectionCandidate {
                points: quad(100.0, 50.0),
                score: 0.9,
            },
            DetectionCandidate {
                points: quad(300.0, 50.0),
                score: 0.4,
            },
        ];

        let regions = regions_in_decoded_pixels(&candidates, &source);

        assert_eq!(regions.len(), 1);
        assert_eq!(regions[0].confidence, 0.9);
        assert_eq!(regions[0].points[0], Point::new(200.0, 100.0));
        assert_eq!(regions[0].points[2], Point::new(220.0, 110.0));
    }

    #[test]
    fn text_blocks_filter_by_confidence_and_scale_blocks_and_characters() {
        let source = SourceImage {
            decoded_width: 2000,
            decoded_height: 1000,
            working: ImageU8::zeros(1000, 500, 3).unwrap(),
        };
        let candidates = vec![
            DetectionCandidate {
                points: quad(100.0, 50.0),
                score: 0.9,
            },
            DetectionCandidate {
                points: quad(300.0, 50.0),
                score: 0.9,
            },
            DetectionCandidate {
                points: quad(500.0, 50.0),
                score: 0.9,
            },
        ];
        let crops = vec![
            recognized("hi", 0.95, false),
            recognized("lo", 0.6, false),
            recognized("flip", 0.85, true),
        ];

        let blocks = text_blocks(&candidates, &crops, &source, MIN_BLOCK_CONFIDENCE);

        assert_eq!(blocks.len(), 2);
        assert_eq!(blocks[0].text, "hi");
        assert_eq!(blocks[0].points[0], Point::new(200.0, 100.0));
        assert_eq!(blocks[0].points[2], Point::new(220.0, 110.0));
        assert_eq!(blocks[0].characters.len(), 1);
        assert_eq!(blocks[0].characters[0].text, "hi");
        assert_eq!(blocks[0].characters[0].points[0], Point::new(200.0, 100.0));
        assert_eq!(blocks[0].characters[0].points[1], Point::new(210.0, 100.0));
        assert_eq!(blocks[1].text, "flip");
        assert_eq!(blocks[1].characters[0].points[0], Point::new(1010.0, 100.0));
        assert_eq!(blocks[1].characters[0].points[1], Point::new(1020.0, 100.0));

        let lenient = text_blocks(&candidates, &crops, &source, min_block_confidence(true));
        assert_eq!(lenient.len(), 3);
    }

    #[test]
    fn a_rotated_result_replaces_the_upright_one_only_when_more_confident() {
        let mut crop = recognized("up", 0.7, false);
        let same = recognized("same", 0.7, false);
        crop.prefer_rotated(same.image, same.recognition);
        assert!(!crop.rotated);
        assert_eq!(crop.recognition.text, "up");

        let better = recognized("down", 0.9, false);
        crop.prefer_rotated(better.image, better.recognition);
        assert!(crop.rotated);
        assert_eq!(crop.recognition.text, "down");
    }

    #[test]
    fn only_low_confidence_horizontal_crops_are_retried() {
        assert!(recognized("a", 0.64, false).needs_retry());
        assert!(!recognized("a", 0.65, false).needs_retry());
        let mut vertical = recognized("a", 0.1, false);
        vertical.orientation = Orientation::Vertical;
        assert!(!vertical.needs_retry());
    }
}