Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Add OCR text region detection

laurenspriemb203329093d7
diff --git a/rust/crates/ml/examples/ocr.rs b/rust/crates/ml/examples/ocr.rs
new file mode 100644
index 0000000000..f368d39a35
--- /dev/null
+++ b/rust/crates/ml/examples/ocr.rs
@@ -0,0 +1,179 @@
+#[allow(dead_code)]
+#[path = "../tests/support/mod.rs"]
+mod support;
+
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use anyhow::{Context, Result, bail};
+use ente_assets::AssetStore;
+use ente_image::decode::decode_image_from_path;
+use ente_ml::ocr::{
+ DetectRegionsRequest, OcrEngine, OcrModelPaths, Point, ProbabilityMap, RegionDetectionDebug,
+ TextRegion, assets,
+};
+use image::{GrayImage, Rgb, RgbImage};
+use imageproc::drawing::draw_line_segment_mut;
+use support::ml_indexing::{asset_cache_dir, load_onnx_runtime, run_with_large_stack};
+
+const OVERLAY_COLOR: Rgb<u8> = Rgb([255, 0, 0]);
+
+struct Options {
+ dump_dir: Option<PathBuf>,
+ images: Vec<String>,
+}
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> Result<()> {
+ let options = parse_args(std::env::args().skip(1))?;
+ load_onnx_runtime().await?;
+ let store = AssetStore::new(asset_cache_dir()?);
+ let paths = assets::ensure_models(&store).await?;
+ run_with_large_stack("ocr", move || run(options, paths))
+}
+
+fn usage() -> &'static str {
+ "usage: cargo run -p ente-ml --example ocr -- [--regions] [--all-confidences] \
+ [--dump-dir DIR] <image>..."
+}
+
+fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Options> {
+ let mut options = Options {
+ dump_dir: None,
+ images: Vec::new(),
+ };
+ while let Some(arg) = args.next() {
+ match arg.as_str() {
+ "--regions" | "--all-confidences" => {}
+ "--dump-dir" => {
+ let dir = args.next().context("--dump-dir needs a directory")?;
+ options.dump_dir = Some(PathBuf::from(dir));
+ }
+ other if other.starts_with("--") => bail!("unknown option {other}\n{}", usage()),
+ image => options.images.push(image.to_string()),
+ }
+ }
+ if options.images.is_empty() {
+ bail!(usage());
+ }
+ Ok(options)
+}
+
+fn run(options: Options, paths: OcrModelPaths) -> Result<()> {
+ let engine = OcrEngine::new(paths)?;
+ let mut failures = 0usize;
+ for image_path in &options.images {
+ match detect_regions(&engine, image_path, options.dump_dir.as_deref()) {
+ Ok(()) => {}
+ Err(error) => {
+ failures += 1;
+ eprintln!("{image_path}: {error:#}");
+ }
+ }
+ }
+ if failures > 0 {
+ bail!("{failures} image(s) failed");
+ }
+ Ok(())
+}
+
+fn detect_regions(engine: &OcrEngine, image_path: &str, dump_dir: Option<&Path>) -> Result<()> {
+ let started = Instant::now();
+ let debug = engine.detect_text_regions_debug(&DetectRegionsRequest {
+ image_path: image_path.to_string(),
+ request_id: None,
+ })?;
+ let elapsed_ms = started.elapsed().as_millis();
+ let result = &debug.result;
+ println!(
+ "{image_path}: decoded {}x{}, working {}x{}, probmap {}x{}, {} regions in {elapsed_ms}ms",
+ result.image_width,
+ result.image_height,
+ debug.working_width,
+ debug.working_height,
+ debug.probability_map.width,
+ debug.probability_map.height,
+ result.regions.len()
+ );
+ for region in &result.regions {
+ println!(" {}", format_region(region));
+ }
+ if let Some(dir) = dump_dir {
+ dump(dir, image_path, &debug)?;
+ }
+ Ok(())
+}
+
+fn format_region(region: &TextRegion) -> String {
+ let corners = region
+ .points
+ .iter()
+ .map(|point| format!("({:.0},{:.0})", point.x, point.y))
+ .collect::<Vec<_>>()
+ .join(" ");
+ format!("[{:.3}] {corners}", region.confidence)
+}
+
+fn dump(dir: &Path, image_path: &str, debug: &RegionDetectionDebug) -> Result<()> {
+ let stem = Path::new(image_path)
+ .file_stem()
+ .and_then(|stem| stem.to_str())
+ .with_context(|| format!("no file stem in {image_path}"))?;
+ let target = dir.join(stem);
+ std::fs::create_dir_all(&target).with_context(|| format!("create {}", target.display()))?;
+ probability_map_image(&debug.probability_map)?
+ .save(target.join("probmap.png"))
+ .context("write probmap.png")?;
+ overlay_image(image_path, &debug.result.regions)?
+ .save(target.join("overlay.png"))
+ .context("write overlay.png")?;
+ println!(" wrote {}", target.display());
+ Ok(())
+}
+
+fn probability_map_image(map: &ProbabilityMap) -> Result<GrayImage> {
+ let pixels = map
+ .values
+ .iter()
+ .map(|value| (value.clamp(0.0, 1.0) * 255.0).round() as u8)
+ .collect();
+ GrayImage::from_raw(map.width as u32, map.height as u32, pixels)
+ .context("probability map size does not match its values")
+}
+
+fn overlay_image(image_path: &str, regions: &[TextRegion]) -> Result<RgbImage> {
+ let decoded = decode_image_from_path(image_path)?;
+ let mut canvas = RgbImage::from_raw(
+ decoded.dimensions.width,
+ decoded.dimensions.height,
+ decoded.rgb,
+ )
+ .context("decoded image size does not match its buffer")?;
+ let thickness = (canvas.width().max(canvas.height()) / 800).clamp(1, 6);
+ for region in regions {
+ draw_quad(&mut canvas, &region.points, thickness);
+ }
+ Ok(canvas)
+}
+
+fn draw_quad(canvas: &mut RgbImage, points: &[Point; 4], thickness: u32) {
+ for index in 0..4 {
+ let start = points[index];
+ let end = points[(index + 1) % 4];
+ for offset in 0..thickness {
+ let shift = offset as f32;
+ draw_line_segment_mut(
+ canvas,
+ (start.x + shift, start.y),
+ (end.x + shift, end.y),
+ OVERLAY_COLOR,
+ );
+ draw_line_segment_mut(
+ canvas,
+ (start.x, start.y + shift),
+ (end.x, end.y + shift),
+ OVERLAY_COLOR,
+ );
+ }
+ }
+}
diff --git a/rust/crates/ml/src/ocr/assets.rs b/rust/crates/ml/src/ocr/assets.rs
new file mode 100644
index 0000000000..c5b3f81f00
--- /dev/null
+++ b/rust/crates/ml/src/ocr/assets.rs
@@ -0,0 +1,186 @@
+use std::time::Instant;
+
+use ente_assets::download::CancellationToken;
+use ente_assets::{Asset, AssetFile, AssetStore};
+
+use super::OcrModelPaths;
+use crate::error::{MlError, MlResult};
+
+const MODEL_BASE_URL: &str = "https://models.ente.com/PP-OCRv5";
+const MODELS: &str = "models";
+
+struct OcrModelFile {
+ key: &'static str,
+ name: &'static str,
+ size: u64,
+ sha256: &'static str,
+}
+
+const DETECTION: OcrModelFile = OcrModelFile {
+ key: "ppocrv5_det",
+ name: "det.onnx",
+ size: 4_748_769,
+ sha256: "d7fe3ea74652890722c0f4d02458b7261d9f5ae6c92904d05707c9eb155c7924",
+};
+
+const CLASSIFICATION: OcrModelFile = OcrModelFile {
+ key: "ppocrv5_cls",
+ name: "cls.onnx",
+ size: 582_663,
+ sha256: "f4bb53707100c5f3d59ba834eb05bb400369f20aed35d4b26807b1bfadd2a70e",
+};
+
+const RECOGNITION: OcrModelFile = OcrModelFile {
+ key: "ppocrv5_rec",
+ name: "rec.onnx",
+ size: 16_517_247,
+ sha256: "bf66820f48fa99f779974c4df78e5274a9d8e0458c4137e8c5357e40e2c3faf2",
+};
+
+const DICTIONARY: OcrModelFile = OcrModelFile {
+ key: "ppocrv5_dict",
+ name: "ppocrv5_dict.txt",
+ size: 74_012,
+ sha256: "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b",
+};
+
+const CATALOG: [&OcrModelFile; 4] = [&DETECTION, &CLASSIFICATION, &RECOGNITION, &DICTIONARY];
+
+impl OcrModelFile {
+ fn url(&self) -> String {
+ format!("{MODEL_BASE_URL}/{}", self.name)
+ }
+
+ fn asset(&self) -> Asset {
+ Asset::file(
+ vec![MODELS.to_string(), self.key.to_string()],
+ AssetFile {
+ name: self.name.to_string(),
+ url: self.url(),
+ size: self.size,
+ sha256: self.sha256.to_string(),
+ },
+ )
+ .expect("valid OCR model catalog")
+ }
+
+ fn path(&self, store: &AssetStore) -> String {
+ store
+ .file_path(&self.asset(), self.name)
+ .expect("OCR model file")
+ .to_string_lossy()
+ .into_owned()
+ }
+}
+
+pub fn model_assets() -> Vec<Asset> {
+ CATALOG.iter().map(|file| file.asset()).collect()
+}
+
+pub async fn ensure_models(store: &AssetStore) -> MlResult<OcrModelPaths> {
+ let missing: Vec<(&OcrModelFile, Asset)> = CATALOG
+ .iter()
+ .map(|file| (*file, file.asset()))
+ .filter(|(_, asset)| !store.is_downloaded(asset))
+ .collect();
+ if missing.is_empty() {
+ log::info!("ocr models: using cached copies");
+ } else {
+ for (file, _) in &missing {
+ log::info!(
+ "ocr models: downloading {} ({} bytes) from {}",
+ file.name,
+ file.size,
+ file.url()
+ );
+ }
+ let start = Instant::now();
+ let assets: Vec<Asset> = missing.iter().map(|(_, asset)| asset.clone()).collect();
+ store
+ .download(&assets, |_| {}, CancellationToken::default())
+ .await
+ .map_err(|error| MlError::Runtime(format!("ocr model download failed: {error}")))?;
+ log::info!(
+ "ocr models: downloaded {} file(s) in {}ms",
+ assets.len(),
+ start.elapsed().as_millis()
+ );
+ }
+ Ok(OcrModelPaths {
+ detection: DETECTION.path(store),
+ classification: CLASSIFICATION.path(store),
+ recognition: RECOGNITION.path(store),
+ dictionary: DICTIONARY.path(store),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashSet;
+ use std::path::Path;
+
+ use super::*;
+
+ #[test]
+ fn catalog_keys_and_names_are_unique() {
+ let keys: HashSet<&str> = CATALOG.iter().map(|file| file.key).collect();
+ let names: HashSet<&str> = CATALOG.iter().map(|file| file.name).collect();
+ assert_eq!(keys.len(), CATALOG.len());
+ assert_eq!(names.len(), CATALOG.len());
+ }
+
+ #[test]
+ fn catalog_directories_do_not_collide_with_the_indexing_catalog() {
+ let store = AssetStore::new(Path::new("cache"));
+ let clip_text = crate::assets::clip_text_asset();
+ let others: Vec<_> = crate::assets::indexing_assets(true, true, true)
+ .iter()
+ .chain(std::iter::once(&clip_text))
+ .map(|asset| store.asset_dir(asset))
+ .collect();
+ assert!(!others.is_empty());
+ for asset in model_assets() {
+ let dir = store.asset_dir(&asset);
+ for other in &others {
+ assert_ne!(&dir, other);
+ assert!(!dir.starts_with(other), "{}", dir.display());
+ assert!(!other.starts_with(&dir), "{}", other.display());
+ }
+ }
+ }
+
+ #[test]
+ fn catalog_points_at_the_pp_ocrv5_cdn_directory() {
+ assert_eq!(DETECTION.url(), "https://models.ente.com/PP-OCRv5/det.onnx");
+ assert_eq!(
+ DICTIONARY.url(),
+ "https://models.ente.com/PP-OCRv5/ppocrv5_dict.txt"
+ );
+ assert_eq!(model_assets().len(), 4);
+ }
+
+ #[test]
+ fn model_paths_live_under_the_store_models_directory() {
+ let store = AssetStore::new(Path::new("cache"));
+ for file in CATALOG {
+ let path = file.path(&store);
+ assert!(path.starts_with("cache/models/"), "{path}");
+ assert!(path.ends_with(file.name), "{path}");
+ assert!(!store.is_downloaded(&file.asset()));
+ }
+ }
+
+ #[test]
+ fn local_copies_match_the_catalog_sizes() {
+ let local = Path::new(
+ "/private/tmp/claude-501/-Users-laurens-dev-ente-repo-ente8/a307b632-3b09-4f43-884c-2830ff54fe8f/scratchpad/models",
+ );
+ if !local.is_dir() {
+ return;
+ }
+ for file in CATALOG {
+ let metadata = std::fs::metadata(local.join(file.name)).unwrap();
+ assert_eq!(metadata.len(), file.size, "{}", file.name);
+ }
+ }
+}
diff --git a/rust/crates/ml/src/ocr/cancel.rs b/rust/crates/ml/src/ocr/cancel.rs
new file mode 100644
index 0000000000..37cbb9100e
--- /dev/null
+++ b/rust/crates/ml/src/ocr/cancel.rs
@@ -0,0 +1,129 @@
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
+
+use super::OcrError;
+
+type Flags = HashMap<String, Arc<AtomicBool>>;
+
+#[derive(Default)]
+pub(crate) struct RequestRegistry {
+ flags: Mutex<Flags>,
+}
+
+impl RequestRegistry {
+ pub(crate) fn begin(&self, request_id: Option<&str>) -> RequestGuard<'_> {
+ let flag = Arc::new(AtomicBool::new(false));
+ if let Some(id) = request_id
+ && let Some(replaced) = self.flags().insert(id.to_string(), Arc::clone(&flag))
+ {
+ replaced.store(true, Ordering::SeqCst);
+ }
+ RequestGuard {
+ registry: self,
+ request_id: request_id.map(str::to_string),
+ flag,
+ }
+ }
+
+ pub(crate) fn cancel(&self, request_id: &str) {
+ if let Some(flag) = self.flags().get(request_id) {
+ flag.store(true, Ordering::SeqCst);
+ }
+ }
+
+ fn finish(&self, request_id: &str, flag: &Arc<AtomicBool>) {
+ let mut flags = self.flags();
+ if flags
+ .get(request_id)
+ .is_some_and(|current| Arc::ptr_eq(current, flag))
+ {
+ flags.remove(request_id);
+ }
+ }
+
+ fn flags(&self) -> MutexGuard<'_, Flags> {
+ self.flags.lock().unwrap_or_else(PoisonError::into_inner)
+ }
+}
+
+pub(crate) struct RequestGuard<'a> {
+ registry: &'a RequestRegistry,
+ request_id: Option<String>,
+ flag: Arc<AtomicBool>,
+}
+
+impl RequestGuard<'_> {
+ pub(crate) fn check(&self) -> Result<(), OcrError> {
+ if self.flag.load(Ordering::SeqCst) {
+ Err(OcrError::Cancelled)
+ } else {
+ Ok(())
+ }
+ }
+}
+
+impl Drop for RequestGuard<'_> {
+ fn drop(&mut self) {
+ if let Some(request_id) = &self.request_id {
+ self.registry.finish(request_id, &self.flag);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn is_cancelled(guard: &RequestGuard<'_>) -> bool {
+ matches!(guard.check(), Err(OcrError::Cancelled))
+ }
+
+ #[test]
+ fn cancel_marks_the_registered_request() {
+ let registry = RequestRegistry::default();
+ let guard = registry.begin(Some("req-1"));
+ assert!(guard.check().is_ok());
+
+ registry.cancel("req-1");
+
+ assert!(is_cancelled(&guard));
+ }
+
+ #[test]
+ fn cancelling_an_unknown_or_anonymous_request_is_a_no_op() {
+ let registry = RequestRegistry::default();
+ let anonymous = registry.begin(None);
+
+ registry.cancel("missing");
+
+ assert!(anonymous.check().is_ok());
+ assert!(registry.flags().is_empty());
+ }
+
+ #[test]
+ fn a_new_request_with_the_same_id_cancels_and_replaces_the_older_one() {
+ let registry = RequestRegistry::default();
+ let first = registry.begin(Some("shared"));
+ let second = registry.begin(Some("shared"));
+
+ assert!(is_cancelled(&first));
+ assert!(second.check().is_ok());
+
+ drop(first);
+ assert_eq!(registry.flags().len(), 1);
+ registry.cancel("shared");
+ assert!(is_cancelled(&second));
+ }
+
+ #[test]
+ fn finishing_a_request_removes_its_flag() {
+ let registry = RequestRegistry::default();
+ let guard = registry.begin(Some("done"));
+ assert_eq!(registry.flags().len(), 1);
+
+ drop(guard);
+
+ assert!(registry.flags().is_empty());
+ }
+}
diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rs
index 588161bcc8..896cfae879 100644
--- a/rust/crates/ml/src/ocr/detect.rs
+++ b/rust/crates/ml/src/ocr/detect.rs
@@ -1,3 +1,5 @@
+use std::sync::{Mutex, PoisonError};
+
use super::Point;
use super::geometry::{
clip_to_bounds, mean_inside_quad, min_area_rect, min_edge, order_corners, scale_points, unclip,
@@ -5,7 +7,13 @@ use super::geometry::{
use crate::cv;
use crate::cv::image::{Contour, ImageU8};
use crate::error::{MlError, MlResult};
+use crate::onnx::{ExecutionMode, OnnxSession, PreparedF32Input, SessionRunError, run_f32};
+const MODEL_NAMESPACE: &str = "ocr-detection";
+const MAX_INPUT_SIDE: i32 = 960;
+const INPUT_STRIDE: i32 = 32;
+const CHANNEL_MEAN_BGR: [f32; 3] = [0.485, 0.456, 0.406];
+const CHANNEL_STD_BGR: [f32; 3] = [0.229, 0.224, 0.225];
const BITMAP_THRESHOLD: f32 = 0.3;
const BOX_SCORE_THRESHOLD: f32 = 0.6;
const UNCLIP_RATIO: f32 = 1.5;
@@ -15,11 +23,113 @@ const MAX_CANDIDATES: usize = 1000;
const READING_LINE_TOLERANCE: f32 = 10.0;
#[derive(Clone, Debug, PartialEq)]
-pub struct DetectionCandidate {
+pub(crate) struct DetectionCandidate {
pub points: [Point; 4],
pub score: f32,
}
+#[derive(Clone, Debug, PartialEq)]
+pub struct ProbabilityMap {
+ pub width: usize,
+ pub height: usize,
+ pub values: Vec<f32>,
+}
+
+pub(crate) struct TextDetection {
+ pub(crate) candidates: Vec<DetectionCandidate>,
+ pub(crate) probability_map: ProbabilityMap,
+}
+
+pub(crate) struct TextDetector {
+ session: Mutex<OnnxSession>,
+}
+
+impl TextDetector {
+ pub(crate) fn new(model_path: &str) -> Self {
+ Self {
+ session: Mutex::new(
+ OnnxSession::new(model_path, MODEL_NAMESPACE, ExecutionMode::CpuAccelerated)
+ .with_unvalidated_acceleration(),
+ ),
+ }
+ }
+
+ pub(crate) fn detect(&self, working: &ImageU8) -> MlResult<TextDetection> {
+ let (input_width, input_height) = detector_input_size(working.width, working.height);
+ let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)
+ .map_err(MlError::Preprocess)?;
+ let input = PreparedF32Input::new(normalized_bgr_planes(&resized));
+ let values = self.infer(&input, input_width, input_height)?;
+ let probability_map = ProbabilityMap {
+ width: input_width as usize,
+ height: input_height as usize,
+ values,
+ };
+ let mut candidates = candidates_from_probability_map(
+ &probability_map.values,
+ probability_map.width,
+ probability_map.height,
+ working.width as u32,
+ working.height as u32,
+ )?;
+ sort_reading_order(&mut candidates);
+ Ok(TextDetection {
+ candidates,
+ probability_map,
+ })
+ }
+
+ fn infer(&self, input: &PreparedF32Input, width: i32, height: i32) -> MlResult<Vec<f32>> {
+ let expected_shape = [1i64, 1, i64::from(height), i64::from(width)];
+ let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
+ let (values, _usage) = session.run(|session| {
+ let (shape, values) =
+ run_f32(session, input, [1, 3, i64::from(height), i64::from(width)])?;
+ if shape != expected_shape {
+ return Err(SessionRunError::from(MlError::CorruptModel(format!(
+ "text detector produced output shape {shape:?}, expected {expected_shape:?}"
+ ))));
+ }
+ Ok(values)
+ })?;
+ Ok(values)
+ }
+}
+
+pub(crate) fn detector_input_size(width: i32, height: i32) -> (i32, i32) {
+ let longest = width.max(height);
+ let ratio = if longest > MAX_INPUT_SIDE {
+ f64::from(MAX_INPUT_SIDE) / f64::from(longest)
+ } else {
+ 1.0
+ };
+ (
+ rounded_to_stride(f64::from(width) * ratio),
+ rounded_to_stride(f64::from(height) * ratio),
+ )
+}
+
+fn rounded_to_stride(side: f64) -> i32 {
+ let stride = f64::from(INPUT_STRIDE);
+ ((side.trunc() / stride).round_ties_even() as i32 * INPUT_STRIDE).max(INPUT_STRIDE)
+}
+
+fn normalized_bgr_planes(rgb: &ImageU8) -> Vec<f32> {
+ let pixels = rgb.width as usize * rgb.height as usize;
+ let mut planes = vec![0.0f32; pixels * 3];
+ let (blue, rest) = planes.split_at_mut(pixels);
+ let (green, red) = rest.split_at_mut(pixels);
+ let normalize = |channel: usize, value: u8| {
+ (value as f32 / 255.0 - CHANNEL_MEAN_BGR[channel]) / CHANNEL_STD_BGR[channel]
+ };
+ for (index, px) in rgb.data.as_chunks::<3>().0.iter().enumerate() {
+ blue[index] = normalize(0, px[2]);
+ green[index] = normalize(1, px[1]);
+ red[index] = normalize(2, px[0]);
+ }
+ planes
+}
+
impl DetectionCandidate {
fn scaled(self, scale_x: f32, scale_y: f32) -> Self {
Self {
@@ -33,13 +143,13 @@ impl DetectionCandidate {
}
}
-struct ProbabilityMap<'a> {
+struct MapView<'a> {
values: &'a [f32],
width: usize,
height: usize,
}
-pub fn candidates_from_probability_map(
+pub(crate) fn candidates_from_probability_map(
values: &[f32],
map_width: usize,
map_height: usize,
@@ -52,7 +162,7 @@ pub fn candidates_from_probability_map(
values.len()
)));
}
- let map = ProbabilityMap {
+ let map = MapView {
values,
width: map_width,
height: map_height,
@@ -67,7 +177,7 @@ pub fn candidates_from_probability_map(
Ok(candidates)
}
-fn threshold_bitmap(map: &ProbabilityMap<'_>) -> MlResult<ImageU8> {
+fn threshold_bitmap(map: &MapView<'_>) -> MlResult<ImageU8> {
let data = map
.values
.iter()
@@ -76,7 +186,7 @@ fn threshold_bitmap(map: &ProbabilityMap<'_>) -> MlResult<ImageU8> {
ImageU8::new(map.width as i32, map.height as i32, 1, data).map_err(MlError::Postprocess)
}
-fn largest_outer_contours(map: &ProbabilityMap<'_>) -> MlResult<Vec<Contour>> {
+fn largest_outer_contours(map: &MapView<'_>) -> MlResult<Vec<Contour>> {
let bitmap = threshold_bitmap(map)?;
let mut contours: Vec<Contour> = cv::find_contours(&bitmap)
.map_err(MlError::Postprocess)?
@@ -88,10 +198,7 @@ fn largest_outer_contours(map: &ProbabilityMap<'_>) -> MlResult<Vec<Contour>> {
Ok(contours)
}
-fn candidate_from_contour(
- contour: &Contour,
- map: &ProbabilityMap<'_>,
-) -> Option<DetectionCandidate> {
+fn candidate_from_contour(contour: &Contour, map: &MapView<'_>) -> Option<DetectionCandidate> {
let rect = order_corners(min_area_rect(&contour.points)?);
if min_edge(&rect) < MIN_BOX_SIDE {
return None;
@@ -108,7 +215,7 @@ fn candidate_from_contour(
Some(DetectionCandidate { points, score })
}
-pub fn sort_reading_order(candidates: &mut [DetectionCandidate]) {
+pub(crate) fn sort_reading_order(candidates: &mut [DetectionCandidate]) {
candidates.sort_by(|a, b| a.top_left().y.total_cmp(&b.top_left().y));
let mut start = 0;
while start < candidates.len() {
@@ -342,4 +449,44 @@ mod tests {
let error = candidates_from_probability_map(&[0.0; 10], 4, 4, 4, 4).unwrap_err();
assert!(matches!(error, MlError::Postprocess(_)), "{error}");
}
+
+ #[test]
+ fn detector_input_truncates_then_rounds_half_to_even_to_a_multiple_of_32() {
+ assert_eq!(detector_input_size(1000, 700), (960, 672));
+ assert_eq!(detector_input_size(500, 300), (512, 288));
+ assert_eq!(detector_input_size(33, 33), (32, 32));
+ assert_eq!(detector_input_size(4000, 3000), (960, 704));
+ assert_eq!(detector_input_size(1280, 960), (960, 704));
+ assert_eq!(detector_input_size(752, 720), (768, 704));
+ assert_eq!(detector_input_size(1920, 1441), (960, 704));
+ assert_eq!(detector_input_size(960, 960), (960, 960));
+ assert_eq!(detector_input_size(1, 1), (32, 32));
+ }
+
+ #[test]
+ fn detector_planes_are_bgr_and_normalised_per_channel() {
+ let pixel = ImageU8::new(1, 1, 3, vec![10, 20, 30]).unwrap();
+ let planes = normalized_bgr_planes(&pixel);
+ let expected = [
+ (30.0 / 255.0 - 0.485) / 0.229,
+ (20.0 / 255.0 - 0.456) / 0.224,
+ (10.0 / 255.0 - 0.406) / 0.225,
+ ];
+ assert_eq!(planes.len(), 3);
+ for (actual, expected) in planes.iter().zip(expected) {
+ assert!((actual - expected).abs() <= 1e-6, "{planes:?}");
+ }
+ }
+
+ #[test]
+ fn detector_planes_are_laid_out_plane_by_plane() {
+ let two_pixels = ImageU8::new(2, 1, 3, vec![255, 0, 0, 0, 0, 255]).unwrap();
+ let planes = normalized_bgr_planes(&two_pixels);
+ let blue_of = |value: f32| (value / 255.0 - 0.485) / 0.229;
+ let red_of = |value: f32| (value / 255.0 - 0.406) / 0.225;
+ assert!((planes[0] - blue_of(0.0)).abs() <= 1e-6);
+ assert!((planes[1] - blue_of(255.0)).abs() <= 1e-6);
+ assert!((planes[4] - red_of(255.0)).abs() <= 1e-6);
+ assert!((planes[5] - red_of(0.0)).abs() <= 1e-6);
+ }
}
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index bcd6519e65..5c8c69cc26 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -1,14 +1,27 @@
+pub mod assets;
+mod cancel;
mod detect;
mod dictionary;
mod geometry;
+mod source;
-pub use detect::{DetectionCandidate, candidates_from_probability_map, sort_reading_order};
+pub use detect::ProbabilityMap;
pub use dictionary::load_dictionary;
pub use geometry::Point;
+use std::time::Instant;
+
use thiserror::Error;
-use crate::error::MlError;
+use cancel::RequestRegistry;
+use detect::{DetectionCandidate, TextDetector};
+use geometry::scale_points;
+use source::{REGIONS_CAP, SourceImage, load_source};
+
+use crate::error::{MlError, MlResult};
+
+const REGION_MIN_SCORE: f32 = 0.5;
+const MAX_REGIONS: usize = 1000;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OcrModelPaths {
@@ -66,6 +79,14 @@ pub struct TextRegionDetectionResult {
pub image_height: u32,
}
+#[derive(Clone, Debug, PartialEq)]
+pub struct RegionDetectionDebug {
+ pub result: TextRegionDetectionResult,
+ pub working_width: u32,
+ pub working_height: u32,
+ pub probability_map: ProbabilityMap,
+}
+
#[derive(Debug, Error)]
pub enum OcrError {
#[error("image not found: {0}")]
@@ -75,3 +96,167 @@ pub enum OcrError {
#[error(transparent)]
Ml(#[from] MlError),
}
+
+pub struct OcrEngine {
+ detector: TextDetector,
+ requests: RequestRegistry,
+}
+
+impl OcrEngine {
+ pub fn new(paths: OcrModelPaths) -> MlResult<Self> {
+ if paths.detection.is_empty() {
+ return Err(MlError::InvalidRequest(
+ "OCR detection model path is empty".to_string(),
+ ));
+ }
+ Ok(Self {
+ detector: TextDetector::new(&paths.detection),
+ requests: RequestRegistry::default(),
+ })
+ }
+
+ pub fn detect_text_regions(
+ &self,
+ req: &DetectRegionsRequest,
+ ) -> Result<TextRegionDetectionResult, OcrError> {
+ self.detect_text_regions_debug(req)
+ .map(|debug| debug.result)
+ }
+
+ pub fn detect_text_regions_debug(
+ &self,
+ req: &DetectRegionsRequest,
+ ) -> Result<RegionDetectionDebug, OcrError> {
+ let started = Instant::now();
+ let request = self.requests.begin(req.request_id.as_deref());
+ let source = load_source(&req.image_path, REGIONS_CAP)?;
+ request.check()?;
+ let detection_started = Instant::now();
+ let detection = self.detector.detect(&source.working)?;
+ let detection_ms = detection_started.elapsed().as_millis();
+ request.check()?;
+ let regions = regions_in_decoded_pixels(&detection.candidates, &source);
+ log::info!(
+ "ocr detect_text_regions: {}x{} -> {}x{}, det {} boxes in {detection_ms}ms, kept {}, total {}ms",
+ source.decoded_width,
+ source.decoded_height,
+ source.working.width,
+ source.working.height,
+ detection.candidates.len(),
+ regions.len(),
+ started.elapsed().as_millis()
+ );
+ Ok(RegionDetectionDebug {
+ result: TextRegionDetectionResult {
+ regions,
+ image_width: source.decoded_width,
+ image_height: source.decoded_height,
+ },
+ working_width: source.working.width as u32,
+ working_height: source.working.height as u32,
+ probability_map: detection.probability_map,
+ })
+ }
+
+ pub fn cancel(&self, request_id: &str) {
+ self.requests.cancel(request_id);
+ }
+}
+
+fn regions_in_decoded_pixels(
+ candidates: &[DetectionCandidate],
+ source: &SourceImage,
+) -> Vec<TextRegion> {
+ let scale_x = source.decoded_width as f32 / source.working.width as f32;
+ let scale_y = source.decoded_height as f32 / source.working.height as f32;
+ candidates
+ .iter()
+ .filter(|candidate| candidate.score >= REGION_MIN_SCORE)
+ .take(MAX_REGIONS)
+ .map(|candidate| TextRegion {
+ confidence: candidate.score,
+ points: scale_points(&candidate.points, scale_x, scale_y),
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn engine_requires_a_detection_model_path() {
+ let error = OcrEngine::new(OcrModelPaths {
+ detection: String::new(),
+ classification: "cls.onnx".to_string(),
+ recognition: "rec.onnx".to_string(),
+ dictionary: "dict.txt".to_string(),
+ })
+ .err()
+ .expect("empty detection path is rejected");
+ assert!(matches!(error, MlError::InvalidRequest(_)), "{error}");
+ }
+
+ #[test]
+ fn engine_accepts_a_detector_only_path_set_without_loading_it() {
+ let engine = OcrEngine::new(OcrModelPaths {
+ detection: "missing/det.onnx".to_string(),
+ classification: String::new(),
+ recognition: String::new(),
+ dictionary: String::new(),
+ });
+ assert!(engine.is_ok());
+ }
+
+ #[test]
+ fn missing_image_is_reported_before_the_detector_is_touched() {
+ let engine = OcrEngine::new(OcrModelPaths {
+ detection: "missing/det.onnx".to_string(),
+ classification: String::new(),
+ recognition: String::new(),
+ dictionary: String::new(),
+ })
+ .unwrap();
+ let error = engine
+ .detect_text_regions(&DetectRegionsRequest {
+ image_path: "missing/image.jpg".to_string(),
+ request_id: Some("r1".to_string()),
+ })
+ .unwrap_err();
+ assert!(matches!(error, OcrError::ImageNotFound(_)), "{error}");
+ }
+
+ #[test]
+ fn regions_scale_from_working_to_decoded_pixels_and_drop_low_scores() {
+ let source = SourceImage {
+ decoded_width: 2000,
+ decoded_height: 1000,
+ working: crate::cv::image::ImageU8::zeros(1000, 500, 3).unwrap(),
+ };
+ let quad = |x: f32, y: f32| {
+ [
+ Point::new(x, y),
+ Point::new(x + 10.0, y),
+ Point::new(x + 10.0, y + 5.0),
+ Point::new(x, y + 5.0),
+ ]
+ };
+ let candidates = vec![
+ DetectionCandidate {
+ points: quad(100.0, 50.0),
+ score: 0.9,
+ },
+ DetectionCandidate {
+ points: quad(300.0, 50.0),
+ score: 0.4,
+ },
+ ];
+
+ let regions = regions_in_decoded_pixels(&candidates, &source);
+
+ assert_eq!(regions.len(), 1);
+ assert_eq!(regions[0].confidence, 0.9);
+ assert_eq!(regions[0].points[0], Point::new(200.0, 100.0));
+ assert_eq!(regions[0].points[2], Point::new(220.0, 110.0));
+ }
+}
diff --git a/rust/crates/ml/src/ocr/source.rs b/rust/crates/ml/src/ocr/source.rs
new file mode 100644
index 0000000000..240ca72bd0
--- /dev/null
+++ b/rust/crates/ml/src/ocr/source.rs
@@ -0,0 +1,170 @@
+use std::path::Path;
+
+use ente_image::decode::decode_image_from_path;
+
+use super::OcrError;
+use crate::cv;
+use crate::cv::image::ImageU8;
+use crate::error::{MlError, MlResult};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct WorkingSizeCap {
+ max_side: u32,
+ max_pixels: u32,
+}
+
+pub(crate) const REGIONS_CAP: WorkingSizeCap = WorkingSizeCap {
+ max_side: 1280,
+ max_pixels: 2_000_000,
+};
+
+impl WorkingSizeCap {
+ fn scale(self, width: u32, height: u32) -> f64 {
+ let longest = f64::from(width.max(height));
+ let pixels = f64::from(width) * f64::from(height);
+ (f64::from(self.max_side) / longest)
+ .min((f64::from(self.max_pixels) / pixels).sqrt())
+ .min(1.0)
+ }
+
+ pub(crate) fn working_size(self, width: u32, height: u32) -> (u32, u32) {
+ let scale = self.scale(width, height);
+ if scale >= 1.0 {
+ return (width, height);
+ }
+ let scaled = |side: u32| ((f64::from(side) * scale).round() as u32).max(1);
+ (scaled(width), scaled(height))
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct SourceImage {
+ pub(crate) decoded_width: u32,
+ pub(crate) decoded_height: u32,
+ pub(crate) working: ImageU8,
+}
+
+pub(crate) fn load_source(image_path: &str, cap: WorkingSizeCap) -> Result<SourceImage, OcrError> {
+ if !Path::new(image_path).is_file() {
+ return Err(OcrError::ImageNotFound(image_path.to_string()));
+ }
+ let decoded = decode_image_from_path(image_path).map_err(MlError::from)?;
+ let (width, height) = (decoded.dimensions.width, decoded.dimensions.height);
+ let full =
+ ImageU8::new(to_side(width)?, to_side(height)?, 3, decoded.rgb).map_err(MlError::Image)?;
+ let (working_width, working_height) = cap.working_size(width, height);
+ let working = if (working_width, working_height) == (width, height) {
+ full
+ } else {
+ cv::resize_u8(
+ &full,
+ to_side(working_width)?,
+ to_side(working_height)?,
+ cv::Interp::Area,
+ )
+ .map_err(MlError::Image)?
+ };
+ Ok(SourceImage {
+ decoded_width: width,
+ decoded_height: height,
+ working,
+ })
+}
+
+fn to_side(value: u32) -> MlResult<i32> {
+ i32::try_from(value).map_err(|_| MlError::Image(format!("image side {value} is too large")))
+}
+
+#[cfg(test)]
+mod tests {
+ use std::fs::File;
+ use std::io::BufWriter;
+ use std::path::Path;
+
+ use image::ExtendedColorType;
+ use image::ImageEncoder;
+ use image::codecs::png::{CompressionType, FilterType, PngEncoder};
+
+ use super::*;
+
+ const FULL_TEXT_CAP: WorkingSizeCap = WorkingSizeCap {
+ max_side: 4096,
+ max_pixels: 12_000_000,
+ };
+
+ fn write_png(dir: &Path, name: &str, width: u32, height: u32) -> String {
+ let path = dir.join(name);
+ let writer = BufWriter::new(File::create(&path).unwrap());
+ let pixels = vec![128u8; (width * height * 3) as usize];
+ PngEncoder::new_with_quality(writer, CompressionType::Fast, FilterType::NoFilter)
+ .write_image(&pixels, width, height, ExtendedColorType::Rgb8)
+ .unwrap();
+ path.to_string_lossy().into_owned()
+ }
+
+ #[test]
+ fn wide_image_is_capped_at_the_longest_side() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = write_png(dir.path(), "wide.png", 5000, 1000);
+
+ let source = load_source(&path, FULL_TEXT_CAP).unwrap();
+
+ assert_eq!((source.decoded_width, source.decoded_height), (5000, 1000));
+ assert_eq!(source.working.width, 4096);
+ assert_eq!(source.working.height, 819);
+ assert_eq!(source.working.channels, 3);
+ }
+
+ #[test]
+ fn square_image_is_capped_at_the_pixel_budget() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = write_png(dir.path(), "square.png", 4000, 4000);
+
+ let source = load_source(&path, FULL_TEXT_CAP).unwrap();
+
+ let pixels = source.working.width as u64 * source.working.height as u64;
+ assert!(pixels <= 12_000_000, "{pixels}");
+ assert!(pixels > 11_900_000, "{pixels}");
+ assert_eq!(source.working.width, source.working.height);
+ }
+
+ #[test]
+ fn small_image_is_kept_as_is() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = write_png(dir.path(), "small.png", 640, 480);
+
+ let source = load_source(&path, REGIONS_CAP).unwrap();
+
+ assert_eq!((source.decoded_width, source.decoded_height), (640, 480));
+ assert_eq!((source.working.width, source.working.height), (640, 480));
+ assert_eq!(source.working.data.len(), 640 * 480 * 3);
+ }
+
+ #[test]
+ fn missing_file_is_reported_before_decoding() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir
+ .path()
+ .join("missing.jpg")
+ .to_string_lossy()
+ .into_owned();
+
+ let error = load_source(&path, REGIONS_CAP).unwrap_err();
+
+ assert!(
+ matches!(&error, OcrError::ImageNotFound(reported) if reported == &path),
+ "{error}"
+ );
+ }
+
+ #[test]
+ fn regions_cap_applies_the_smaller_of_both_limits() {
+ assert_eq!(REGIONS_CAP.working_size(5000, 1000), (1280, 256));
+ assert_eq!(REGIONS_CAP.working_size(2000, 2000), (1280, 1280));
+ assert_eq!(REGIONS_CAP.working_size(1280, 960), (1280, 960));
+ assert_eq!(FULL_TEXT_CAP.working_size(4000, 3000), (4000, 3000));
+ assert_eq!(FULL_TEXT_CAP.working_size(5000, 3000), (4096, 2458));
+ assert_eq!(FULL_TEXT_CAP.working_size(4000, 4000), (3464, 3464));
+ assert_eq!(FULL_TEXT_CAP.working_size(1, 9000), (1, 4096));
+ }
+}
diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rs
index 96a4536afd..c6434c6429 100644
--- a/rust/crates/ml/src/onnx/providers.rs
+++ b/rust/crates/ml/src/onnx/providers.rs
@@ -48,6 +48,7 @@ const ENABLE_PERSISTENT_COREML_CACHE: bool = true;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ExecutionMode {
PlatformDefault,
+ CpuAccelerated,
CpuOnly,
}
@@ -77,6 +78,7 @@ impl ProviderPlan {
) -> Self {
let providers = match mode {
ExecutionMode::PlatformDefault => platform_default_providers(model_path, validation),
+ ExecutionMode::CpuAccelerated => cpu_accelerated_providers(),
ExecutionMode::CpuOnly => vec![ExecutionProvider::Cpu],
};
Self::from_providers(providers)
@@ -271,6 +273,16 @@ fn platform_default_providers(
vec![ExecutionProvider::Cpu]
}
+#[cfg(target_os = "android")]
+fn cpu_accelerated_providers() -> Vec<ExecutionProvider> {
+ vec![ExecutionProvider::Xnnpack, ExecutionProvider::Cpu]
+}
+
+#[cfg(not(target_os = "android"))]
+fn cpu_accelerated_providers() -> Vec<ExecutionProvider> {
+ vec![ExecutionProvider::Cpu]
+}
+
// Missing goldens fail closed, usually indicating they were not regenerated
// after an update.
#[cfg(any(
@@ -478,6 +490,21 @@ mod tests {
assert!(!plan.has_fallback());
}
+ #[test]
+ fn cpu_accelerated_plan_ends_with_cpu_and_never_uses_gpu_providers() {
+ let plan = ProviderPlan::new(
+ ExecutionMode::CpuAccelerated,
+ "model.onnx",
+ AccelerationValidation::Unvalidated,
+ );
+
+ assert_eq!(plan.providers.last(), Some(&ExecutionProvider::Cpu));
+ assert!(!plan.providers.iter().any(|provider| matches!(
+ provider,
+ ExecutionProvider::WebGpu | ExecutionProvider::CoreMl
+ )));
+ }
+
fn accelerated_provider_plan() -> ProviderPlan {
ProviderPlan::from_providers(vec![
ExecutionProvider::WebGpu,
diff --git a/rust/crates/ml/tests/support/ml_indexing.rs b/rust/crates/ml/tests/support/ml_indexing.rs
index e5f80fbd0f..1c77338c0e 100644
--- a/rust/crates/ml/tests/support/ml_indexing.rs
+++ b/rust/crates/ml/tests/support/ml_indexing.rs
@@ -188,11 +188,7 @@ impl MlIndexingTestContext {
let golden_results = load_golden_results(&golden_path)?;
let fixture_paths = fetch_fixtures(&store, &asset_lock.fixture_base_url, &manifest).await?;
- let onnx_runtime_library =
- resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;
- let _ = ort::init_from(&onnx_runtime_library)
- .context("load ONNX Runtime dynamic library")?
- .commit();
+ init_onnx_runtime(&store, &asset_lock).await?;
let model_paths = resolve_model_paths(&store, &asset_lock.models).await?;
@@ -415,11 +411,7 @@ impl GoldenTestAssets {
let cache_dir = cache_dir(&repo_root);
let store = AssetStore::new(&cache_dir);
- let onnx_runtime_library =
- resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;
- let _ = ort::init_from(&onnx_runtime_library)
- .context("load ONNX Runtime dynamic library")?
- .commit();
+ init_onnx_runtime(&store, &asset_lock).await?;
let models = &asset_lock.models;
Ok(Self {
@@ -555,6 +547,28 @@ struct ComparableFace {
embedding: Vec<f64>,
}
+#[allow(dead_code)]
+pub(crate) async fn load_onnx_runtime() -> Result<()> {
+ let repo_root = repo_root()?;
+ let asset_lock = load_asset_lock(&repo_root)?;
+ let store = AssetStore::new(cache_dir(&repo_root));
+ init_onnx_runtime(&store, &asset_lock).await
+}
+
+#[allow(dead_code)]
+pub(crate) fn asset_cache_dir() -> Result<PathBuf> {
+ Ok(cache_dir(&repo_root()?))
+}
+
+async fn init_onnx_runtime(store: &AssetStore, asset_lock: &AssetLock) -> Result<()> {
+ let onnx_runtime_library =
+ resolve_onnx_runtime_library(store, &asset_lock.onnx_runtime).await?;
+ let _ = ort::init_from(&onnx_runtime_library)
+ .context("load ONNX Runtime dynamic library")?
+ .commit();
+ Ok(())
+}
+
fn repo_root() -> Result<PathBuf> {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
manifest_dir