π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][rust] Manage OCR models with ente-assets
laurenspriema5ff12dc78bd
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart b/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dartindex 2ca02bf2e7..5e875dc204 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart@@ -155,63 +155,3 @@ class PetBodyEmbeddingCatModel extends MlModelAsset {@overrideString get modelSha256 => _sha256;}--class OcrDetectionModel extends MlModelAsset {- static const remoteFileName = "PP-OCRv5/det.onnx";- static const _sha256 =- "d7fe3ea74652890722c0f4d02458b7261d9f5ae6c92904d05707c9eb155c7924";-- OcrDetectionModel._();- static final instance = OcrDetectionModel._();-- @override- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;-- @override- String get modelSha256 => _sha256;-}--class OcrClassificationModel extends MlModelAsset {- static const remoteFileName = "PP-OCRv5/cls.onnx";- static const _sha256 =- "f4bb53707100c5f3d59ba834eb05bb400369f20aed35d4b26807b1bfadd2a70e";-- OcrClassificationModel._();- static final instance = OcrClassificationModel._();-- @override- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;-- @override- String get modelSha256 => _sha256;-}--class OcrRecognitionModel extends MlModelAsset {- static const remoteFileName = "PP-OCRv5/rec.onnx";- static const _sha256 =- "bf66820f48fa99f779974c4df78e5274a9d8e0458c4137e8c5357e40e2c3faf2";-- OcrRecognitionModel._();- static final instance = OcrRecognitionModel._();-- @override- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;-- @override- String get modelSha256 => _sha256;-}--class OcrDictionaryAsset extends MlModelAsset {- static const remoteFileName = "PP-OCRv5/ppocrv5_dict.txt";- static const _sha256 =- "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b";-- OcrDictionaryAsset._();- static final instance = OcrDictionaryAsset._();-- @override- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;-- @override- String get modelSha256 => _sha256;-}diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dartindex 67ccd52c59..d2744d2cfe 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart@@ -1,81 +1,63 @@-import "dart:io";import "dart:ui";import "package:logging/logging.dart";-import "package:photos/services/machine_learning/ml_model_assets.dart";+import "package:path/path.dart" as p;+import "package:path_provider/path_provider.dart";import "package:photos/services/machine_learning/ocr/ocr_backend.dart";import "package:photos/services/machine_learning/ocr/ocr_models.dart";-import "package:photos/services/remote_assets_service.dart";import "package:photos/src/rust/api/ocr_api.dart";import "package:synchronized/synchronized.dart";class RustOcrBackend implements OcrBackend {static final _logger = Logger("RustOcrBackend");static const _modelVersion = "pp-ocrv5";- static const _noModelPaths = RustOcrModelPaths(- detection: "",- classification: "",- recognition: "",- dictionary: "",- );final _engineLock = Lock();OcrEngine? _engine;- RustOcrModelPaths _enginePaths = _noModelPaths;+ String? _assetsDir;+ bool _includesRecognizer = false;@overrideFuture<ModelPreparationStatus> prepareModels(Set<OcrModelComponent> components,) {return _engineLock.synchronized(() async {- final paths = await _downloadModels(+ final assetsDir = await _getAssetsDirectory();+ await _ensureEngine(+ assetsDir,includeRecognizer: components.contains(OcrModelComponent.recognizer),);- await _ensureEngine(paths);return ModelPreparationStatus(isReady: true,version: _modelVersion,- modelPath: File(paths.detection).parent.path,+ modelPath: assetsDir,);});}- Future<RustOcrModelPaths> _downloadModels({- required bool includeRecognizer,- }) async {- if (!includeRecognizer) {- final detection = await OcrDetectionModel.instance.downloadModel();- return RustOcrModelPaths(- detection: detection,- classification: _enginePaths.classification,- recognition: _enginePaths.recognition,- dictionary: _enginePaths.dictionary,- );- }- final paths = await Future.wait([- OcrDetectionModel.instance.downloadModel(),- OcrClassificationModel.instance.downloadModel(),- OcrRecognitionModel.instance.downloadModel(),- OcrDictionaryAsset.instance.downloadModel(),- ]);- return RustOcrModelPaths(- detection: paths[0],- classification: paths[1],- recognition: paths[2],- dictionary: paths[3],+ Future<String> _getAssetsDirectory() async {+ return _assetsDir ??= p.join(+ (await getApplicationSupportDirectory()).path,+ "assets",);}- Future<void> _ensureEngine(RustOcrModelPaths paths) async {- if (_engine != null && paths == _enginePaths) {+ Future<void> _ensureEngine(+ String assetsDir, {+ required bool includeRecognizer,+ }) async {+ if (_engine != null && (!includeRecognizer || _includesRecognizer)) {return;}try {- _engine = await OcrEngine.create(paths: paths);- _enginePaths = paths;- final loadedModels = paths.recognition.isEmpty- ? "detector only"- : "detector, classifier and recognizer";+ _engine = await OcrEngine.create(+ assetsDir: assetsDir,+ includeRecognizer: includeRecognizer,+ );+ _includesRecognizer = includeRecognizer;+ final loadedModels = includeRecognizer+ ? "detector, classifier and recognizer"+ : "detector only";_logger.info("Created Rust OCR engine ($loadedModels)");} on RustOcrError catch (error) {_logger.severe("Could not create the Rust OCR engine: $error");@@ -139,29 +121,32 @@ class RustOcrBackend implements OcrBackend {Future<OcrEngine> _recognitionEngine() async {final engine = _engine;- if (engine != null && _enginePaths.recognition.isNotEmpty) {+ if (engine != null && _includesRecognizer) {return engine;}await prepareModels(OcrModelComponent.values.toSet());return _requireEngine();}- Future<OcrEngine> _detectionEngine() async {- final engine = _engine;- if (engine != null) {- return engine;- }- final detectorAvailable = await RemoteAssetsService.instance.hasAsset(- OcrDetectionModel.instance.modelRemotePath,- );- if (!detectorAvailable) {- throw const OcrException(- code: "MODEL_NOT_READY",- message: "The OCR detector model is not available locally",+ Future<OcrEngine> _detectionEngine() {+ return _engineLock.synchronized(() async {+ final engine = _engine;+ if (engine != null) {+ return engine;+ }+ final assetsDir = await _getAssetsDirectory();+ final detectorAvailable = await OcrEngine.isDetectorDownloaded(+ assetsDir: assetsDir,);- }- await prepareModels({OcrModelComponent.detector});- return _requireEngine();+ if (!detectorAvailable) {+ throw const OcrException(+ code: "MODEL_NOT_READY",+ message: "The OCR detector model is not available locally",+ );+ }+ await _ensureEngine(assetsDir, includeRecognizer: false);+ return _requireEngine();+ });}OcrEngine _requireEngine() {diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 197478ad04..d700136e81 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2718,6 +2718,7 @@ dependencies = [name = "ente-photos-frb"version = "0.0.0"dependencies = [+ "ente-assets","ente-cast","ente-core","ente-frb-lib",diff --git a/rust/bindings/frb/photos/Cargo.toml b/rust/bindings/frb/photos/Cargo.tomlindex ba8afabf2c..8745430c33 100644--- a/rust/bindings/frb/photos/Cargo.toml+++ b/rust/bindings/frb/photos/Cargo.toml@@ -6,6 +6,7 @@ edition = "2024"crate-type = ["cdylib", "staticlib"][dependencies]+ente-assets.workspace = trueente-cast.workspace = trueente-core.workspace = trueente-frb-lib = { workspace = true, features = ["contacts"] }diff --git a/rust/bindings/frb/photos/src/api/ocr_api.rs b/rust/bindings/frb/photos/src/api/ocr_api.rsindex 64b32a5fc3..7b578ed7df 100644--- a/rust/bindings/frb/photos/src/api/ocr_api.rs+++ b/rust/bindings/frb/photos/src/api/ocr_api.rs@@ -1,17 +1,10 @@use std::path::Path;+use ente_assets::AssetStore;use ente_ml::error::MlError;use ente_ml::ocr;use flutter_rust_bridge::frb;-#[derive(Clone, Debug)]-pub struct RustOcrModelPaths {- pub detection: String,- pub classification: String,- pub recognition: String,- pub dictionary: String,-}-#[derive(Clone, Copy, Debug)]pub struct RustOcrPoint {pub x: f64,@@ -68,18 +61,29 @@ pub struct OcrEngine {}impl OcrEngine {- pub fn create(paths: RustOcrModelPaths) -> Result<OcrEngine, RustOcrError> {- ocr::OcrEngine::new(to_model_paths(paths))- .map(|inner| OcrEngine { inner })- .map_err(|error| {- let error = RustOcrError::from(ocr::OcrError::Ml(error));- log::log!(- error.log_level(),- "Rust OCR create failed: {}",- error.description()- );- error- })+ pub async fn create(+ assets_dir: String,+ include_recognizer: bool,+ ) -> Result<OcrEngine, RustOcrError> {+ let store = AssetStore::new(assets_dir);+ let result = async {+ let paths = ocr::assets::ensure_models(&store, include_recognizer).await?;+ ocr::OcrEngine::new(paths)+ }+ .await;+ result.map(|inner| OcrEngine { inner }).map_err(|error| {+ let error = RustOcrError::from(ocr::OcrError::Ml(error));+ log::log!(+ error.log_level(),+ "Rust OCR create failed: {}",+ error.description()+ );+ error+ })+ }++ pub fn is_detector_downloaded(assets_dir: String) -> bool {+ ocr::assets::is_detector_downloaded(&AssetStore::new(assets_dir))}pub fn detect_text(@@ -183,15 +187,6 @@ fn image_file_name(image_path: &str) -> String {.unwrap_or_default()}-fn to_model_paths(paths: RustOcrModelPaths) -> ocr::OcrModelPaths {- ocr::OcrModelPaths {- detection: paths.detection,- classification: paths.classification,- recognition: paths.recognition,- dictionary: paths.dictionary,- }-}-fn to_api_points(points: &[ocr::Point; 4]) -> Vec<RustOcrPoint> {points.iter()diff --git a/rust/crates/ml/examples/ocr.rs b/rust/crates/ml/examples/ocr.rsindex 849c434776..a109eeb590 100644--- a/rust/crates/ml/examples/ocr.rs+++ b/rust/crates/ml/examples/ocr.rs@@ -51,7 +51,7 @@ async fn main() -> Result<()> {install_logger();load_onnx_runtime().await?;let store = AssetStore::new(asset_cache_dir()?);- let paths = assets::ensure_models(&store).await?;+ let paths = assets::ensure_models(&store, !options.regions).await?;run_with_large_stack("ocr", move || run(options, paths))}diff --git a/rust/crates/ml/src/ocr/assets.rs b/rust/crates/ml/src/ocr/assets.rsindex 7f84f646f6..dd065a46be 100644--- a/rust/crates/ml/src/ocr/assets.rs+++ b/rust/crates/ml/src/ocr/assets.rs@@ -77,8 +77,20 @@ pub fn model_assets() -> Vec<Asset> {CATALOG.iter().map(|file| file.asset()).collect()}-pub async fn ensure_models(store: &AssetStore) -> MlResult<OcrModelPaths> {- let missing: Vec<(&OcrModelFile, Asset)> = CATALOG+pub fn is_detector_downloaded(store: &AssetStore) -> bool {+ store.is_downloaded(&DETECTION.asset())+}++pub async fn ensure_models(+ store: &AssetStore,+ include_recognizer: bool,+) -> MlResult<OcrModelPaths> {+ let files = if include_recognizer {+ &CATALOG[..]+ } else {+ &CATALOG[..1]+ };+ let missing: Vec<(&OcrModelFile, Asset)> = files.iter().map(|file| (*file, file.asset())).filter(|(_, asset)| !store.is_downloaded(asset))@@ -108,18 +120,91 @@ pub async fn ensure_models(store: &AssetStore) -> MlResult<OcrModelPaths> {}Ok(OcrModelPaths {detection: DETECTION.path(store),- classification: CLASSIFICATION.path(store),- recognition: RECOGNITION.path(store),- dictionary: DICTIONARY.path(store),+ classification: if include_recognizer {+ CLASSIFICATION.path(store)+ } else {+ String::new()+ },+ recognition: if include_recognizer {+ RECOGNITION.path(store)+ } else {+ String::new()+ },+ dictionary: if include_recognizer {+ DICTIONARY.path(store)+ } else {+ String::new()+ },})}#[cfg(test)]mod tests {+ use std::fs;use std::path::Path;use super::*;+ fn cache_model(store: &AssetStore, model: &OcrModelFile) {+ let asset = model.asset();+ fs::create_dir_all(store.asset_dir(&asset)).unwrap();+ fs::write(model.path(store), b"cached model").unwrap();+ }++ #[test]+ fn detector_availability_requires_the_published_file() {+ let root = tempfile::tempdir().unwrap();+ let store = AssetStore::new(root.path());+ assert!(!is_detector_downloaded(&store));++ fs::create_dir_all(store.asset_dir(&DETECTION.asset())).unwrap();+ assert!(!is_detector_downloaded(&store));++ cache_model(&store, &RECOGNITION);+ assert!(!is_detector_downloaded(&store));++ cache_model(&store, &DETECTION);+ assert!(is_detector_downloaded(&store));++ store.remove(&DETECTION.asset()).unwrap();+ assert!(!is_detector_downloaded(&store));+ }++ #[tokio::test]+ async fn cached_detector_can_be_prepared_without_recognition_models() {+ let root = tempfile::tempdir().unwrap();+ let store = AssetStore::new(root.path());+ cache_model(&store, &DETECTION);++ let paths = ensure_models(&store, false).await.unwrap();++ assert_eq!(paths.detection, DETECTION.path(&store));+ assert!(paths.classification.is_empty());+ assert!(paths.recognition.is_empty());+ assert!(paths.dictionary.is_empty());+ for model in [&CLASSIFICATION, &RECOGNITION, &DICTIONARY] {+ assert!(!store.is_downloaded(&model.asset()));+ }+ }++ #[tokio::test]+ async fn cached_full_models_can_be_prepared_after_detector_only() {+ let root = tempfile::tempdir().unwrap();+ let store = AssetStore::new(root.path());+ for model in CATALOG {+ cache_model(&store, model);+ }++ let detector_paths = ensure_models(&store, false).await.unwrap();+ let paths = ensure_models(&store, true).await.unwrap();++ assert_eq!(paths.detection, detector_paths.detection);+ assert_eq!(paths.classification, CLASSIFICATION.path(&store));+ assert_eq!(paths.recognition, RECOGNITION.path(&store));+ assert_eq!(paths.dictionary, DICTIONARY.path(&store));+ assert_eq!(fs::read(&paths.detection).unwrap(), b"cached model");+ }+#[test]fn catalog_directories_do_not_collide_with_the_indexing_catalog() {let store = AssetStore::new(Path::new("cache"));