Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][rust] Manage OCR models with ente-assets

laurenspriema5ff12dc78bd
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart b/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart
index 2ca02bf2e7..5e875dc204 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_model_assets.dart
@@ -155,63 +155,3 @@ class PetBodyEmbeddingCatModel extends MlModelAsset {
@override
String get modelSha256 => _sha256;
}
-
-class OcrDetectionModel extends MlModelAsset {
- static const remoteFileName = "PP-OCRv5/det.onnx";
- static const _sha256 =
- "d7fe3ea74652890722c0f4d02458b7261d9f5ae6c92904d05707c9eb155c7924";
-
- OcrDetectionModel._();
- static final instance = OcrDetectionModel._();
-
- @override
- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;
-
- @override
- String get modelSha256 => _sha256;
-}
-
-class OcrClassificationModel extends MlModelAsset {
- static const remoteFileName = "PP-OCRv5/cls.onnx";
- static const _sha256 =
- "f4bb53707100c5f3d59ba834eb05bb400369f20aed35d4b26807b1bfadd2a70e";
-
- OcrClassificationModel._();
- static final instance = OcrClassificationModel._();
-
- @override
- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;
-
- @override
- String get modelSha256 => _sha256;
-}
-
-class OcrRecognitionModel extends MlModelAsset {
- static const remoteFileName = "PP-OCRv5/rec.onnx";
- static const _sha256 =
- "bf66820f48fa99f779974c4df78e5274a9d8e0458c4137e8c5357e40e2c3faf2";
-
- OcrRecognitionModel._();
- static final instance = OcrRecognitionModel._();
-
- @override
- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;
-
- @override
- String get modelSha256 => _sha256;
-}
-
-class OcrDictionaryAsset extends MlModelAsset {
- static const remoteFileName = "PP-OCRv5/ppocrv5_dict.txt";
- static const _sha256 =
- "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b";
-
- OcrDictionaryAsset._();
- static final instance = OcrDictionaryAsset._();
-
- @override
- String get modelRemotePath => kModelBucketEndpoint + remoteFileName;
-
- @override
- String get modelSha256 => _sha256;
-}
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
index 67ccd52c59..d2744d2cfe 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
@@ -1,81 +1,63 @@
-import "dart:io";
import "dart:ui";
import "package:logging/logging.dart";
-import "package:photos/services/machine_learning/ml_model_assets.dart";
+import "package:path/path.dart" as p;
+import "package:path_provider/path_provider.dart";
import "package:photos/services/machine_learning/ocr/ocr_backend.dart";
import "package:photos/services/machine_learning/ocr/ocr_models.dart";
-import "package:photos/services/remote_assets_service.dart";
import "package:photos/src/rust/api/ocr_api.dart";
import "package:synchronized/synchronized.dart";
class RustOcrBackend implements OcrBackend {
static final _logger = Logger("RustOcrBackend");
static const _modelVersion = "pp-ocrv5";
- static const _noModelPaths = RustOcrModelPaths(
- detection: "",
- classification: "",
- recognition: "",
- dictionary: "",
- );
final _engineLock = Lock();
OcrEngine? _engine;
- RustOcrModelPaths _enginePaths = _noModelPaths;
+ String? _assetsDir;
+ bool _includesRecognizer = false;
@override
Future<ModelPreparationStatus> prepareModels(
Set<OcrModelComponent> components,
) {
return _engineLock.synchronized(() async {
- final paths = await _downloadModels(
+ final assetsDir = await _getAssetsDirectory();
+ await _ensureEngine(
+ assetsDir,
includeRecognizer: components.contains(OcrModelComponent.recognizer),
);
- await _ensureEngine(paths);
return ModelPreparationStatus(
isReady: true,
version: _modelVersion,
- modelPath: File(paths.detection).parent.path,
+ modelPath: assetsDir,
);
});
}
- Future<RustOcrModelPaths> _downloadModels({
- required bool includeRecognizer,
- }) async {
- if (!includeRecognizer) {
- final detection = await OcrDetectionModel.instance.downloadModel();
- return RustOcrModelPaths(
- detection: detection,
- classification: _enginePaths.classification,
- recognition: _enginePaths.recognition,
- dictionary: _enginePaths.dictionary,
- );
- }
- final paths = await Future.wait([
- OcrDetectionModel.instance.downloadModel(),
- OcrClassificationModel.instance.downloadModel(),
- OcrRecognitionModel.instance.downloadModel(),
- OcrDictionaryAsset.instance.downloadModel(),
- ]);
- return RustOcrModelPaths(
- detection: paths[0],
- classification: paths[1],
- recognition: paths[2],
- dictionary: paths[3],
+ Future<String> _getAssetsDirectory() async {
+ return _assetsDir ??= p.join(
+ (await getApplicationSupportDirectory()).path,
+ "assets",
);
}
- Future<void> _ensureEngine(RustOcrModelPaths paths) async {
- if (_engine != null && paths == _enginePaths) {
+ Future<void> _ensureEngine(
+ String assetsDir, {
+ required bool includeRecognizer,
+ }) async {
+ if (_engine != null && (!includeRecognizer || _includesRecognizer)) {
return;
}
try {
- _engine = await OcrEngine.create(paths: paths);
- _enginePaths = paths;
- final loadedModels = paths.recognition.isEmpty
- ? "detector only"
- : "detector, classifier and recognizer";
+ _engine = await OcrEngine.create(
+ assetsDir: assetsDir,
+ includeRecognizer: includeRecognizer,
+ );
+ _includesRecognizer = includeRecognizer;
+ final loadedModels = includeRecognizer
+ ? "detector, classifier and recognizer"
+ : "detector only";
_logger.info("Created Rust OCR engine ($loadedModels)");
} on RustOcrError catch (error) {
_logger.severe("Could not create the Rust OCR engine: $error");
@@ -139,29 +121,32 @@ class RustOcrBackend implements OcrBackend {
Future<OcrEngine> _recognitionEngine() async {
final engine = _engine;
- if (engine != null && _enginePaths.recognition.isNotEmpty) {
+ if (engine != null && _includesRecognizer) {
return engine;
}
await prepareModels(OcrModelComponent.values.toSet());
return _requireEngine();
}
- Future<OcrEngine> _detectionEngine() async {
- final engine = _engine;
- if (engine != null) {
- return engine;
- }
- final detectorAvailable = await RemoteAssetsService.instance.hasAsset(
- OcrDetectionModel.instance.modelRemotePath,
- );
- if (!detectorAvailable) {
- throw const OcrException(
- code: "MODEL_NOT_READY",
- message: "The OCR detector model is not available locally",
+ Future<OcrEngine> _detectionEngine() {
+ return _engineLock.synchronized(() async {
+ final engine = _engine;
+ if (engine != null) {
+ return engine;
+ }
+ final assetsDir = await _getAssetsDirectory();
+ final detectorAvailable = await OcrEngine.isDetectorDownloaded(
+ assetsDir: assetsDir,
);
- }
- await prepareModels({OcrModelComponent.detector});
- return _requireEngine();
+ if (!detectorAvailable) {
+ throw const OcrException(
+ code: "MODEL_NOT_READY",
+ message: "The OCR detector model is not available locally",
+ );
+ }
+ await _ensureEngine(assetsDir, includeRecognizer: false);
+ return _requireEngine();
+ });
}
OcrEngine _requireEngine() {
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 197478ad04..d700136e81 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2718,6 +2718,7 @@ dependencies = [
name = "ente-photos-frb"
version = "0.0.0"
dependencies = [
+ "ente-assets",
"ente-cast",
"ente-core",
"ente-frb-lib",
diff --git a/rust/bindings/frb/photos/Cargo.toml b/rust/bindings/frb/photos/Cargo.toml
index ba8afabf2c..8745430c33 100644
--- a/rust/bindings/frb/photos/Cargo.toml
+++ b/rust/bindings/frb/photos/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2024"
crate-type = ["cdylib", "staticlib"]
[dependencies]
+ente-assets.workspace = true
ente-cast.workspace = true
ente-core.workspace = true
ente-frb-lib = { workspace = true, features = ["contacts"] }
diff --git a/rust/bindings/frb/photos/src/api/ocr_api.rs b/rust/bindings/frb/photos/src/api/ocr_api.rs
index 64b32a5fc3..7b578ed7df 100644
--- a/rust/bindings/frb/photos/src/api/ocr_api.rs
+++ b/rust/bindings/frb/photos/src/api/ocr_api.rs
@@ -1,17 +1,10 @@
use std::path::Path;
+use ente_assets::AssetStore;
use ente_ml::error::MlError;
use ente_ml::ocr;
use flutter_rust_bridge::frb;
-#[derive(Clone, Debug)]
-pub struct RustOcrModelPaths {
- pub detection: String,
- pub classification: String,
- pub recognition: String,
- pub dictionary: String,
-}
-
#[derive(Clone, Copy, Debug)]
pub struct RustOcrPoint {
pub x: f64,
@@ -68,18 +61,29 @@ pub struct OcrEngine {
}
impl OcrEngine {
- pub fn create(paths: RustOcrModelPaths) -> Result<OcrEngine, RustOcrError> {
- ocr::OcrEngine::new(to_model_paths(paths))
- .map(|inner| OcrEngine { inner })
- .map_err(|error| {
- let error = RustOcrError::from(ocr::OcrError::Ml(error));
- log::log!(
- error.log_level(),
- "Rust OCR create failed: {}",
- error.description()
- );
- error
- })
+ pub async fn create(
+ assets_dir: String,
+ include_recognizer: bool,
+ ) -> Result<OcrEngine, RustOcrError> {
+ let store = AssetStore::new(assets_dir);
+ let result = async {
+ let paths = ocr::assets::ensure_models(&store, include_recognizer).await?;
+ ocr::OcrEngine::new(paths)
+ }
+ .await;
+ result.map(|inner| OcrEngine { inner }).map_err(|error| {
+ let error = RustOcrError::from(ocr::OcrError::Ml(error));
+ log::log!(
+ error.log_level(),
+ "Rust OCR create failed: {}",
+ error.description()
+ );
+ error
+ })
+ }
+
+ pub fn is_detector_downloaded(assets_dir: String) -> bool {
+ ocr::assets::is_detector_downloaded(&AssetStore::new(assets_dir))
}
pub fn detect_text(
@@ -183,15 +187,6 @@ fn image_file_name(image_path: &str) -> String {
.unwrap_or_default()
}
-fn to_model_paths(paths: RustOcrModelPaths) -> ocr::OcrModelPaths {
- ocr::OcrModelPaths {
- detection: paths.detection,
- classification: paths.classification,
- recognition: paths.recognition,
- dictionary: paths.dictionary,
- }
-}
-
fn to_api_points(points: &[ocr::Point; 4]) -> Vec<RustOcrPoint> {
points
.iter()
diff --git a/rust/crates/ml/examples/ocr.rs b/rust/crates/ml/examples/ocr.rs
index 849c434776..a109eeb590 100644
--- a/rust/crates/ml/examples/ocr.rs
+++ b/rust/crates/ml/examples/ocr.rs
@@ -51,7 +51,7 @@ async fn main() -> Result<()> {
install_logger();
load_onnx_runtime().await?;
let store = AssetStore::new(asset_cache_dir()?);
- let paths = assets::ensure_models(&store).await?;
+ let paths = assets::ensure_models(&store, !options.regions).await?;
run_with_large_stack("ocr", move || run(options, paths))
}
diff --git a/rust/crates/ml/src/ocr/assets.rs b/rust/crates/ml/src/ocr/assets.rs
index 7f84f646f6..dd065a46be 100644
--- a/rust/crates/ml/src/ocr/assets.rs
+++ b/rust/crates/ml/src/ocr/assets.rs
@@ -77,8 +77,20 @@ pub fn model_assets() -> Vec<Asset> {
CATALOG.iter().map(|file| file.asset()).collect()
}
-pub async fn ensure_models(store: &AssetStore) -> MlResult<OcrModelPaths> {
- let missing: Vec<(&OcrModelFile, Asset)> = CATALOG
+pub fn is_detector_downloaded(store: &AssetStore) -> bool {
+ store.is_downloaded(&DETECTION.asset())
+}
+
+pub async fn ensure_models(
+ store: &AssetStore,
+ include_recognizer: bool,
+) -> MlResult<OcrModelPaths> {
+ let files = if include_recognizer {
+ &CATALOG[..]
+ } else {
+ &CATALOG[..1]
+ };
+ let missing: Vec<(&OcrModelFile, Asset)> = files
.iter()
.map(|file| (*file, file.asset()))
.filter(|(_, asset)| !store.is_downloaded(asset))
@@ -108,18 +120,91 @@ pub async fn ensure_models(store: &AssetStore) -> MlResult<OcrModelPaths> {
}
Ok(OcrModelPaths {
detection: DETECTION.path(store),
- classification: CLASSIFICATION.path(store),
- recognition: RECOGNITION.path(store),
- dictionary: DICTIONARY.path(store),
+ classification: if include_recognizer {
+ CLASSIFICATION.path(store)
+ } else {
+ String::new()
+ },
+ recognition: if include_recognizer {
+ RECOGNITION.path(store)
+ } else {
+ String::new()
+ },
+ dictionary: if include_recognizer {
+ DICTIONARY.path(store)
+ } else {
+ String::new()
+ },
})
}
#[cfg(test)]
mod tests {
+ use std::fs;
use std::path::Path;
use super::*;
+ fn cache_model(store: &AssetStore, model: &OcrModelFile) {
+ let asset = model.asset();
+ fs::create_dir_all(store.asset_dir(&asset)).unwrap();
+ fs::write(model.path(store), b"cached model").unwrap();
+ }
+
+ #[test]
+ fn detector_availability_requires_the_published_file() {
+ let root = tempfile::tempdir().unwrap();
+ let store = AssetStore::new(root.path());
+ assert!(!is_detector_downloaded(&store));
+
+ fs::create_dir_all(store.asset_dir(&DETECTION.asset())).unwrap();
+ assert!(!is_detector_downloaded(&store));
+
+ cache_model(&store, &RECOGNITION);
+ assert!(!is_detector_downloaded(&store));
+
+ cache_model(&store, &DETECTION);
+ assert!(is_detector_downloaded(&store));
+
+ store.remove(&DETECTION.asset()).unwrap();
+ assert!(!is_detector_downloaded(&store));
+ }
+
+ #[tokio::test]
+ async fn cached_detector_can_be_prepared_without_recognition_models() {
+ let root = tempfile::tempdir().unwrap();
+ let store = AssetStore::new(root.path());
+ cache_model(&store, &DETECTION);
+
+ let paths = ensure_models(&store, false).await.unwrap();
+
+ assert_eq!(paths.detection, DETECTION.path(&store));
+ assert!(paths.classification.is_empty());
+ assert!(paths.recognition.is_empty());
+ assert!(paths.dictionary.is_empty());
+ for model in [&CLASSIFICATION, &RECOGNITION, &DICTIONARY] {
+ assert!(!store.is_downloaded(&model.asset()));
+ }
+ }
+
+ #[tokio::test]
+ async fn cached_full_models_can_be_prepared_after_detector_only() {
+ let root = tempfile::tempdir().unwrap();
+ let store = AssetStore::new(root.path());
+ for model in CATALOG {
+ cache_model(&store, model);
+ }
+
+ let detector_paths = ensure_models(&store, false).await.unwrap();
+ let paths = ensure_models(&store, true).await.unwrap();
+
+ assert_eq!(paths.detection, detector_paths.detection);
+ assert_eq!(paths.classification, CLASSIFICATION.path(&store));
+ assert_eq!(paths.recognition, RECOGNITION.path(&store));
+ assert_eq!(paths.dictionary, DICTIONARY.path(&store));
+ assert_eq!(fs::read(&paths.detection).unwrap(), b"cached model");
+ }
+
#[test]
fn catalog_directories_do_not_collide_with_the_indexing_catalog() {
let store = AssetStore::new(Path::new("cache"));