Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile] Preload Android OCR models when opening photos

laurenspriema3b3922459f5
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
index 32cf77a8a6..005c45b5d2 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
@@ -55,10 +55,12 @@ class RustOcrBackend implements OcrBackend {
await setMlExecutionConfig(
enableWebgpu: await webGpuExecutionPolicy.isEligible(),
);
- _engine = await OcrEngine.create(
+ final engine = await OcrEngine.create(
assetsDir: assetsDir,
includeRecognizer: includeRecognizer,
);
+ await engine.loadModels();
+ _engine = engine;
_includesRecognizer = includeRecognizer;
final loadedModels = includeRecognizer
? "detector, classifier and recognizer"
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
index 5f5357dfc0..ee80938fed 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
@@ -32,6 +32,7 @@ class OcrService {
final OcrBackend Function() _createVisionBackend;
late final OcrBackendKind backendKind = _chooseBackendKind();
late final OcrBackend _backend = _createBackend(backendKind);
+ Future<void>? _modelPreload;
OcrBackendKind _chooseBackendKind() {
final kind = _preferredBackendKind();
@@ -49,6 +50,16 @@ class OcrService {
throw UnsupportedError("OCR is only supported on Android and iOS");
}
+ Future<void> preloadModels() {
+ if (!_isAndroid) return Future.value();
+ return _modelPreload ??= Future.sync(() => prepareModels())
+ .then<void>((_) {})
+ .catchError((Object error, StackTrace stackTrace) {
+ _modelPreload = null;
+ _logger.warning("Could not preload OCR models", error, stackTrace);
+ });
+ }
+
Future<ModelPreparationStatus> prepareModels({
Set<OcrModelComponent>? components,
}) {
diff --git a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart
index 362231cc59..5b8894d1e3 100644
--- a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart
+++ b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart
@@ -94,6 +94,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {
String? _resolvedImageSizePath;
Size? _resolvedImageSize;
int _imageSizeRequestId = 0;
+ Animation<double>? _routeAnimation;
@override
void initState() {
@@ -105,6 +106,36 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {
_evaluateFile();
}
+ @override
+ void didChangeDependencies() {
+ super.didChangeDependencies();
+ final animation = ModalRoute.of(context)?.animation;
+ if (animation != _routeAnimation) {
+ _routeAnimation?.removeStatusListener(_onRouteAnimationStatus);
+ _routeAnimation = animation;
+ _routeAnimation?.addStatusListener(_onRouteAnimationStatus);
+ }
+ _scheduleModelPreload();
+ }
+
+ void _onRouteAnimationStatus(AnimationStatus status) {
+ if (status == AnimationStatus.completed) {
+ _scheduleModelPreload();
+ }
+ }
+
+ void _scheduleModelPreload() {
+ WidgetsBinding.instance.addPostFrameCallback((_) {
+ if (!mounted || !_isEligible) return;
+ if (ModalRoute.of(context)?.isCurrent == false) return;
+ final animation = _routeAnimation;
+ if (animation != null && animation.status != AnimationStatus.completed) {
+ return;
+ }
+ unawaited(_ocrService.preloadModels());
+ });
+ }
+
@override
void didUpdateWidget(covariant InlineTextDetection oldWidget) {
super.didUpdateWidget(oldWidget);
@@ -121,6 +152,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {
@override
void dispose() {
+ _routeAnimation?.removeStatusListener(_onRouteAnimationStatus);
_cancelActiveRegionRequest();
_regionRetryTimer?.cancel();
_zoomSettleTimer?.cancel();
@@ -217,6 +249,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {
_localFilePath = null;
_detectedRegions = null;
});
+ _scheduleModelPreload();
}
if (!_requiresRegionRouting) {
return;
diff --git a/rust/bindings/frb/photos/src/api/ocr_api.rs b/rust/bindings/frb/photos/src/api/ocr_api.rs
index 7b578ed7df..520b7e446d 100644
--- a/rust/bindings/frb/photos/src/api/ocr_api.rs
+++ b/rust/bindings/frb/photos/src/api/ocr_api.rs
@@ -86,6 +86,12 @@ impl OcrEngine {
ocr::assets::is_detector_downloaded(&AssetStore::new(assets_dir))
}
+ pub fn load_models(&self) -> Result<(), RustOcrError> {
+ self.inner
+ .load_models()
+ .map_err(|error| RustOcrError::from(ocr::OcrError::Ml(error)))
+ }
+
pub fn detect_text(
&self,
image_path: String,
diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rs
index c67372b6a0..af0569593e 100644
--- a/rust/crates/ml/src/ocr/classify.rs
+++ b/rust/crates/ml/src/ocr/classify.rs
@@ -42,6 +42,13 @@ impl AngleClassifier {
}
}
+ pub(crate) fn load(&self) -> MlResult<()> {
+ self.session
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .load()
+ }
+
pub(crate) fn classify(&self, crops: &mut [ImageU8]) -> MlResult<Vec<AngleDecision>> {
classify_in_batches(crops, |batch| self.score_batch(batch))
}
diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rs
index 0731629ca2..9b8386502e 100644
--- a/rust/crates/ml/src/ocr/detect.rs
+++ b/rust/crates/ml/src/ocr/detect.rs
@@ -50,6 +50,13 @@ impl TextDetector {
}
}
+ pub(crate) fn load(&self) -> MlResult<()> {
+ self.session
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .load()
+ }
+
pub(crate) fn detect(&self, working: &ImageU8) -> MlResult<TextDetection> {
let (input_width, input_height) = detector_input_size(working.width, working.height);
let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index 10ab3a4514..8265aca4d4 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -161,6 +161,17 @@ impl OcrEngine {
})
}
+ pub fn load_models(&self) -> MlResult<()> {
+ self.detector.load()?;
+ if let Some(classifier) = &self.classifier {
+ classifier.load()?;
+ }
+ if let Some(recognizer) = &self.recognizer {
+ recognizer.load()?;
+ }
+ Ok(())
+ }
+
fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {
match (&self.classifier, &self.recognizer) {
(Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index a823c02a51..1dc014f13c 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -70,6 +70,14 @@ impl TextRecognizer {
}
}
+ pub(crate) fn load(&self) -> MlResult<()> {
+ self.dictionary.entries()?;
+ self.session
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .load()
+ }
+
pub(crate) fn recognize(
&self,
crops: &[&ImageU8],
diff --git a/rust/crates/ml/src/ocr/session.rs b/rust/crates/ml/src/ocr/session.rs
index 4f82a1f44f..6b06c41757 100644
--- a/rust/crates/ml/src/ocr/session.rs
+++ b/rust/crates/ml/src/ocr/session.rs
@@ -33,6 +33,10 @@ impl OcrSession {
}
}
+ pub(super) fn load(&mut self) -> MlResult<()> {
+ self.session.load()
+ }
+
pub(super) fn run<T>(
&mut self,
operation: impl FnMut(&mut SessionHandle) -> SessionRunResult<T>,
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index 1ea0418ef5..a6420d2652 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -69,6 +69,7 @@ pub(crate) struct OnnxSession {
provider_plan: Option<ProviderPlan>,
session: Option<(Session, ExecutionProvider)>,
first_run_canary: Option<webgpu::ArmedCanary>,
+ deferred_first_run_canary: bool,
}
impl OnnxSession {
@@ -82,6 +83,7 @@ impl OnnxSession {
provider_plan: None,
session: None,
first_run_canary: None,
+ deferred_first_run_canary: false,
}
}
@@ -104,9 +106,16 @@ impl OnnxSession {
pub(crate) fn unload(&mut self) {
self.provider_plan = None;
self.session = None;
+ self.deferred_first_run_canary = false;
self.leave_first_run_canary_armed();
}
+ pub(crate) fn load(&mut self) -> MlResult<()> {
+ self.ensure_loaded()?;
+ self.defer_first_run_canary();
+ Ok(())
+ }
+
#[cfg(test)]
pub(crate) fn is_loaded(&self) -> bool {
self.session.is_some()
@@ -131,6 +140,12 @@ impl OnnxSession {
mut operation: impl FnMut(&mut Session) -> SessionRunResult<T>,
) -> MlResult<(T, ProviderUsage)> {
loop {
+ if let Err(error) = self.arm_deferred_first_run_canary() {
+ if self.retry_after_provider_failure(&error) {
+ continue;
+ }
+ return Err(error.into_ml_error());
+ }
let (session, execution_provider) = self.ensure_loaded()?;
match operation(session) {
Ok(value) => {
@@ -182,6 +197,33 @@ impl OnnxSession {
Ok((session, *provider))
}
+ fn defer_first_run_canary(&mut self) {
+ if self.first_run_canary.is_some() {
+ self.disarm_first_run_canary();
+ self.deferred_first_run_canary = true;
+ }
+ }
+
+ fn arm_deferred_first_run_canary(&mut self) -> SessionRunResult<()> {
+ #[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+ ))]
+ if self.deferred_first_run_canary {
+ self.first_run_canary = Some(
+ webgpu::arm_canary(&self.model_path, &self.model_namespace).map_err(|error| {
+ SessionRunError::retryable(MlError::Ort(format!(
+ "failed to arm WebGPU crash canary: {error}"
+ )))
+ })?,
+ );
+ self.deferred_first_run_canary = false;
+ }
+ Ok(())
+ }
+
fn disarm_first_run_canary(&mut self) {
if let Some(canary) = self.first_run_canary.take() {
canary.disarm();
@@ -203,6 +245,7 @@ impl OnnxSession {
}
self.session = None;
+ self.deferred_first_run_canary = false;
log::warn!(
"execution provider failed, retrying model with the next provider fallback: {error}"
);
@@ -686,6 +729,67 @@ mod tests {
assert!(has_canary(&temp));
}
+ #[test]
+ fn unused_preloaded_session_does_not_leave_a_crash_canary() {
+ let temp = tempfile::tempdir().unwrap();
+ let model = temp.path().join("model.onnx");
+ let mut session = OnnxSession::new(
+ model.to_str().unwrap(),
+ "scanner",
+ ExecutionMode::GpuPreferred,
+ );
+ session.first_run_canary = Some(first_run_canary(&temp));
+
+ session.defer_first_run_canary();
+ session.defer_first_run_canary();
+
+ assert!(session.deferred_first_run_canary);
+ assert!(!has_canary(&temp));
+ drop(session);
+ assert!(!has_canary(&temp));
+ }
+
+ #[test]
+ fn preloaded_session_rearms_canary_only_for_first_inference() {
+ let temp = tempfile::tempdir().unwrap();
+ let model = temp.path().join("model.onnx");
+ let mut session = OnnxSession::new(
+ model.to_str().unwrap(),
+ "scanner",
+ ExecutionMode::GpuPreferred,
+ );
+ session.first_run_canary = Some(first_run_canary(&temp));
+ session.defer_first_run_canary();
+
+ session.arm_deferred_first_run_canary().unwrap();
+
+ assert!(has_canary(&temp));
+ assert!(!session.deferred_first_run_canary);
+ session.disarm_first_run_canary();
+ session.arm_deferred_first_run_canary().unwrap();
+ assert!(!has_canary(&temp));
+ }
+
+ #[test]
+ fn failed_canary_rearm_remains_required_on_retry() {
+ let temp = tempfile::tempdir().unwrap();
+ let parent = temp.path().join("not-a-directory");
+ std::fs::write(&parent, []).unwrap();
+ let model = parent.join("model.onnx");
+ let mut session = OnnxSession::new(
+ model.to_str().unwrap(),
+ "scanner",
+ ExecutionMode::GpuPreferred,
+ );
+ session.deferred_first_run_canary = true;
+
+ for _ in 0..2 {
+ let error = session.arm_deferred_first_run_canary().unwrap_err();
+ assert!(error.is_retryable());
+ assert!(session.deferred_first_run_canary);
+ }
+ }
+
#[test]
fn unload_preserves_model_identity_and_armed_canary() {
let temp = tempfile::tempdir().unwrap();