π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mobile] Preload Android OCR models when opening photos
laurenspriema3b3922459f5
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dartindex 32cf77a8a6..005c45b5d2 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart@@ -55,10 +55,12 @@ class RustOcrBackend implements OcrBackend {await setMlExecutionConfig(enableWebgpu: await webGpuExecutionPolicy.isEligible(),);- _engine = await OcrEngine.create(+ final engine = await OcrEngine.create(assetsDir: assetsDir,includeRecognizer: includeRecognizer,);+ await engine.loadModels();+ _engine = engine;_includesRecognizer = includeRecognizer;final loadedModels = includeRecognizer? "detector, classifier and recognizer"diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dartindex 5f5357dfc0..ee80938fed 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart@@ -32,6 +32,7 @@ class OcrService {final OcrBackend Function() _createVisionBackend;late final OcrBackendKind backendKind = _chooseBackendKind();late final OcrBackend _backend = _createBackend(backendKind);+ Future<void>? _modelPreload;OcrBackendKind _chooseBackendKind() {final kind = _preferredBackendKind();@@ -49,6 +50,16 @@ class OcrService {throw UnsupportedError("OCR is only supported on Android and iOS");}+ Future<void> preloadModels() {+ if (!_isAndroid) return Future.value();+ return _modelPreload ??= Future.sync(() => prepareModels())+ .then<void>((_) {})+ .catchError((Object error, StackTrace stackTrace) {+ _modelPreload = null;+ _logger.warning("Could not preload OCR models", error, stackTrace);+ });+ }+Future<ModelPreparationStatus> prepareModels({Set<OcrModelComponent>? components,}) {diff --git a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dartindex 362231cc59..5b8894d1e3 100644--- a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart+++ b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart@@ -94,6 +94,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {String? _resolvedImageSizePath;Size? _resolvedImageSize;int _imageSizeRequestId = 0;+ Animation<double>? _routeAnimation;@overridevoid initState() {@@ -105,6 +106,36 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {_evaluateFile();}+ @override+ void didChangeDependencies() {+ super.didChangeDependencies();+ final animation = ModalRoute.of(context)?.animation;+ if (animation != _routeAnimation) {+ _routeAnimation?.removeStatusListener(_onRouteAnimationStatus);+ _routeAnimation = animation;+ _routeAnimation?.addStatusListener(_onRouteAnimationStatus);+ }+ _scheduleModelPreload();+ }++ void _onRouteAnimationStatus(AnimationStatus status) {+ if (status == AnimationStatus.completed) {+ _scheduleModelPreload();+ }+ }++ void _scheduleModelPreload() {+ WidgetsBinding.instance.addPostFrameCallback((_) {+ if (!mounted || !_isEligible) return;+ if (ModalRoute.of(context)?.isCurrent == false) return;+ final animation = _routeAnimation;+ if (animation != null && animation.status != AnimationStatus.completed) {+ return;+ }+ unawaited(_ocrService.preloadModels());+ });+ }+@overridevoid didUpdateWidget(covariant InlineTextDetection oldWidget) {super.didUpdateWidget(oldWidget);@@ -121,6 +152,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {@overridevoid dispose() {+ _routeAnimation?.removeStatusListener(_onRouteAnimationStatus);_cancelActiveRegionRequest();_regionRetryTimer?.cancel();_zoomSettleTimer?.cancel();@@ -217,6 +249,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {_localFilePath = null;_detectedRegions = null;});+ _scheduleModelPreload();}if (!_requiresRegionRouting) {return;diff --git a/rust/bindings/frb/photos/src/api/ocr_api.rs b/rust/bindings/frb/photos/src/api/ocr_api.rsindex 7b578ed7df..520b7e446d 100644--- a/rust/bindings/frb/photos/src/api/ocr_api.rs+++ b/rust/bindings/frb/photos/src/api/ocr_api.rs@@ -86,6 +86,12 @@ impl OcrEngine {ocr::assets::is_detector_downloaded(&AssetStore::new(assets_dir))}+ pub fn load_models(&self) -> Result<(), RustOcrError> {+ self.inner+ .load_models()+ .map_err(|error| RustOcrError::from(ocr::OcrError::Ml(error)))+ }+pub fn detect_text(&self,image_path: String,diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rsindex c67372b6a0..af0569593e 100644--- a/rust/crates/ml/src/ocr/classify.rs+++ b/rust/crates/ml/src/ocr/classify.rs@@ -42,6 +42,13 @@ impl AngleClassifier {}}+ pub(crate) fn load(&self) -> MlResult<()> {+ self.session+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .load()+ }+pub(crate) fn classify(&self, crops: &mut [ImageU8]) -> MlResult<Vec<AngleDecision>> {classify_in_batches(crops, |batch| self.score_batch(batch))}diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rsindex 0731629ca2..9b8386502e 100644--- a/rust/crates/ml/src/ocr/detect.rs+++ b/rust/crates/ml/src/ocr/detect.rs@@ -50,6 +50,13 @@ impl TextDetector {}}+ pub(crate) fn load(&self) -> MlResult<()> {+ self.session+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .load()+ }+pub(crate) fn detect(&self, working: &ImageU8) -> MlResult<TextDetection> {let (input_width, input_height) = detector_input_size(working.width, working.height);let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex 10ab3a4514..8265aca4d4 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -161,6 +161,17 @@ impl OcrEngine {})}+ pub fn load_models(&self) -> MlResult<()> {+ self.detector.load()?;+ if let Some(classifier) = &self.classifier {+ classifier.load()?;+ }+ if let Some(recognizer) = &self.recognizer {+ recognizer.load()?;+ }+ Ok(())+ }+fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {match (&self.classifier, &self.recognizer) {(Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex a823c02a51..1dc014f13c 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -70,6 +70,14 @@ impl TextRecognizer {}}+ pub(crate) fn load(&self) -> MlResult<()> {+ self.dictionary.entries()?;+ self.session+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .load()+ }+pub(crate) fn recognize(&self,crops: &[&ImageU8],diff --git a/rust/crates/ml/src/ocr/session.rs b/rust/crates/ml/src/ocr/session.rsindex 4f82a1f44f..6b06c41757 100644--- a/rust/crates/ml/src/ocr/session.rs+++ b/rust/crates/ml/src/ocr/session.rs@@ -33,6 +33,10 @@ impl OcrSession {}}+ pub(super) fn load(&mut self) -> MlResult<()> {+ self.session.load()+ }+pub(super) fn run<T>(&mut self,operation: impl FnMut(&mut SessionHandle) -> SessionRunResult<T>,diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rsindex 1ea0418ef5..a6420d2652 100644--- a/rust/crates/ml/src/onnx.rs+++ b/rust/crates/ml/src/onnx.rs@@ -69,6 +69,7 @@ pub(crate) struct OnnxSession {provider_plan: Option<ProviderPlan>,session: Option<(Session, ExecutionProvider)>,first_run_canary: Option<webgpu::ArmedCanary>,+ deferred_first_run_canary: bool,}impl OnnxSession {@@ -82,6 +83,7 @@ impl OnnxSession {provider_plan: None,session: None,first_run_canary: None,+ deferred_first_run_canary: false,}}@@ -104,9 +106,16 @@ impl OnnxSession {pub(crate) fn unload(&mut self) {self.provider_plan = None;self.session = None;+ self.deferred_first_run_canary = false;self.leave_first_run_canary_armed();}+ pub(crate) fn load(&mut self) -> MlResult<()> {+ self.ensure_loaded()?;+ self.defer_first_run_canary();+ Ok(())+ }+#[cfg(test)]pub(crate) fn is_loaded(&self) -> bool {self.session.is_some()@@ -131,6 +140,12 @@ impl OnnxSession {mut operation: impl FnMut(&mut Session) -> SessionRunResult<T>,) -> MlResult<(T, ProviderUsage)> {loop {+ if let Err(error) = self.arm_deferred_first_run_canary() {+ if self.retry_after_provider_failure(&error) {+ continue;+ }+ return Err(error.into_ml_error());+ }let (session, execution_provider) = self.ensure_loaded()?;match operation(session) {Ok(value) => {@@ -182,6 +197,33 @@ impl OnnxSession {Ok((session, *provider))}+ fn defer_first_run_canary(&mut self) {+ if self.first_run_canary.is_some() {+ self.disarm_first_run_canary();+ self.deferred_first_run_canary = true;+ }+ }++ fn arm_deferred_first_run_canary(&mut self) -> SessionRunResult<()> {+ #[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+ ))]+ if self.deferred_first_run_canary {+ self.first_run_canary = Some(+ webgpu::arm_canary(&self.model_path, &self.model_namespace).map_err(|error| {+ SessionRunError::retryable(MlError::Ort(format!(+ "failed to arm WebGPU crash canary: {error}"+ )))+ })?,+ );+ self.deferred_first_run_canary = false;+ }+ Ok(())+ }+fn disarm_first_run_canary(&mut self) {if let Some(canary) = self.first_run_canary.take() {canary.disarm();@@ -203,6 +245,7 @@ impl OnnxSession {}self.session = None;+ self.deferred_first_run_canary = false;log::warn!("execution provider failed, retrying model with the next provider fallback: {error}");@@ -686,6 +729,67 @@ mod tests {assert!(has_canary(&temp));}+ #[test]+ fn unused_preloaded_session_does_not_leave_a_crash_canary() {+ let temp = tempfile::tempdir().unwrap();+ let model = temp.path().join("model.onnx");+ let mut session = OnnxSession::new(+ model.to_str().unwrap(),+ "scanner",+ ExecutionMode::GpuPreferred,+ );+ session.first_run_canary = Some(first_run_canary(&temp));++ session.defer_first_run_canary();+ session.defer_first_run_canary();++ assert!(session.deferred_first_run_canary);+ assert!(!has_canary(&temp));+ drop(session);+ assert!(!has_canary(&temp));+ }++ #[test]+ fn preloaded_session_rearms_canary_only_for_first_inference() {+ let temp = tempfile::tempdir().unwrap();+ let model = temp.path().join("model.onnx");+ let mut session = OnnxSession::new(+ model.to_str().unwrap(),+ "scanner",+ ExecutionMode::GpuPreferred,+ );+ session.first_run_canary = Some(first_run_canary(&temp));+ session.defer_first_run_canary();++ session.arm_deferred_first_run_canary().unwrap();++ assert!(has_canary(&temp));+ assert!(!session.deferred_first_run_canary);+ session.disarm_first_run_canary();+ session.arm_deferred_first_run_canary().unwrap();+ assert!(!has_canary(&temp));+ }++ #[test]+ fn failed_canary_rearm_remains_required_on_retry() {+ let temp = tempfile::tempdir().unwrap();+ let parent = temp.path().join("not-a-directory");+ std::fs::write(&parent, []).unwrap();+ let model = parent.join("model.onnx");+ let mut session = OnnxSession::new(+ model.to_str().unwrap(),+ "scanner",+ ExecutionMode::GpuPreferred,+ );+ session.deferred_first_run_canary = true;++ for _ in 0..2 {+ let error = session.arm_deferred_first_run_canary().unwrap_err();+ assert!(error.is_retryable());+ assert!(session.deferred_first_run_canary);+ }+ }+#[test]fn unload_preserves_model_identity_and_armed_canary() {let temp = tempfile::tempdir().unwrap();