π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mobile] Unload idle Android OCR models
laurenspriem5f6ba80dd591
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dartindex 8904e2ba98..3eeb4b2629 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart@@ -5,6 +5,8 @@ abstract class OcrBackend {Set<OcrModelComponent> components,);+ Future<void> unloadModels();+Future<TextDetectionResult> detectText({required String imagePath,bool includeAllConfidenceScores = false,diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dartindex 005c45b5d2..43a9fe339c 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart@@ -19,6 +19,19 @@ class RustOcrBackend implements OcrBackend {String? _assetsDir;bool _includesRecognizer = false;+ @override+ Future<void> unloadModels() {+ return _engineLock.synchronized(() async {+ final engine = _engine;+ if (engine == null) return;+ await engine.unloadModels();+ _engine = null;+ _includesRecognizer = false;+ engine.dispose();+ _logger.info("Unloaded Rust OCR models");+ });+ }+@overrideFuture<ModelPreparationStatus> prepareModels(Set<OcrModelComponent> components,diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dartindex 4873fcb8a2..b7d0ddec71 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart@@ -15,6 +15,9 @@ class VisionOcrBackend implements OcrBackend {final VisionTextRecognizer _recognizer;+ @override+ Future<void> unloadModels() async {}+@overrideFuture<ModelPreparationStatus> prepareModels(Set<OcrModelComponent> components,diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dartindex ee80938fed..9cb2665efa 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart@@ -1,5 +1,7 @@+import "dart:async";import "dart:io";+import "package:flutter/widgets.dart";import "package:logging/logging.dart";import "package:photos/services/machine_learning/ocr/ocr_backend.dart";import "package:photos/services/machine_learning/ocr/ocr_models.dart";@@ -8,7 +10,7 @@ import "package:photos/services/machine_learning/ocr/vision_ocr_backend.dart";enum OcrBackendKind { rust, vision }-class OcrService {+class OcrService with WidgetsBindingObserver {OcrService({required bool isAndroid,required bool isIOS,@@ -22,9 +24,10 @@ class OcrService {static final instance = OcrService(isAndroid: Platform.isAndroid,isIOS: Platform.isIOS,- );+ ).._observeAppLifecycle();static final _logger = Logger("OcrService");+ static const _modelUnloadDelay = Duration(seconds: 30);final bool _isAndroid;final bool _isIOS;@@ -33,6 +36,99 @@ class OcrService {late final OcrBackendKind backendKind = _chooseBackendKind();late final OcrBackend _backend = _createBackend(backendKind);Future<void>? _modelPreload;+ Future<void>? _modelUnload;+ Timer? _modelUnloadTimer;+ int _viewerCount = 0;+ int _activeOperations = 0;+ bool _isBackgrounded = false;+ bool _preloadRequested = false;+ bool _unloadRequested = false;+ bool _modelsMayBeLoaded = false;++ void _observeAppLifecycle() {+ if (!_isAndroid) return;+ final binding = WidgetsBinding.instance;+ binding.addObserver(this);+ final state = binding.lifecycleState;+ if (state != null) didChangeAppLifecycleState(state);+ }++ void onViewerOpened() {+ if (!_isAndroid) return;+ _viewerCount++;+ _modelUnloadTimer?.cancel();+ _modelUnloadTimer = null;+ _unloadRequested = _isBackgrounded;+ }++ void onViewerClosed() {+ if (!_isAndroid) return;+ _viewerCount--;+ if (_viewerCount != 0) return;+ _preloadRequested = false;+ if (_isBackgrounded) {+ _requestModelUnload();+ } else {+ _modelUnloadTimer = Timer(_modelUnloadDelay, _requestModelUnload);+ }+ }++ @override+ void didChangeAppLifecycleState(AppLifecycleState state) {+ if (!_isAndroid) return;+ switch (state) {+ case AppLifecycleState.hidden:+ case AppLifecycleState.paused:+ case AppLifecycleState.detached:+ _isBackgrounded = true;+ _requestModelUnload();+ case AppLifecycleState.resumed:+ _isBackgrounded = false;+ if (_viewerCount > 0) {+ _unloadRequested = false;+ if (_preloadRequested) unawaited(preloadModels());+ }+ case AppLifecycleState.inactive:+ break;+ }+ }++ void _requestModelUnload() {+ _modelUnloadTimer?.cancel();+ _modelUnloadTimer = null;+ _unloadRequested = true;+ _unloadModelsIfIdle();+ }++ void _unloadModelsIfIdle() {+ if (!_unloadRequested ||+ !_modelsMayBeLoaded ||+ _activeOperations != 0 ||+ _modelUnload != null) {+ return;+ }+ _modelPreload = null;+ _modelsMayBeLoaded = false;+ _modelUnload = Future.sync(_backend.unloadModels)+ .catchError((Object error, StackTrace stackTrace) {+ _modelsMayBeLoaded = true;+ _logger.warning("Could not unload OCR models", error, stackTrace);+ })+ .whenComplete(() => _modelUnload = null);+ }++ Future<T> _withModels<T>(Future<T> Function() operation) async {+ if (!_isAndroid) return operation();+ _activeOperations++;+ try {+ await _modelUnload;+ _modelsMayBeLoaded = true;+ return await operation();+ } finally {+ _activeOperations--;+ _unloadModelsIfIdle();+ }+ }OcrBackendKind _chooseBackendKind() {final kind = _preferredBackendKind();@@ -52,19 +148,29 @@ class OcrService {Future<void> preloadModels() {if (!_isAndroid) return Future.value();- return _modelPreload ??= Future.sync(() => prepareModels())- .then<void>((_) {})- .catchError((Object error, StackTrace stackTrace) {- _modelPreload = null;- _logger.warning("Could not preload OCR models", error, stackTrace);- });+ _preloadRequested = true;+ if (_isBackgrounded) return Future.value();+ final pending = _modelPreload;+ if (pending != null) return pending;+ late final Future<void> preload;+ preload = Future.sync(() => prepareModels()).then<void>((_) {}).catchError((+ Object error,+ StackTrace stackTrace,+ ) {+ if (identical(_modelPreload, preload)) _modelPreload = null;+ _logger.warning("Could not preload OCR models", error, stackTrace);+ });+ _modelPreload = preload;+ return preload;}Future<ModelPreparationStatus> prepareModels({Set<OcrModelComponent>? components,}) {- return _backend.prepareModels(- components ?? OcrModelComponent.values.toSet(),+ return _withModels(+ () => _backend.prepareModels(+ components ?? OcrModelComponent.values.toSet(),+ ),);}@@ -74,10 +180,12 @@ class OcrService {String? requestId,}) async {_ensureImageExists(imagePath);- return _backend.detectText(- imagePath: imagePath,- includeAllConfidenceScores: includeAllConfidenceScores,- requestId: requestId,+ return _withModels(+ () => _backend.detectText(+ imagePath: imagePath,+ includeAllConfidenceScores: includeAllConfidenceScores,+ requestId: requestId,+ ),);}@@ -86,9 +194,11 @@ class OcrService {String? requestId,}) async {_ensureImageExists(imagePath);- return _backend.detectTextRegions(- imagePath: imagePath,- requestId: requestId,+ return _withModels(+ () => _backend.detectTextRegions(+ imagePath: imagePath,+ requestId: requestId,+ ),);}diff --git a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dartindex 5b8894d1e3..a3e79fd2a3 100644--- a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart+++ b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart@@ -99,6 +99,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {@overridevoid initState() {super.initState();+ _ocrService.onViewerOpened();widget.controller._attach(this);GestureBinding.instance.pointerRouter.addGlobalRoute(_handleGlobalPointerEvent,@@ -152,6 +153,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {@overridevoid dispose() {+ _ocrService.onViewerClosed();_routeAnimation?.removeStatusListener(_onRouteAnimationStatus);_cancelActiveRegionRequest();_regionRetryTimer?.cancel();diff --git a/rust/bindings/frb/photos/src/api/ocr_api.rs b/rust/bindings/frb/photos/src/api/ocr_api.rsindex 520b7e446d..abf1fee8de 100644--- a/rust/bindings/frb/photos/src/api/ocr_api.rs+++ b/rust/bindings/frb/photos/src/api/ocr_api.rs@@ -92,6 +92,10 @@ impl OcrEngine {.map_err(|error| RustOcrError::from(ocr::OcrError::Ml(error)))}+ pub fn unload_models(&self) {+ self.inner.unload_models();+ }+pub fn detect_text(&self,image_path: String,diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rsindex af0569593e..d217c9b110 100644--- a/rust/crates/ml/src/ocr/classify.rs+++ b/rust/crates/ml/src/ocr/classify.rs@@ -49,6 +49,13 @@ impl AngleClassifier {.load()}+ pub(crate) fn unload(&self) {+ self.session+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .unload();+ }+pub(crate) fn classify(&self, crops: &mut [ImageU8]) -> MlResult<Vec<AngleDecision>> {classify_in_batches(crops, |batch| self.score_batch(batch))}diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rsindex 9b8386502e..e7efd4a29b 100644--- a/rust/crates/ml/src/ocr/detect.rs+++ b/rust/crates/ml/src/ocr/detect.rs@@ -57,6 +57,13 @@ impl TextDetector {.load()}+ pub(crate) fn unload(&self) {+ self.session+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .unload();+ }+pub(crate) fn detect(&self, working: &ImageU8) -> MlResult<TextDetection> {let (input_width, input_height) = detector_input_size(working.width, working.height);let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex 8265aca4d4..06f139bf6e 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -172,6 +172,16 @@ impl OcrEngine {Ok(())}+ pub fn unload_models(&self) {+ self.detector.unload();+ if let Some(classifier) = &self.classifier {+ classifier.unload();+ }+ if let Some(recognizer) = &self.recognizer {+ recognizer.unload();+ }+ }+fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {match (&self.classifier, &self.recognizer) {(Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex 1dc014f13c..c1018122df 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -78,6 +78,13 @@ impl TextRecognizer {.load()}+ pub(crate) fn unload(&self) {+ self.session+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .unload();+ }+pub(crate) fn recognize(&self,crops: &[&ImageU8],diff --git a/rust/crates/ml/src/ocr/session.rs b/rust/crates/ml/src/ocr/session.rsindex 6b06c41757..6430ef367a 100644--- a/rust/crates/ml/src/ocr/session.rs+++ b/rust/crates/ml/src/ocr/session.rs@@ -37,6 +37,10 @@ impl OcrSession {self.session.load()}+ pub(super) fn unload(&mut self) {+ self.session.unload();+ }+pub(super) fn run<T>(&mut self,operation: impl FnMut(&mut SessionHandle) -> SessionRunResult<T>,diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rsindex a6420d2652..113d263403 100644--- a/rust/crates/ml/src/onnx.rs+++ b/rust/crates/ml/src/onnx.rs@@ -790,6 +790,27 @@ mod tests {}}+ #[test]+ fn unloading_preloaded_session_clears_deferred_canary() {+ let temp = tempfile::tempdir().unwrap();+ let model = temp.path().join("model.onnx");+ let mut session = OnnxSession::new(+ model.to_str().unwrap(),+ "ocr-detection-fixed-v1",+ ExecutionMode::GpuPreferred,+ );+ session.initialize_load_state();+ session.first_run_canary = Some(first_run_canary(&temp));+ session.defer_first_run_canary();++ session.unload();+ session.arm_deferred_first_run_canary().unwrap();++ assert!(!session.has_load_state());+ assert!(!session.deferred_first_run_canary);+ assert!(!has_canary(&temp));+ }+#[test]fn unload_preserves_model_identity_and_armed_canary() {let temp = tempfile::tempdir().unwrap();