Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile] Unload idle Android OCR models

laurenspriem5f6ba80dd591
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart
index 8904e2ba98..3eeb4b2629 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/ocr_backend.dart
@@ -5,6 +5,8 @@ abstract class OcrBackend {
Set<OcrModelComponent> components,
);
+ Future<void> unloadModels();
+
Future<TextDetectionResult> detectText({
required String imagePath,
bool includeAllConfidenceScores = false,
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
index 005c45b5d2..43a9fe339c 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
@@ -19,6 +19,19 @@ class RustOcrBackend implements OcrBackend {
String? _assetsDir;
bool _includesRecognizer = false;
+ @override
+ Future<void> unloadModels() {
+ return _engineLock.synchronized(() async {
+ final engine = _engine;
+ if (engine == null) return;
+ await engine.unloadModels();
+ _engine = null;
+ _includesRecognizer = false;
+ engine.dispose();
+ _logger.info("Unloaded Rust OCR models");
+ });
+ }
+
@override
Future<ModelPreparationStatus> prepareModels(
Set<OcrModelComponent> components,
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart
index 4873fcb8a2..b7d0ddec71 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/vision_ocr_backend.dart
@@ -15,6 +15,9 @@ class VisionOcrBackend implements OcrBackend {
final VisionTextRecognizer _recognizer;
+ @override
+ Future<void> unloadModels() async {}
+
@override
Future<ModelPreparationStatus> prepareModels(
Set<OcrModelComponent> components,
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
index ee80938fed..9cb2665efa 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
@@ -1,5 +1,7 @@
+import "dart:async";
import "dart:io";
+import "package:flutter/widgets.dart";
import "package:logging/logging.dart";
import "package:photos/services/machine_learning/ocr/ocr_backend.dart";
import "package:photos/services/machine_learning/ocr/ocr_models.dart";
@@ -8,7 +10,7 @@ import "package:photos/services/machine_learning/ocr/vision_ocr_backend.dart";
enum OcrBackendKind { rust, vision }
-class OcrService {
+class OcrService with WidgetsBindingObserver {
OcrService({
required bool isAndroid,
required bool isIOS,
@@ -22,9 +24,10 @@ class OcrService {
static final instance = OcrService(
isAndroid: Platform.isAndroid,
isIOS: Platform.isIOS,
- );
+ ).._observeAppLifecycle();
static final _logger = Logger("OcrService");
+ static const _modelUnloadDelay = Duration(seconds: 30);
final bool _isAndroid;
final bool _isIOS;
@@ -33,6 +36,99 @@ class OcrService {
late final OcrBackendKind backendKind = _chooseBackendKind();
late final OcrBackend _backend = _createBackend(backendKind);
Future<void>? _modelPreload;
+ Future<void>? _modelUnload;
+ Timer? _modelUnloadTimer;
+ int _viewerCount = 0;
+ int _activeOperations = 0;
+ bool _isBackgrounded = false;
+ bool _preloadRequested = false;
+ bool _unloadRequested = false;
+ bool _modelsMayBeLoaded = false;
+
+ void _observeAppLifecycle() {
+ if (!_isAndroid) return;
+ final binding = WidgetsBinding.instance;
+ binding.addObserver(this);
+ final state = binding.lifecycleState;
+ if (state != null) didChangeAppLifecycleState(state);
+ }
+
+ void onViewerOpened() {
+ if (!_isAndroid) return;
+ _viewerCount++;
+ _modelUnloadTimer?.cancel();
+ _modelUnloadTimer = null;
+ _unloadRequested = _isBackgrounded;
+ }
+
+ void onViewerClosed() {
+ if (!_isAndroid) return;
+ _viewerCount--;
+ if (_viewerCount != 0) return;
+ _preloadRequested = false;
+ if (_isBackgrounded) {
+ _requestModelUnload();
+ } else {
+ _modelUnloadTimer = Timer(_modelUnloadDelay, _requestModelUnload);
+ }
+ }
+
+ @override
+ void didChangeAppLifecycleState(AppLifecycleState state) {
+ if (!_isAndroid) return;
+ switch (state) {
+ case AppLifecycleState.hidden:
+ case AppLifecycleState.paused:
+ case AppLifecycleState.detached:
+ _isBackgrounded = true;
+ _requestModelUnload();
+ case AppLifecycleState.resumed:
+ _isBackgrounded = false;
+ if (_viewerCount > 0) {
+ _unloadRequested = false;
+ if (_preloadRequested) unawaited(preloadModels());
+ }
+ case AppLifecycleState.inactive:
+ break;
+ }
+ }
+
+ void _requestModelUnload() {
+ _modelUnloadTimer?.cancel();
+ _modelUnloadTimer = null;
+ _unloadRequested = true;
+ _unloadModelsIfIdle();
+ }
+
+ void _unloadModelsIfIdle() {
+ if (!_unloadRequested ||
+ !_modelsMayBeLoaded ||
+ _activeOperations != 0 ||
+ _modelUnload != null) {
+ return;
+ }
+ _modelPreload = null;
+ _modelsMayBeLoaded = false;
+ _modelUnload = Future.sync(_backend.unloadModels)
+ .catchError((Object error, StackTrace stackTrace) {
+ _modelsMayBeLoaded = true;
+ _logger.warning("Could not unload OCR models", error, stackTrace);
+ })
+ .whenComplete(() => _modelUnload = null);
+ }
+
+ Future<T> _withModels<T>(Future<T> Function() operation) async {
+ if (!_isAndroid) return operation();
+ _activeOperations++;
+ try {
+ await _modelUnload;
+ _modelsMayBeLoaded = true;
+ return await operation();
+ } finally {
+ _activeOperations--;
+ _unloadModelsIfIdle();
+ }
+ }
OcrBackendKind _chooseBackendKind() {
final kind = _preferredBackendKind();
@@ -52,19 +148,29 @@ class OcrService {
Future<void> preloadModels() {
if (!_isAndroid) return Future.value();
- return _modelPreload ??= Future.sync(() => prepareModels())
- .then<void>((_) {})
- .catchError((Object error, StackTrace stackTrace) {
- _modelPreload = null;
- _logger.warning("Could not preload OCR models", error, stackTrace);
- });
+ _preloadRequested = true;
+ if (_isBackgrounded) return Future.value();
+ final pending = _modelPreload;
+ if (pending != null) return pending;
+ late final Future<void> preload;
+ preload = Future.sync(() => prepareModels()).then<void>((_) {}).catchError((
+ Object error,
+ StackTrace stackTrace,
+ ) {
+ if (identical(_modelPreload, preload)) _modelPreload = null;
+ _logger.warning("Could not preload OCR models", error, stackTrace);
+ });
+ _modelPreload = preload;
+ return preload;
}
Future<ModelPreparationStatus> prepareModels({
Set<OcrModelComponent>? components,
}) {
- return _backend.prepareModels(
- components ?? OcrModelComponent.values.toSet(),
+ return _withModels(
+ () => _backend.prepareModels(
+ components ?? OcrModelComponent.values.toSet(),
+ ),
);
}
@@ -74,10 +180,12 @@ class OcrService {
String? requestId,
}) async {
_ensureImageExists(imagePath);
- return _backend.detectText(
- imagePath: imagePath,
- includeAllConfidenceScores: includeAllConfidenceScores,
- requestId: requestId,
+ return _withModels(
+ () => _backend.detectText(
+ imagePath: imagePath,
+ includeAllConfidenceScores: includeAllConfidenceScores,
+ requestId: requestId,
+ ),
);
}
@@ -86,9 +194,11 @@ class OcrService {
String? requestId,
}) async {
_ensureImageExists(imagePath);
- return _backend.detectTextRegions(
- imagePath: imagePath,
- requestId: requestId,
+ return _withModels(
+ () => _backend.detectTextRegions(
+ imagePath: imagePath,
+ requestId: requestId,
+ ),
);
}
diff --git a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart
index 5b8894d1e3..a3e79fd2a3 100644
--- a/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart
+++ b/mobile/apps/photos/lib/ui/viewer/file/ocr/inline_text_detection.dart
@@ -99,6 +99,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {
@override
void initState() {
super.initState();
+ _ocrService.onViewerOpened();
widget.controller._attach(this);
GestureBinding.instance.pointerRouter.addGlobalRoute(
_handleGlobalPointerEvent,
@@ -152,6 +153,7 @@ class _InlineTextDetectionState extends State<InlineTextDetection> {
@override
void dispose() {
+ _ocrService.onViewerClosed();
_routeAnimation?.removeStatusListener(_onRouteAnimationStatus);
_cancelActiveRegionRequest();
_regionRetryTimer?.cancel();
diff --git a/rust/bindings/frb/photos/src/api/ocr_api.rs b/rust/bindings/frb/photos/src/api/ocr_api.rs
index 520b7e446d..abf1fee8de 100644
--- a/rust/bindings/frb/photos/src/api/ocr_api.rs
+++ b/rust/bindings/frb/photos/src/api/ocr_api.rs
@@ -92,6 +92,10 @@ impl OcrEngine {
.map_err(|error| RustOcrError::from(ocr::OcrError::Ml(error)))
}
+ pub fn unload_models(&self) {
+ self.inner.unload_models();
+ }
+
pub fn detect_text(
&self,
image_path: String,
diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rs
index af0569593e..d217c9b110 100644
--- a/rust/crates/ml/src/ocr/classify.rs
+++ b/rust/crates/ml/src/ocr/classify.rs
@@ -49,6 +49,13 @@ impl AngleClassifier {
.load()
}
+ pub(crate) fn unload(&self) {
+ self.session
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .unload();
+ }
+
pub(crate) fn classify(&self, crops: &mut [ImageU8]) -> MlResult<Vec<AngleDecision>> {
classify_in_batches(crops, |batch| self.score_batch(batch))
}
diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rs
index 9b8386502e..e7efd4a29b 100644
--- a/rust/crates/ml/src/ocr/detect.rs
+++ b/rust/crates/ml/src/ocr/detect.rs
@@ -57,6 +57,13 @@ impl TextDetector {
.load()
}
+ pub(crate) fn unload(&self) {
+ self.session
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .unload();
+ }
+
pub(crate) fn detect(&self, working: &ImageU8) -> MlResult<TextDetection> {
let (input_width, input_height) = detector_input_size(working.width, working.height);
let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index 8265aca4d4..06f139bf6e 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -172,6 +172,16 @@ impl OcrEngine {
Ok(())
}
+ pub fn unload_models(&self) {
+ self.detector.unload();
+ if let Some(classifier) = &self.classifier {
+ classifier.unload();
+ }
+ if let Some(recognizer) = &self.recognizer {
+ recognizer.unload();
+ }
+ }
+
fn text_models(&self) -> MlResult<(&AngleClassifier, &TextRecognizer)> {
match (&self.classifier, &self.recognizer) {
(Some(classifier), Some(recognizer)) if self.missing_text_model_paths.is_empty() => {
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index 1dc014f13c..c1018122df 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -78,6 +78,13 @@ impl TextRecognizer {
.load()
}
+ pub(crate) fn unload(&self) {
+ self.session
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .unload();
+ }
+
pub(crate) fn recognize(
&self,
crops: &[&ImageU8],
diff --git a/rust/crates/ml/src/ocr/session.rs b/rust/crates/ml/src/ocr/session.rs
index 6b06c41757..6430ef367a 100644
--- a/rust/crates/ml/src/ocr/session.rs
+++ b/rust/crates/ml/src/ocr/session.rs
@@ -37,6 +37,10 @@ impl OcrSession {
self.session.load()
}
+ pub(super) fn unload(&mut self) {
+ self.session.unload();
+ }
+
pub(super) fn run<T>(
&mut self,
operation: impl FnMut(&mut SessionHandle) -> SessionRunResult<T>,
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index a6420d2652..113d263403 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -790,6 +790,27 @@ mod tests {
}
}
+ #[test]
+ fn unloading_preloaded_session_clears_deferred_canary() {
+ let temp = tempfile::tempdir().unwrap();
+ let model = temp.path().join("model.onnx");
+ let mut session = OnnxSession::new(
+ model.to_str().unwrap(),
+ "ocr-detection-fixed-v1",
+ ExecutionMode::GpuPreferred,
+ );
+ session.initialize_load_state();
+ session.first_run_canary = Some(first_run_canary(&temp));
+ session.defer_first_run_canary();
+
+ session.unload();
+ session.arm_deferred_first_run_canary().unwrap();
+
+ assert!(!session.has_load_state());
+ assert!(!session.deferred_first_run_canary);
+ assert!(!has_canary(&temp));
+ }
+
#[test]
fn unload_preserves_model_identity_and_armed_canary() {
let temp = tempfile::tempdir().unwrap();