π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mobile][rust] Harden OCR preload cleanup and crash tracking
laurenspriem50dad1a008ad
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dartindex 7082bcc3a5..0a27bbb245 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart@@ -15,6 +15,7 @@ class RustOcrBackend implements OcrBackend {static const _modelVersion = "pp-ocrv5-fixed-v1";final _engineLock = Lock();+ final _retiredEngines = <OcrEngine>[];OcrEngine? _engine;String? _assetsDir;bool _includesRecognizer = false;@@ -23,11 +24,18 @@ class RustOcrBackend implements OcrBackend {Future<void> unloadModels() {return _engineLock.synchronized(() async {final engine = _engine;- if (engine == null) return;- await engine.unloadModels();- _engine = null;- _includesRecognizer = false;- engine.dispose();+ if (engine != null) {+ _retiredEngines.add(engine);+ _engine = null;+ _includesRecognizer = false;+ }+ if (_retiredEngines.isEmpty) return;+ while (_retiredEngines.isNotEmpty) {+ final retiredEngine = _retiredEngines.last;+ await retiredEngine.unloadModels();+ retiredEngine.dispose();+ _retiredEngines.removeLast();+ }_logger.info("Unloaded Rust OCR models");});}@@ -72,7 +80,14 @@ class RustOcrBackend implements OcrBackend {assetsDir: assetsDir,includeRecognizer: includeRecognizer,);- await engine.loadModels();+ try {+ await engine.loadModels();+ } catch (_) {+ engine.dispose();+ rethrow;+ }+ final previousEngine = _engine;+ if (previousEngine != null) _retiredEngines.add(previousEngine);_engine = engine;_includesRecognizer = includeRecognizer;final loadedModels = includeRecognizerdiff --git a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dartindex e4270d8841..c2f0b594a7 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart@@ -158,10 +158,9 @@ class OcrService with WidgetsBindingObserver {late final Future<void> preload;preload = Future.sync(() => prepareModels()).then<void>((_) {}).catchError((Object error,- StackTrace stackTrace,) {if (identical(_modelPreload, preload)) _modelPreload = null;- _logger.warning("Could not preload OCR models", error, stackTrace);+ _logger.warning("Could not preload OCR models: $error");});_modelPreload = preload;return preload;diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rsindex 113d263403..05546116d5 100644--- a/rust/crates/ml/src/onnx.rs+++ b/rust/crates/ml/src/onnx.rs@@ -198,8 +198,8 @@ impl OnnxSession {}fn defer_first_run_canary(&mut self) {- if self.first_run_canary.is_some() {- self.disarm_first_run_canary();+ if let Some(canary) = self.first_run_canary.take() {+ canary.defer();self.deferred_first_run_canary = true;}}@@ -770,6 +770,44 @@ mod tests {assert!(!has_canary(&temp));}+ #[test]+ fn preload_preserves_prior_failures_until_successful_inference() {+ for previous_failures in 1..=2 {+ let temp = tempfile::tempdir().unwrap();+ let model = temp.path().join("model.onnx");+ let mut session = OnnxSession::new(+ model.to_str().unwrap(),+ "scanner",+ ExecutionMode::GpuPreferred,+ );+ for _ in 0..previous_failures {+ drop(first_run_canary(&temp));+ }+ session.first_run_canary = Some(first_run_canary(&temp));++ session.defer_first_run_canary();+ session.defer_first_run_canary();++ let canary = std::fs::read_dir(temp.path())+ .unwrap()+ .next()+ .unwrap()+ .unwrap()+ .path();+ assert_eq!(+ std::fs::read_to_string(&canary).unwrap().trim(),+ previous_failures.to_string()+ );+ session.arm_deferred_first_run_canary().unwrap();+ assert_eq!(+ std::fs::read_to_string(&canary).unwrap().trim(),+ (previous_failures + 1).to_string()+ );+ session.disarm_first_run_canary();+ assert!(!has_canary(&temp));+ }+ }+#[test]fn failed_canary_rearm_remains_required_on_retry() {let temp = tempfile::tempdir().unwrap();diff --git a/rust/crates/ml/src/onnx/webgpu.rs b/rust/crates/ml/src/onnx/webgpu.rsindex ae685780c5..2b489d9758 100644--- a/rust/crates/ml/src/onnx/webgpu.rs+++ b/rust/crates/ml/src/onnx/webgpu.rs@@ -1,8 +1,3 @@-// Arm the durable canary before touching the driver and disarm it only after-// session construction and warm-up. Three recorded failures quarantine the-// model directory across restarts. The caller's model-slot lock serializes-// access.-#[cfg(all(not(target_os = "windows"),any(target_os = "android", target_os = "linux", test)@@ -19,9 +14,11 @@ use std::sync::atomic::{AtomicBool, Ordering};test))]use std::{+ collections::BTreeMap,fs::{self, OpenOptions},io::{self, Write},path::{Path, PathBuf},+ sync::{Mutex, PoisonError},};// Bump to retry quarantined devices after major ONNX Runtime or Dawn changes.@@ -41,6 +38,14 @@ const CANARY_FILE_PREFIX: &str = ".ente-webgpu-canary-v1.";))]const MAX_CONSECUTIVE_FAILURES: u32 = 3;+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]+static ACTIVE_CANARIES: Mutex<BTreeMap<PathBuf, u32>> = Mutex::new(BTreeMap::new());+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]static WEBGPU_ENABLED: AtomicBool = AtomicBool::new(false);@@ -149,6 +154,13 @@ pub(super) struct ArmedCanary {test))]path: PathBuf,+ #[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+ ))]+ active: bool,}#[cfg(any(@@ -165,12 +177,42 @@ pub(super) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<)})?;let path = dir.join(format!("{CANARY_FILE_PREFIX}{model_namespace}"));+ let mut active = ACTIVE_CANARIES+ .lock()+ .unwrap_or_else(PoisonError::into_inner);let failures = read_failure_count(&path)?;persist_failure_count(&path, failures.saturating_add(1))?;- Ok(ArmedCanary { path })+ *active.entry(path.clone()).or_default() += 1;+ Ok(ArmedCanary { path, active: true })}impl ArmedCanary {+ pub(super) fn defer(self) {+ #[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+ ))]+ {+ let mut canary = self;+ let mut active = ACTIVE_CANARIES+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ canary.active = false;+ retire_canary(&mut active, &canary.path);+ let result = read_failure_count(&canary.path).and_then(|failures| {+ update_failure_count(&canary.path, failures.saturating_sub(1))+ });+ if let Err(error) = result {+ log::warn!(+ "failed to defer WebGPU crash canary at '{}': {error}",+ canary.path.display()+ );+ }+ }+ }+pub(super) fn disarm(self) {#[cfg(any(target_os = "android",@@ -179,16 +221,71 @@ impl ArmedCanary {test))]{- if let Err(error) = remove_file_durably(&self.path) {+ let mut canary = self;+ let mut active = ACTIVE_CANARIES+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ canary.active = false;+ let remaining = retire_canary(&mut active, &canary.path);+ if let Err(error) = update_failure_count(&canary.path, remaining) {log::warn!("failed to disarm WebGPU crash canary at '{}': {error}",- self.path.display()+ canary.path.display());}}}}+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]+impl Drop for ArmedCanary {+ fn drop(&mut self) {+ if self.active {+ let mut active = ACTIVE_CANARIES+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ retire_canary(&mut active, &self.path);+ }+ }+}++#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]+fn retire_canary(active: &mut BTreeMap<PathBuf, u32>, path: &Path) -> u32 {+ let Some(count) = active.get_mut(path) else {+ return 0;+ };+ *count -= 1;+ let remaining = *count;+ if remaining == 0 {+ active.remove(path);+ }+ remaining+}++#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]+fn update_failure_count(path: &Path, failures: u32) -> io::Result<()> {+ if failures == 0 {+ remove_file_durably(path)+ } else {+ persist_failure_count(path, failures)+ }+}+#[cfg(any(target_os = "android",target_os = "linux",@@ -212,6 +309,9 @@ fn model_dir(model_path: &str) -> Option<PathBuf> {test))]fn quarantined(dir: &Path) -> bool {+ let _active = ACTIVE_CANARIES+ .lock()+ .unwrap_or_else(PoisonError::into_inner);let Ok(entries) = fs::read_dir(dir) else {return true;};@@ -375,6 +475,171 @@ mod tests {assert!(quarantined(dir.path()));}+ #[test]+ fn deferred_preloads_preserve_failures_until_quarantine() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());++ for attempt in 1..=MAX_CONSECUTIVE_FAILURES {+ assert!(!quarantined(dir.path()));+ arm_canary(&model, "ocr-detection").unwrap().defer();+ let canary = arm_canary(&model, "ocr-detection").unwrap();+ assert_eq!(super::read_failure_count(&canary.path).unwrap(), attempt);+ drop(canary);+ }++ assert!(quarantined(dir.path()));+ }++ #[test]+ fn overlapping_success_and_deferral_clear_prior_failures() {+ for prior_failures in 0..=2 {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ for _ in 0..prior_failures {+ drop(arm_canary(&model, "ocr-detection").unwrap());+ }+ let inference = arm_canary(&model, "ocr-detection").unwrap();+ let preload = arm_canary(&model, "ocr-detection").unwrap();+ let path = preload.path.clone();++ inference.disarm();++ assert_eq!(super::read_failure_count(&path).unwrap(), 1);+ preload.defer();+ assert!(!path.exists());+ assert!(!quarantined(dir.path()));+ }+ }++ #[test]+ fn deferring_preload_preserves_a_newer_attempt() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ let first = arm_canary(&model, "ocr-detection").unwrap();+ let preload = arm_canary(&model, "ocr-detection").unwrap();+ first.disarm();+ let newer = arm_canary(&model, "ocr-detection").unwrap();+ let path = newer.path.clone();++ preload.defer();++ assert_eq!(super::read_failure_count(&path).unwrap(), 1);+ let newest = arm_canary(&model, "ocr-detection").unwrap();+ newer.disarm();+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);+ drop(newest);+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);+ }++ #[test]+ fn preload_deferral_preserves_a_later_inference_failure() {+ for prior_failures in 0..=2 {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ for _ in 0..prior_failures {+ drop(arm_canary(&model, "ocr-detection").unwrap());+ }+ let preload = arm_canary(&model, "ocr-detection").unwrap();+ let inference = arm_canary(&model, "ocr-detection").unwrap();+ let path = inference.path.clone();++ preload.defer();+ drop(inference);++ assert_eq!(+ super::read_failure_count(&path).unwrap(),+ prior_failures + 1+ );+ assert_eq!(quarantined(dir.path()), prior_failures == 2);+ }+ }++ #[test]+ fn overlapping_deferrals_preserve_only_prior_failures() {+ for prior_failures in 0..=2 {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ for _ in 0..prior_failures {+ drop(arm_canary(&model, "ocr-detection").unwrap());+ }+ let first = arm_canary(&model, "ocr-detection").unwrap();+ let second = arm_canary(&model, "ocr-detection").unwrap();+ let path = first.path.clone();++ first.defer();+ second.defer();++ assert_eq!(super::read_failure_count(&path).unwrap(), prior_failures);+ assert!(!quarantined(dir.path()));+ }+ }++ #[test]+ fn successful_attempt_preserves_another_live_attempt() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ let first = arm_canary(&model, "ocr-detection").unwrap();+ let second = arm_canary(&model, "ocr-detection").unwrap();+ let path = second.path.clone();++ first.disarm();+ drop(second);++ assert_eq!(super::read_failure_count(&path).unwrap(), 1);+ arm_canary(&model, "ocr-detection").unwrap().disarm();+ assert!(!path.exists());+ }++ #[test]+ fn failed_arm_persistence_does_not_register_an_active_attempt() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ let namespace = "missing-directory/ocr-detection";+ let path = dir+ .path()+ .join(format!("{}{namespace}", super::CANARY_FILE_PREFIX));++ assert!(arm_canary(&model, namespace).is_err());++ assert!(!super::ACTIVE_CANARIES.lock().unwrap().contains_key(&path));+ }++ #[test]+ fn failed_deferral_retires_its_active_attempt() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ let canary = arm_canary(&model, "ocr-detection").unwrap();+ let path = canary.path.clone();+ std::fs::write(&path, "invalid count").unwrap();++ canary.defer();++ assert!(quarantined(dir.path()));+ assert!(!super::ACTIVE_CANARIES.lock().unwrap().contains_key(&path));+ std::fs::remove_file(&path).unwrap();+ arm_canary(&model, "ocr-detection").unwrap().disarm();+ assert!(!path.exists());+ }++ #[test]+ fn failed_disarm_retires_its_active_attempt() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());+ let canary = arm_canary(&model, "ocr-detection").unwrap();+ let path = canary.path.clone();+ std::fs::remove_file(&path).unwrap();+ std::fs::create_dir(&path).unwrap();++ canary.disarm();++ assert!(quarantined(dir.path()));+ assert!(!super::ACTIVE_CANARIES.lock().unwrap().contains_key(&path));+ std::fs::remove_dir(&path).unwrap();+ arm_canary(&model, "ocr-detection").unwrap().disarm();+ assert!(!path.exists());+ }+#[test]fn successful_attempt_resets_consecutive_failures() {let dir = tempfile::tempdir().unwrap();