Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile][rust] Harden OCR preload cleanup and crash tracking

laurenspriem50dad1a008ad
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
index 7082bcc3a5..0a27bbb245 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
@@ -15,6 +15,7 @@ class RustOcrBackend implements OcrBackend {
static const _modelVersion = "pp-ocrv5-fixed-v1";
final _engineLock = Lock();
+ final _retiredEngines = <OcrEngine>[];
OcrEngine? _engine;
String? _assetsDir;
bool _includesRecognizer = false;
@@ -23,11 +24,18 @@ class RustOcrBackend implements OcrBackend {
Future<void> unloadModels() {
return _engineLock.synchronized(() async {
final engine = _engine;
- if (engine == null) return;
- await engine.unloadModels();
- _engine = null;
- _includesRecognizer = false;
- engine.dispose();
+ if (engine != null) {
+ _retiredEngines.add(engine);
+ _engine = null;
+ _includesRecognizer = false;
+ }
+ if (_retiredEngines.isEmpty) return;
+ while (_retiredEngines.isNotEmpty) {
+ final retiredEngine = _retiredEngines.last;
+ await retiredEngine.unloadModels();
+ retiredEngine.dispose();
+ _retiredEngines.removeLast();
+ }
_logger.info("Unloaded Rust OCR models");
});
}
@@ -72,7 +80,14 @@ class RustOcrBackend implements OcrBackend {
assetsDir: assetsDir,
includeRecognizer: includeRecognizer,
);
- await engine.loadModels();
+ try {
+ await engine.loadModels();
+ } catch (_) {
+ engine.dispose();
+ rethrow;
+ }
+ final previousEngine = _engine;
+ if (previousEngine != null) _retiredEngines.add(previousEngine);
_engine = engine;
_includesRecognizer = includeRecognizer;
final loadedModels = includeRecognizer
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
index e4270d8841..c2f0b594a7 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr_service.dart
@@ -158,10 +158,9 @@ class OcrService with WidgetsBindingObserver {
late final Future<void> preload;
preload = Future.sync(() => prepareModels()).then<void>((_) {}).catchError((
Object error,
- StackTrace stackTrace,
) {
if (identical(_modelPreload, preload)) _modelPreload = null;
- _logger.warning("Could not preload OCR models", error, stackTrace);
+ _logger.warning("Could not preload OCR models: $error");
});
_modelPreload = preload;
return preload;
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index 113d263403..05546116d5 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -198,8 +198,8 @@ impl OnnxSession {
}
fn defer_first_run_canary(&mut self) {
- if self.first_run_canary.is_some() {
- self.disarm_first_run_canary();
+ if let Some(canary) = self.first_run_canary.take() {
+ canary.defer();
self.deferred_first_run_canary = true;
}
}
@@ -770,6 +770,44 @@ mod tests {
assert!(!has_canary(&temp));
}
+ #[test]
+ fn preload_preserves_prior_failures_until_successful_inference() {
+ for previous_failures in 1..=2 {
+ let temp = tempfile::tempdir().unwrap();
+ let model = temp.path().join("model.onnx");
+ let mut session = OnnxSession::new(
+ model.to_str().unwrap(),
+ "scanner",
+ ExecutionMode::GpuPreferred,
+ );
+ for _ in 0..previous_failures {
+ drop(first_run_canary(&temp));
+ }
+ session.first_run_canary = Some(first_run_canary(&temp));
+
+ session.defer_first_run_canary();
+ session.defer_first_run_canary();
+
+ let canary = std::fs::read_dir(temp.path())
+ .unwrap()
+ .next()
+ .unwrap()
+ .unwrap()
+ .path();
+ assert_eq!(
+ std::fs::read_to_string(&canary).unwrap().trim(),
+ previous_failures.to_string()
+ );
+ session.arm_deferred_first_run_canary().unwrap();
+ assert_eq!(
+ std::fs::read_to_string(&canary).unwrap().trim(),
+ (previous_failures + 1).to_string()
+ );
+ session.disarm_first_run_canary();
+ assert!(!has_canary(&temp));
+ }
+ }
+
#[test]
fn failed_canary_rearm_remains_required_on_retry() {
let temp = tempfile::tempdir().unwrap();
diff --git a/rust/crates/ml/src/onnx/webgpu.rs b/rust/crates/ml/src/onnx/webgpu.rs
index ae685780c5..2b489d9758 100644
--- a/rust/crates/ml/src/onnx/webgpu.rs
+++ b/rust/crates/ml/src/onnx/webgpu.rs
@@ -1,8 +1,3 @@
-// Arm the durable canary before touching the driver and disarm it only after
-// session construction and warm-up. Three recorded failures quarantine the
-// model directory across restarts. The caller's model-slot lock serializes
-// access.
-
#[cfg(all(
not(target_os = "windows"),
any(target_os = "android", target_os = "linux", test)
@@ -19,9 +14,11 @@ use std::sync::atomic::{AtomicBool, Ordering};
test
))]
use std::{
+ collections::BTreeMap,
fs::{self, OpenOptions},
io::{self, Write},
path::{Path, PathBuf},
+ sync::{Mutex, PoisonError},
};
// Bump to retry quarantined devices after major ONNX Runtime or Dawn changes.
@@ -41,6 +38,14 @@ const CANARY_FILE_PREFIX: &str = ".ente-webgpu-canary-v1.";
))]
const MAX_CONSECUTIVE_FAILURES: u32 = 3;
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
+static ACTIVE_CANARIES: Mutex<BTreeMap<PathBuf, u32>> = Mutex::new(BTreeMap::new());
+
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
static WEBGPU_ENABLED: AtomicBool = AtomicBool::new(false);
@@ -149,6 +154,13 @@ pub(super) struct ArmedCanary {
test
))]
path: PathBuf,
+ #[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+ ))]
+ active: bool,
}
#[cfg(any(
@@ -165,12 +177,42 @@ pub(super) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<
)
})?;
let path = dir.join(format!("{CANARY_FILE_PREFIX}{model_namespace}"));
+ let mut active = ACTIVE_CANARIES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
let failures = read_failure_count(&path)?;
persist_failure_count(&path, failures.saturating_add(1))?;
- Ok(ArmedCanary { path })
+ *active.entry(path.clone()).or_default() += 1;
+ Ok(ArmedCanary { path, active: true })
}
impl ArmedCanary {
+ pub(super) fn defer(self) {
+ #[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+ ))]
+ {
+ let mut canary = self;
+ let mut active = ACTIVE_CANARIES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ canary.active = false;
+ retire_canary(&mut active, &canary.path);
+ let result = read_failure_count(&canary.path).and_then(|failures| {
+ update_failure_count(&canary.path, failures.saturating_sub(1))
+ });
+ if let Err(error) = result {
+ log::warn!(
+ "failed to defer WebGPU crash canary at '{}': {error}",
+ canary.path.display()
+ );
+ }
+ }
+ }
+
pub(super) fn disarm(self) {
#[cfg(any(
target_os = "android",
@@ -179,16 +221,71 @@ impl ArmedCanary {
test
))]
{
- if let Err(error) = remove_file_durably(&self.path) {
+ let mut canary = self;
+ let mut active = ACTIVE_CANARIES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ canary.active = false;
+ let remaining = retire_canary(&mut active, &canary.path);
+ if let Err(error) = update_failure_count(&canary.path, remaining) {
log::warn!(
"failed to disarm WebGPU crash canary at '{}': {error}",
- self.path.display()
+ canary.path.display()
);
}
}
}
}
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
+impl Drop for ArmedCanary {
+ fn drop(&mut self) {
+ if self.active {
+ let mut active = ACTIVE_CANARIES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ retire_canary(&mut active, &self.path);
+ }
+ }
+}
+
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
+fn retire_canary(active: &mut BTreeMap<PathBuf, u32>, path: &Path) -> u32 {
+ let Some(count) = active.get_mut(path) else {
+ return 0;
+ };
+ *count -= 1;
+ let remaining = *count;
+ if remaining == 0 {
+ active.remove(path);
+ }
+ remaining
+}
+
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
+fn update_failure_count(path: &Path, failures: u32) -> io::Result<()> {
+ if failures == 0 {
+ remove_file_durably(path)
+ } else {
+ persist_failure_count(path, failures)
+ }
+}
+
#[cfg(any(
target_os = "android",
target_os = "linux",
@@ -212,6 +309,9 @@ fn model_dir(model_path: &str) -> Option<PathBuf> {
test
))]
fn quarantined(dir: &Path) -> bool {
+ let _active = ACTIVE_CANARIES
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
let Ok(entries) = fs::read_dir(dir) else {
return true;
};
@@ -375,6 +475,171 @@ mod tests {
assert!(quarantined(dir.path()));
}
+ #[test]
+ fn deferred_preloads_preserve_failures_until_quarantine() {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+
+ for attempt in 1..=MAX_CONSECUTIVE_FAILURES {
+ assert!(!quarantined(dir.path()));
+ arm_canary(&model, "ocr-detection").unwrap().defer();
+ let canary = arm_canary(&model, "ocr-detection").unwrap();
+ assert_eq!(super::read_failure_count(&canary.path).unwrap(), attempt);
+ drop(canary);
+ }
+
+ assert!(quarantined(dir.path()));
+ }
+
+ #[test]
+ fn overlapping_success_and_deferral_clear_prior_failures() {
+ for prior_failures in 0..=2 {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ for _ in 0..prior_failures {
+ drop(arm_canary(&model, "ocr-detection").unwrap());
+ }
+ let inference = arm_canary(&model, "ocr-detection").unwrap();
+ let preload = arm_canary(&model, "ocr-detection").unwrap();
+ let path = preload.path.clone();
+
+ inference.disarm();
+
+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);
+ preload.defer();
+ assert!(!path.exists());
+ assert!(!quarantined(dir.path()));
+ }
+ }
+
+ #[test]
+ fn deferring_preload_preserves_a_newer_attempt() {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ let first = arm_canary(&model, "ocr-detection").unwrap();
+ let preload = arm_canary(&model, "ocr-detection").unwrap();
+ first.disarm();
+ let newer = arm_canary(&model, "ocr-detection").unwrap();
+ let path = newer.path.clone();
+
+ preload.defer();
+
+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);
+ let newest = arm_canary(&model, "ocr-detection").unwrap();
+ newer.disarm();
+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);
+ drop(newest);
+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);
+ }
+
+ #[test]
+ fn preload_deferral_preserves_a_later_inference_failure() {
+ for prior_failures in 0..=2 {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ for _ in 0..prior_failures {
+ drop(arm_canary(&model, "ocr-detection").unwrap());
+ }
+ let preload = arm_canary(&model, "ocr-detection").unwrap();
+ let inference = arm_canary(&model, "ocr-detection").unwrap();
+ let path = inference.path.clone();
+
+ preload.defer();
+ drop(inference);
+
+ assert_eq!(
+ super::read_failure_count(&path).unwrap(),
+ prior_failures + 1
+ );
+ assert_eq!(quarantined(dir.path()), prior_failures == 2);
+ }
+ }
+
+ #[test]
+ fn overlapping_deferrals_preserve_only_prior_failures() {
+ for prior_failures in 0..=2 {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ for _ in 0..prior_failures {
+ drop(arm_canary(&model, "ocr-detection").unwrap());
+ }
+ let first = arm_canary(&model, "ocr-detection").unwrap();
+ let second = arm_canary(&model, "ocr-detection").unwrap();
+ let path = first.path.clone();
+
+ first.defer();
+ second.defer();
+
+ assert_eq!(super::read_failure_count(&path).unwrap(), prior_failures);
+ assert!(!quarantined(dir.path()));
+ }
+ }
+
+ #[test]
+ fn successful_attempt_preserves_another_live_attempt() {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ let first = arm_canary(&model, "ocr-detection").unwrap();
+ let second = arm_canary(&model, "ocr-detection").unwrap();
+ let path = second.path.clone();
+
+ first.disarm();
+ drop(second);
+
+ assert_eq!(super::read_failure_count(&path).unwrap(), 1);
+ arm_canary(&model, "ocr-detection").unwrap().disarm();
+ assert!(!path.exists());
+ }
+
+ #[test]
+ fn failed_arm_persistence_does_not_register_an_active_attempt() {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ let namespace = "missing-directory/ocr-detection";
+ let path = dir
+ .path()
+ .join(format!("{}{namespace}", super::CANARY_FILE_PREFIX));
+
+ assert!(arm_canary(&model, namespace).is_err());
+
+ assert!(!super::ACTIVE_CANARIES.lock().unwrap().contains_key(&path));
+ }
+
+ #[test]
+ fn failed_deferral_retires_its_active_attempt() {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ let canary = arm_canary(&model, "ocr-detection").unwrap();
+ let path = canary.path.clone();
+ std::fs::write(&path, "invalid count").unwrap();
+
+ canary.defer();
+
+ assert!(quarantined(dir.path()));
+ assert!(!super::ACTIVE_CANARIES.lock().unwrap().contains_key(&path));
+ std::fs::remove_file(&path).unwrap();
+ arm_canary(&model, "ocr-detection").unwrap().disarm();
+ assert!(!path.exists());
+ }
+
+ #[test]
+ fn failed_disarm_retires_its_active_attempt() {
+ let dir = tempfile::tempdir().unwrap();
+ let model = model_path(dir.path());
+ let canary = arm_canary(&model, "ocr-detection").unwrap();
+ let path = canary.path.clone();
+ std::fs::remove_file(&path).unwrap();
+ std::fs::create_dir(&path).unwrap();
+
+ canary.disarm();
+
+ assert!(quarantined(dir.path()));
+ assert!(!super::ACTIVE_CANARIES.lock().unwrap().contains_key(&path));
+ std::fs::remove_dir(&path).unwrap();
+ arm_canary(&model, "ocr-detection").unwrap().disarm();
+ assert!(!path.exists());
+ }
+
#[test]
fn successful_attempt_resets_consecutive_failures() {
let dir = tempfile::tempdir().unwrap();