π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
simplify ML runtime event reporting
laurenspriem74fee987dd40
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dartindex 487752c703..bb10a77959 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart@@ -9,7 +9,6 @@ import "package:photos/models/ml/vector.dart";import "package:photos/services/machine_learning/ml_constants.dart";import "package:photos/services/machine_learning/ml_model_assets.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";-import "package:photos/services/machine_learning/ml_runtime_events.dart";import "package:photos/services/machine_learning/semantic_search/query_result.dart";import "package:photos/services/machine_learning/webgpu_execution_policy.dart";import "package:photos/services/remote_assets_service.dart";@@ -85,17 +84,12 @@ class MLComputer extends SuperIsolate {);}final enableWebGpu = await webGpuExecutionPolicy.isEligible();- final dynamic isolateResult;- try {- isolateResult = await runInIsolate(IsolateOperation.runClipText, {- "text": query,- "clipTextModelPath": modelPath,- "clipTextVocabPath": vocabPath,- "enableWebGpu": enableWebGpu,- });- } finally {- await logRustMlRuntimeEvents();- }+ final isolateResult = await runInIsolate(IsolateOperation.runClipText, {+ "text": query,+ "clipTextModelPath": modelPath,+ "clipTextVocabPath": vocabPath,+ "enableWebGpu": enableWebGpu,+ });if (isolateResult is RustCorruptModelCacheDeletedException) {_clipTextModelPath = null;MLModelDownloadService.instance.invalidateModelDownloadCache(diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dartindex 7c5a4f7a95..38aae475dc 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart@@ -1,11 +1,12 @@import "package:flutter/foundation.dart" show debugPrint, kDebugMode;import "package:logging/logging.dart";+import "package:photos/models/ml/ml_versions.dart"+ show mlIndexFlagCoreML, mlIndexFlagRuntimeRust, mlIndexFlagWebGPU;import "package:photos/service_locator.dart" show flagService, localSettings;import "package:photos/services/machine_learning/ml_model_assets.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";import "package:photos/services/machine_learning/ml_models_overview.dart";import "package:photos/services/machine_learning/ml_result.dart";-import "package:photos/services/machine_learning/ml_runtime_events.dart";import "package:photos/services/machine_learning/webgpu_execution_policy.dart";import "package:photos/services/remote_assets_service.dart";import "package:photos/utils/isolate/isolate_operations.dart";@@ -27,6 +28,7 @@ class MLIndexingIsolate extends SuperIsolate {final _rustRuntimeLock = Lock();Map<String, dynamic>? _cachedRustRuntimeArgs;+ final Set<int> _runtimeFlagCombinations = {};@overrideFuture<void> onDispose() => releaseRustRuntime();@@ -52,20 +54,15 @@ class MLIndexingIsolate extends SuperIsolate {final enableWebGpu = await webGpuExecutionPolicy.isEligible();_logger.info("Analyzing image ${instruction.fileKey} with Rust ML");- final dynamic isolateResult;- try {- isolateResult = await runInIsolate(IsolateOperation.analyzeImage, {- "enteFileID": instruction.fileKey,- "filePath": filePath,- "runFaces": instruction.shouldRunFaces,- "runClip": instruction.shouldRunClip,- "runPets": instruction.shouldRunPets,- ...rustRuntimeArgs,- "enableWebGpu": enableWebGpu,- });- } finally {- await logRustMlRuntimeEvents();- }+ final isolateResult = await runInIsolate(IsolateOperation.analyzeImage, {+ "enteFileID": instruction.fileKey,+ "filePath": filePath,+ "runFaces": instruction.shouldRunFaces,+ "runClip": instruction.shouldRunClip,+ "runPets": instruction.shouldRunPets,+ ...rustRuntimeArgs,+ "enableWebGpu": enableWebGpu,+ });if (isolateResult is RustCorruptModelCacheDeletedException) {_logger.warning("Deleted corrupt Rust ONNX model cache at ${isolateResult.modelPath}; "@@ -81,7 +78,9 @@ class MLIndexingIsolate extends SuperIsolate {}return null;}- return MLResult.fromJsonString(resultJsonString);+ final result = MLResult.fromJsonString(resultJsonString);+ _runtimeFlagCombinations.add(result.remoteFlags);+ return result;} catch (e, s) {if (e is RustCorruptModelCacheDeletedException ||isExpectedMlSkipError(e)) {@@ -103,6 +102,7 @@ class MLIndexingIsolate extends SuperIsolate {Future<void> prepareRustRuntime() {return _rustRuntimeLock.synchronized(() async {+ _runtimeFlagCombinations.clear();final rustRuntimeArgs = await _buildRustRuntimeArgs();final frozenRuntimeArgs = Map<String, dynamic>.unmodifiable(rustRuntimeArgs,@@ -116,29 +116,63 @@ class MLIndexingIsolate extends SuperIsolate {}Future<void> releaseRustRuntime() async {- final cachedRustRuntimeArgs = _cachedRustRuntimeArgs;- if (cachedRustRuntimeArgs == null) {- return;- }- if (!isIsolateSpawned) {- _cachedRustRuntimeArgs = null;- return;- }- return _rustRuntimeLock.synchronized(() async {- if (_cachedRustRuntimeArgs == null) {+ try {+ final cachedRustRuntimeArgs = _cachedRustRuntimeArgs;+ if (cachedRustRuntimeArgs == null) {return;}if (!isIsolateSpawned) {_cachedRustRuntimeArgs = null;return;}- try {- await runInIsolate(IsolateOperation.releaseRustMlRuntime, {});- _cachedRustRuntimeArgs = null;- } catch (e, s) {- _logger.warning("Could not release Rust runtime in isolate", e, s);- }- });+ await _rustRuntimeLock.synchronized(() async {+ if (_cachedRustRuntimeArgs == null) {+ return;+ }+ if (!isIsolateSpawned) {+ _cachedRustRuntimeArgs = null;+ return;+ }+ try {+ await runInIsolate(IsolateOperation.releaseRustMlRuntime, {});+ _cachedRustRuntimeArgs = null;+ } catch (e, s) {+ _logger.warning("Could not release Rust runtime in isolate", e, s);+ }+ });+ } finally {+ _logAndResetRuntimeSummary();+ }+ }++ void _logAndResetRuntimeSummary() {+ if (_runtimeFlagCombinations.isEmpty) {+ return;+ }+ final summaries =+ _runtimeFlagCombinations+ .map(_formatRuntimeFlags)+ .toList(growable: false)+ ..sort();+ _logger.info("Rust ML indexing runtime summary: ${summaries.join(', ')}");+ _runtimeFlagCombinations.clear();+ }++ String _formatRuntimeFlags(int flags) {+ if ((flags & mlIndexFlagRuntimeRust) == 0) {+ return "unknown ($flags)";+ }+ final acceleratedProviders = <String>[];+ if ((flags & mlIndexFlagCoreML) != 0) {+ acceleratedProviders.add("CoreML");+ }+ if ((flags & mlIndexFlagWebGPU) != 0) {+ acceleratedProviders.add("WebGPU");+ }+ if (acceleratedProviders.isEmpty) {+ return "Rust (no CoreML/WebGPU flag)";+ }+ return "Rust+${acceleratedProviders.join('+')}";}Future<void> cleanupLocalIndexingModels({bool delete = false}) async {diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_runtime_events.dart b/mobile/apps/photos/lib/services/machine_learning/ml_runtime_events.dartdeleted file mode 100644index e1fadfd5af..0000000000--- a/mobile/apps/photos/lib/services/machine_learning/ml_runtime_events.dart+++ /dev/null@@ -1,35 +0,0 @@-import "package:photos/utils/isolate/isolate_operations.dart";-import "package:photos/utils/isolate/super_isolate.dart";--/// Surfaces notable Rust ML runtime events (execution provider fallbacks,-/// golden self-test failures) in the app logs.-///-/// The Rust runtime degrades gracefully without failing the calling-/// operation, so these events are the only signal that a device is running-/// ML in a degraded or misbehaving configuration.-extension MlRuntimeEventLogging on SuperIsolate {- /// Drains the Rust runtime's process-wide event buffer and logs each event- /// at its severity. Call after ML operations; best-effort.- Future<void> logRustMlRuntimeEvents() async {- try {- final result = await runInIsolate(- IsolateOperation.takeMlRuntimeEvents,- {},- );- for (final event in (result as List?) ?? const []) {- final map = event as Map;- final message = "Rust ML runtime: ${map["message"]}";- switch (map["severity"]) {- case "severe":- logger.severe(message);- case "warning":- logger.warning(message);- default:- logger.info(message);- }- }- } catch (e, s) {- logger.warning("Failed to take Rust ML runtime events", e, s);- }- }-}diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dartindex b98f0491f9..efd8d62488 100644--- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart+++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart@@ -4,6 +4,7 @@ import 'dart:typed_data' show Float32List, Uint8List;import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Uint64List;import "package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart"show Int64List;+import "package:logging/logging.dart";import "package:ml_linalg/linalg.dart";import "package:photos/db/ml/clip_vector_db.dart";import "package:photos/models/ml/face/box.dart";@@ -22,6 +23,7 @@ import "package:photos/utils/ml_util.dart";final Map<String, dynamic> _isolateCache = {};const _rustLibLoadedCacheKey = "rustLibLoaded";const _rustMlModelPathsCacheKey = "rustMlModelPaths";+final _rustMlRuntimeLogger = Logger("RustMLRuntime");class RustCorruptModelCacheDeletedException implements Exception {const RustCorruptModelCacheDeletedException(this.modelPath);@@ -42,9 +44,6 @@ enum IsolateOperation {/// [MLIndexingIsolate]releaseRustMlRuntime,- /// [MLIndexingIsolate] and [MLComputer]- takeMlRuntimeEvents,-/// [MLComputer]generateFaceThumbnails,@@ -135,19 +134,13 @@ Future<dynamic> isolateFunction(/// MLIndexingIsolatecase IsolateOperation.releaseRustMlRuntime:- await _releaseRustRuntime();+ try {+ await _releaseRustRuntime();+ } finally {+ await _logRustMlRuntimeEvents();+ }return true;- /// MLIndexingIsolate and MLComputer- case IsolateOperation.takeMlRuntimeEvents:- await _ensureRustLoaded();- final events = await rust_ml.takeMlRuntimeEvents();- return events- .map(- (event) => {"severity": event.severity, "message": event.message},- )- .toList();-/// Cases for MLIndexingIsolate stop here/// Cases for MLComputer start here@@ -195,30 +188,34 @@ Future<dynamic> isolateFunction();}- // Configure execution behavior before the CLIP text session is- // created; the session is process-global and cannot be reconfigured- // once built.- await rust_ml.setMlExecutionConfig(- enableWebgpu: (args["enableWebGpu"] as bool?) ?? false,- );-- final rust_ml.RunClipTextResult result;try {- result = await rust_ml.runClipTextRust(- req: rust_ml.RunClipTextRequest(- text: text,- modelPath: clipTextModelPath,- vocabPath: clipTextVocabPath,- ),+ // Configure execution behavior before the CLIP text session is+ // created; the session is process-global and cannot be reconfigured+ // once built.+ await rust_ml.setMlExecutionConfig(+ enableWebgpu: (args["enableWebGpu"] as bool?) ?? false,);- } on rust_ml.RustMlError_CorruptModel catch (e) {- final file = File(e.field0);- if (await file.exists()) {- await file.delete();++ final rust_ml.RunClipTextResult result;+ try {+ result = await rust_ml.runClipTextRust(+ req: rust_ml.RunClipTextRequest(+ text: text,+ modelPath: clipTextModelPath,+ vocabPath: clipTextVocabPath,+ ),+ );+ } on rust_ml.RustMlError_CorruptModel catch (e) {+ final file = File(e.field0);+ if (await file.exists()) {+ await file.delete();+ }+ return RustCorruptModelCacheDeletedException(e.field0);}- return RustCorruptModelCacheDeletedException(e.field0);+ return List<double>.from(result.embedding, growable: false);+ } finally {+ await _logRustMlRuntimeEvents();}- return List<double>.from(result.embedding, growable: false);/// MLComputercase IsolateOperation.computeBulkSimilarities:@@ -459,6 +456,25 @@ Future<void> _releaseRustRuntime() async {_isolateCache.remove(_rustMlModelPathsCacheKey);}+Future<void> _logRustMlRuntimeEvents() async {+ try {+ final events = await rust_ml.takeMlRuntimeEvents();+ for (final event in events) {+ final message = "Rust ML runtime: ${event.message}";+ switch (event.severity) {+ case "severe":+ _rustMlRuntimeLogger.severe(message);+ case "warning":+ _rustMlRuntimeLogger.warning(message);+ default:+ _rustMlRuntimeLogger.info(message);+ }+ }+ } catch (e, s) {+ _rustMlRuntimeLogger.warning("Failed to take Rust ML runtime events", e, s);+ }+}+String _modelPathsCacheKey(rust_ml.RustModelPaths modelPaths) {return [modelPaths.faceDetection,diff --git a/rust/crates/photos/src/ml/events.rs b/rust/crates/photos/src/ml/events.rsindex a476004cc7..f9fae2fded 100644--- a/rust/crates/photos/src/ml/events.rs+++ b/rust/crates/photos/src/ml/events.rs@@ -45,6 +45,12 @@ pub(crate) fn record(severity: Severity, message: String) {let mut events = EVENTS.lock().unwrap_or_else(|poisoned| poisoned.into_inner());+ if events+ .iter()+ .any(|event| event.severity == severity && event.message == message)+ {+ return;+ }if events.len() >= MAX_BUFFERED_EVENTS {events.remove(0);}@@ -78,6 +84,12 @@ mod tests {assert_eq!(events[1].severity, Severity::Severe);assert!(take_events().is_empty());+ record(Severity::Warning, "duplicate".to_string());+ record(Severity::Warning, "duplicate".to_string());+ let events = take_events();+ assert_eq!(events.len(), 1);+ assert_eq!(events[0].message, "duplicate");+for index in 0..(MAX_BUFFERED_EVENTS + 3) {record(Severity::Info, format!("event {index}"));}