Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

simplify ML runtime event reporting

laurenspriem74fee987dd40
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
index 487752c703..bb10a77959 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
@@ -9,7 +9,6 @@ import "package:photos/models/ml/vector.dart";
import "package:photos/services/machine_learning/ml_constants.dart";
import "package:photos/services/machine_learning/ml_model_assets.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
-import "package:photos/services/machine_learning/ml_runtime_events.dart";
import "package:photos/services/machine_learning/semantic_search/query_result.dart";
import "package:photos/services/machine_learning/webgpu_execution_policy.dart";
import "package:photos/services/remote_assets_service.dart";
@@ -85,17 +84,12 @@ class MLComputer extends SuperIsolate {
);
}
final enableWebGpu = await webGpuExecutionPolicy.isEligible();
- final dynamic isolateResult;
- try {
- isolateResult = await runInIsolate(IsolateOperation.runClipText, {
- "text": query,
- "clipTextModelPath": modelPath,
- "clipTextVocabPath": vocabPath,
- "enableWebGpu": enableWebGpu,
- });
- } finally {
- await logRustMlRuntimeEvents();
- }
+ final isolateResult = await runInIsolate(IsolateOperation.runClipText, {
+ "text": query,
+ "clipTextModelPath": modelPath,
+ "clipTextVocabPath": vocabPath,
+ "enableWebGpu": enableWebGpu,
+ });
if (isolateResult is RustCorruptModelCacheDeletedException) {
_clipTextModelPath = null;
MLModelDownloadService.instance.invalidateModelDownloadCache(
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
index 7c5a4f7a95..38aae475dc 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
@@ -1,11 +1,12 @@
import "package:flutter/foundation.dart" show debugPrint, kDebugMode;
import "package:logging/logging.dart";
+import "package:photos/models/ml/ml_versions.dart"
+ show mlIndexFlagCoreML, mlIndexFlagRuntimeRust, mlIndexFlagWebGPU;
import "package:photos/service_locator.dart" show flagService, localSettings;
import "package:photos/services/machine_learning/ml_model_assets.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
import "package:photos/services/machine_learning/ml_models_overview.dart";
import "package:photos/services/machine_learning/ml_result.dart";
-import "package:photos/services/machine_learning/ml_runtime_events.dart";
import "package:photos/services/machine_learning/webgpu_execution_policy.dart";
import "package:photos/services/remote_assets_service.dart";
import "package:photos/utils/isolate/isolate_operations.dart";
@@ -27,6 +28,7 @@ class MLIndexingIsolate extends SuperIsolate {
final _rustRuntimeLock = Lock();
Map<String, dynamic>? _cachedRustRuntimeArgs;
+ final Set<int> _runtimeFlagCombinations = {};
@override
Future<void> onDispose() => releaseRustRuntime();
@@ -52,20 +54,15 @@ class MLIndexingIsolate extends SuperIsolate {
final enableWebGpu = await webGpuExecutionPolicy.isEligible();
_logger.info("Analyzing image ${instruction.fileKey} with Rust ML");
- final dynamic isolateResult;
- try {
- isolateResult = await runInIsolate(IsolateOperation.analyzeImage, {
- "enteFileID": instruction.fileKey,
- "filePath": filePath,
- "runFaces": instruction.shouldRunFaces,
- "runClip": instruction.shouldRunClip,
- "runPets": instruction.shouldRunPets,
- ...rustRuntimeArgs,
- "enableWebGpu": enableWebGpu,
- });
- } finally {
- await logRustMlRuntimeEvents();
- }
+ final isolateResult = await runInIsolate(IsolateOperation.analyzeImage, {
+ "enteFileID": instruction.fileKey,
+ "filePath": filePath,
+ "runFaces": instruction.shouldRunFaces,
+ "runClip": instruction.shouldRunClip,
+ "runPets": instruction.shouldRunPets,
+ ...rustRuntimeArgs,
+ "enableWebGpu": enableWebGpu,
+ });
if (isolateResult is RustCorruptModelCacheDeletedException) {
_logger.warning(
"Deleted corrupt Rust ONNX model cache at ${isolateResult.modelPath}; "
@@ -81,7 +78,9 @@ class MLIndexingIsolate extends SuperIsolate {
}
return null;
}
- return MLResult.fromJsonString(resultJsonString);
+ final result = MLResult.fromJsonString(resultJsonString);
+ _runtimeFlagCombinations.add(result.remoteFlags);
+ return result;
} catch (e, s) {
if (e is RustCorruptModelCacheDeletedException ||
isExpectedMlSkipError(e)) {
@@ -103,6 +102,7 @@ class MLIndexingIsolate extends SuperIsolate {
Future<void> prepareRustRuntime() {
return _rustRuntimeLock.synchronized(() async {
+ _runtimeFlagCombinations.clear();
final rustRuntimeArgs = await _buildRustRuntimeArgs();
final frozenRuntimeArgs = Map<String, dynamic>.unmodifiable(
rustRuntimeArgs,
@@ -116,29 +116,63 @@ class MLIndexingIsolate extends SuperIsolate {
}
Future<void> releaseRustRuntime() async {
- final cachedRustRuntimeArgs = _cachedRustRuntimeArgs;
- if (cachedRustRuntimeArgs == null) {
- return;
- }
- if (!isIsolateSpawned) {
- _cachedRustRuntimeArgs = null;
- return;
- }
- return _rustRuntimeLock.synchronized(() async {
- if (_cachedRustRuntimeArgs == null) {
+ try {
+ final cachedRustRuntimeArgs = _cachedRustRuntimeArgs;
+ if (cachedRustRuntimeArgs == null) {
return;
}
if (!isIsolateSpawned) {
_cachedRustRuntimeArgs = null;
return;
}
- try {
- await runInIsolate(IsolateOperation.releaseRustMlRuntime, {});
- _cachedRustRuntimeArgs = null;
- } catch (e, s) {
- _logger.warning("Could not release Rust runtime in isolate", e, s);
- }
- });
+ await _rustRuntimeLock.synchronized(() async {
+ if (_cachedRustRuntimeArgs == null) {
+ return;
+ }
+ if (!isIsolateSpawned) {
+ _cachedRustRuntimeArgs = null;
+ return;
+ }
+ try {
+ await runInIsolate(IsolateOperation.releaseRustMlRuntime, {});
+ _cachedRustRuntimeArgs = null;
+ } catch (e, s) {
+ _logger.warning("Could not release Rust runtime in isolate", e, s);
+ }
+ });
+ } finally {
+ _logAndResetRuntimeSummary();
+ }
+ }
+
+ void _logAndResetRuntimeSummary() {
+ if (_runtimeFlagCombinations.isEmpty) {
+ return;
+ }
+ final summaries =
+ _runtimeFlagCombinations
+ .map(_formatRuntimeFlags)
+ .toList(growable: false)
+ ..sort();
+ _logger.info("Rust ML indexing runtime summary: ${summaries.join(', ')}");
+ _runtimeFlagCombinations.clear();
+ }
+
+ String _formatRuntimeFlags(int flags) {
+ if ((flags & mlIndexFlagRuntimeRust) == 0) {
+ return "unknown ($flags)";
+ }
+ final acceleratedProviders = <String>[];
+ if ((flags & mlIndexFlagCoreML) != 0) {
+ acceleratedProviders.add("CoreML");
+ }
+ if ((flags & mlIndexFlagWebGPU) != 0) {
+ acceleratedProviders.add("WebGPU");
+ }
+ if (acceleratedProviders.isEmpty) {
+ return "Rust (no CoreML/WebGPU flag)";
+ }
+ return "Rust+${acceleratedProviders.join('+')}";
}
Future<void> cleanupLocalIndexingModels({bool delete = false}) async {
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_runtime_events.dart b/mobile/apps/photos/lib/services/machine_learning/ml_runtime_events.dart
deleted file mode 100644
index e1fadfd5af..0000000000
--- a/mobile/apps/photos/lib/services/machine_learning/ml_runtime_events.dart
+++ /dev/null
@@ -1,35 +0,0 @@
-import "package:photos/utils/isolate/isolate_operations.dart";
-import "package:photos/utils/isolate/super_isolate.dart";
-
-/// Surfaces notable Rust ML runtime events (execution provider fallbacks,
-/// golden self-test failures) in the app logs.
-///
-/// The Rust runtime degrades gracefully without failing the calling
-/// operation, so these events are the only signal that a device is running
-/// ML in a degraded or misbehaving configuration.
-extension MlRuntimeEventLogging on SuperIsolate {
- /// Drains the Rust runtime's process-wide event buffer and logs each event
- /// at its severity. Call after ML operations; best-effort.
- Future<void> logRustMlRuntimeEvents() async {
- try {
- final result = await runInIsolate(
- IsolateOperation.takeMlRuntimeEvents,
- {},
- );
- for (final event in (result as List?) ?? const []) {
- final map = event as Map;
- final message = "Rust ML runtime: ${map["message"]}";
- switch (map["severity"]) {
- case "severe":
- logger.severe(message);
- case "warning":
- logger.warning(message);
- default:
- logger.info(message);
- }
- }
- } catch (e, s) {
- logger.warning("Failed to take Rust ML runtime events", e, s);
- }
- }
-}
diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
index b98f0491f9..efd8d62488 100644
--- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
+++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
@@ -4,6 +4,7 @@ import 'dart:typed_data' show Float32List, Uint8List;
import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Uint64List;
import "package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart"
show Int64List;
+import "package:logging/logging.dart";
import "package:ml_linalg/linalg.dart";
import "package:photos/db/ml/clip_vector_db.dart";
import "package:photos/models/ml/face/box.dart";
@@ -22,6 +23,7 @@ import "package:photos/utils/ml_util.dart";
final Map<String, dynamic> _isolateCache = {};
const _rustLibLoadedCacheKey = "rustLibLoaded";
const _rustMlModelPathsCacheKey = "rustMlModelPaths";
+final _rustMlRuntimeLogger = Logger("RustMLRuntime");
class RustCorruptModelCacheDeletedException implements Exception {
const RustCorruptModelCacheDeletedException(this.modelPath);
@@ -42,9 +44,6 @@ enum IsolateOperation {
/// [MLIndexingIsolate]
releaseRustMlRuntime,
- /// [MLIndexingIsolate] and [MLComputer]
- takeMlRuntimeEvents,
-
/// [MLComputer]
generateFaceThumbnails,
@@ -135,19 +134,13 @@ Future<dynamic> isolateFunction(
/// MLIndexingIsolate
case IsolateOperation.releaseRustMlRuntime:
- await _releaseRustRuntime();
+ try {
+ await _releaseRustRuntime();
+ } finally {
+ await _logRustMlRuntimeEvents();
+ }
return true;
- /// MLIndexingIsolate and MLComputer
- case IsolateOperation.takeMlRuntimeEvents:
- await _ensureRustLoaded();
- final events = await rust_ml.takeMlRuntimeEvents();
- return events
- .map(
- (event) => {"severity": event.severity, "message": event.message},
- )
- .toList();
-
/// Cases for MLIndexingIsolate stop here
/// Cases for MLComputer start here
@@ -195,30 +188,34 @@ Future<dynamic> isolateFunction(
);
}
- // Configure execution behavior before the CLIP text session is
- // created; the session is process-global and cannot be reconfigured
- // once built.
- await rust_ml.setMlExecutionConfig(
- enableWebgpu: (args["enableWebGpu"] as bool?) ?? false,
- );
-
- final rust_ml.RunClipTextResult result;
try {
- result = await rust_ml.runClipTextRust(
- req: rust_ml.RunClipTextRequest(
- text: text,
- modelPath: clipTextModelPath,
- vocabPath: clipTextVocabPath,
- ),
+ // Configure execution behavior before the CLIP text session is
+ // created; the session is process-global and cannot be reconfigured
+ // once built.
+ await rust_ml.setMlExecutionConfig(
+ enableWebgpu: (args["enableWebGpu"] as bool?) ?? false,
);
- } on rust_ml.RustMlError_CorruptModel catch (e) {
- final file = File(e.field0);
- if (await file.exists()) {
- await file.delete();
+
+ final rust_ml.RunClipTextResult result;
+ try {
+ result = await rust_ml.runClipTextRust(
+ req: rust_ml.RunClipTextRequest(
+ text: text,
+ modelPath: clipTextModelPath,
+ vocabPath: clipTextVocabPath,
+ ),
+ );
+ } on rust_ml.RustMlError_CorruptModel catch (e) {
+ final file = File(e.field0);
+ if (await file.exists()) {
+ await file.delete();
+ }
+ return RustCorruptModelCacheDeletedException(e.field0);
}
- return RustCorruptModelCacheDeletedException(e.field0);
+ return List<double>.from(result.embedding, growable: false);
+ } finally {
+ await _logRustMlRuntimeEvents();
}
- return List<double>.from(result.embedding, growable: false);
/// MLComputer
case IsolateOperation.computeBulkSimilarities:
@@ -459,6 +456,25 @@ Future<void> _releaseRustRuntime() async {
_isolateCache.remove(_rustMlModelPathsCacheKey);
}
+Future<void> _logRustMlRuntimeEvents() async {
+ try {
+ final events = await rust_ml.takeMlRuntimeEvents();
+ for (final event in events) {
+ final message = "Rust ML runtime: ${event.message}";
+ switch (event.severity) {
+ case "severe":
+ _rustMlRuntimeLogger.severe(message);
+ case "warning":
+ _rustMlRuntimeLogger.warning(message);
+ default:
+ _rustMlRuntimeLogger.info(message);
+ }
+ }
+ } catch (e, s) {
+ _rustMlRuntimeLogger.warning("Failed to take Rust ML runtime events", e, s);
+ }
+}
+
String _modelPathsCacheKey(rust_ml.RustModelPaths modelPaths) {
return [
modelPaths.faceDetection,
diff --git a/rust/crates/photos/src/ml/events.rs b/rust/crates/photos/src/ml/events.rs
index a476004cc7..f9fae2fded 100644
--- a/rust/crates/photos/src/ml/events.rs
+++ b/rust/crates/photos/src/ml/events.rs
@@ -45,6 +45,12 @@ pub(crate) fn record(severity: Severity, message: String) {
let mut events = EVENTS
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
+ if events
+ .iter()
+ .any(|event| event.severity == severity && event.message == message)
+ {
+ return;
+ }
if events.len() >= MAX_BUFFERED_EVENTS {
events.remove(0);
}
@@ -78,6 +84,12 @@ mod tests {
assert_eq!(events[1].severity, Severity::Severe);
assert!(take_events().is_empty());
+ record(Severity::Warning, "duplicate".to_string());
+ record(Severity::Warning, "duplicate".to_string());
+ let events = take_events();
+ assert_eq!(events.len(), 1);
+ assert_eq!(events[0].message, "duplicate");
+
for index in 0..(MAX_BUFFERED_EVENTS + 3) {
record(Severity::Info, format!("event {index}"));
}