π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Preserve expected ML download errors and fix test sockets
laurenspriemc4a783e2bed5
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dartindex 2e53a26a10..fdf237e034 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart@@ -9,6 +9,7 @@ import "package:photos/db/ml/db.dart";import "package:photos/models/ml/vector.dart";import "package:photos/service_locator.dart" show isLocalGalleryMode;import "package:photos/services/machine_learning/ml_constants.dart";+import "package:photos/services/machine_learning/ml_exceptions.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";import "package:photos/services/machine_learning/semantic_search/query_result.dart";import "package:photos/services/machine_learning/webgpu_execution_policy.dart";@@ -94,6 +95,9 @@ class MLComputer extends SuperIsolate {if (isolateResult is RustCorruptModelException) {throw isolateResult;}+ if (isolateResult is ModelDownloadNetworkException) {+ throw isolateResult;+ }final textEmbedding = isolateResult as List<double>;return textEmbedding;} on WiFiUnavailableError catch (e, s) {diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart b/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dartindex 055553d580..794e93e700 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart@@ -1,5 +1,14 @@import "package:photos/core/exceptions.dart";+class ModelDownloadNetworkException implements Exception, LocallyHandledError {+ const ModelDownloadNetworkException(this.message);++ final String message;++ @override+ String toString() => "ModelDownloadNetworkException: $message";+}+class ThumbnailRetrievalException implements Exception {final String message;final StackTrace stackTrace;diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dartindex 74b61fd297..f65527cd3f 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart@@ -3,6 +3,7 @@ import "package:logging/logging.dart";import "package:photos/models/ml/ml_versions.dart"show mlIndexFlagCoreML, mlIndexFlagRuntimeRust, mlIndexFlagWebGPU;import "package:photos/service_locator.dart" show flagService, localSettings;+import "package:photos/services/machine_learning/ml_exceptions.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";import "package:photos/services/machine_learning/ml_result.dart";import "package:photos/services/machine_learning/webgpu_execution_policy.dart";@@ -68,6 +69,9 @@ class MLIndexingIsolate extends SuperIsolate {shouldPauseIndexingAndClustering = true;throw isolateResult;}+ if (isolateResult is ModelDownloadNetworkException) {+ throw isolateResult;+ }final resultJsonString = isolateResult as String?;if (resultJsonString == null) {if (!shouldPauseIndexingAndClustering) {@@ -103,10 +107,13 @@ class MLIndexingIsolate extends SuperIsolate {final frozenRuntimeArgs = Map<String, dynamic>.unmodifiable(rustRuntimeArgs,);- await runInIsolate(+ final result = await runInIsolate(IsolateOperation.prepareRustMlRuntime,frozenRuntimeArgs,);+ if (result is ModelDownloadNetworkException) {+ throw result;+ }_cachedRustRuntimeArgs = frozenRuntimeArgs;});}diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dartindex ebbfb5dbf4..453c3b48ce 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart@@ -6,6 +6,7 @@ import "package:path/path.dart" as p;import "package:path_provider/path_provider.dart";import "package:photos/service_locator.dart"show flagService, hasGrantedMLConsent, isLocalGalleryMode, localSettings;+import "package:photos/services/machine_learning/ml_exceptions.dart";import "package:photos/src/rust/api/ml_indexing_api.dart" as rust_ml;import "package:photos/utils/network_util.dart";import "package:synchronized/synchronized.dart";@@ -116,11 +117,20 @@ class MLModelDownloadService {runPets: downloadIndexing && _shouldDownloadPetModels,includeClipText: downloadText,)) {- _progressController.add((- update.model,- update.downloadedBytes.toInt(),- update.totalBytes?.toInt() ?? 0,- ));+ switch (update) {+ case rust_ml.ModelDownloadEvent_Progress():+ _progressController.add((+ update.model,+ update.downloadedBytes.toInt(),+ update.totalBytes?.toInt() ?? 0,+ ));+ case rust_ml.ModelDownloadEvent_Failed():+ final error = update.error;+ if (error is rust_ml.RustMlError_ModelDownloadNetwork) {+ throw ModelDownloadNetworkException(error.message);+ }+ throw error;+ }}} catch (e, s) {_logger.warning(diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dartindex a9c5051ce8..6f479072fe 100644--- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart+++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart@@ -10,6 +10,7 @@ import "package:photos/models/ml/face/box.dart";import "package:photos/models/ml/vector.dart";import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";import "package:photos/services/machine_learning/ml_constants.dart";+import "package:photos/services/machine_learning/ml_exceptions.dart";import "package:photos/services/machine_learning/ml_result.dart";import "package:photos/services/machine_learning/semantic_search/query_result.dart";import "package:photos/src/rust/api/image_processing_api.dart"@@ -100,12 +101,18 @@ Future<dynamic> isolateFunction(result = await analyzeImageRust(args);} on rust_ml.RustMlError_CorruptModel catch (e) {return RustCorruptModelException(e.message);+ } on ModelDownloadNetworkException catch (e) {+ return e;}return result.toJsonString();case IsolateOperation.prepareRustMlRuntime:await _ensureRustLoaded();- await _ensureRustRuntimePrepared(args);+ try {+ await _ensureRustRuntimePrepared(args);+ } on rust_ml.RustMlError_ModelDownloadNetwork catch (e) {+ return ModelDownloadNetworkException(e.message);+ }return true;case IsolateOperation.releaseRustMlRuntime:@@ -153,6 +160,8 @@ Future<dynamic> isolateFunction();} on rust_ml.RustMlError_CorruptModel catch (e) {return RustCorruptModelException(e.message);+ } on rust_ml.RustMlError_ModelDownloadNetwork catch (e) {+ return ModelDownloadNetworkException(e.message);}return List<double>.from(result.embedding, growable: false);diff --git a/mobile/apps/photos/lib/utils/ml_util.dart b/mobile/apps/photos/lib/utils/ml_util.dartindex 560f9581f3..2408bbde10 100644--- a/mobile/apps/photos/lib/utils/ml_util.dart+++ b/mobile/apps/photos/lib/utils/ml_util.dart@@ -822,6 +822,8 @@ Future<MLResult> analyzeImageRust(Map args) async {rust_ml.AnalyzeImageResult rustResult;try {rustResult = await runRustAnalyzeForPath(imagePath);+ } on rust_ml.RustMlError_ModelDownloadNetwork catch (e) {+ throw ModelDownloadNetworkException(e.message);} catch (e, s) {if (!_isRustImageIssue(e)) {if (_isRustCorruptModelIssue(e)) {@@ -950,7 +952,7 @@ Future<MLResult> analyzeImageRust(Map args) async {return result;} catch (e, s) {- if (isExpectedMlSkipError(e)) {+ if (e is ModelDownloadNetworkException || isExpectedMlSkipError(e)) {rethrow;}_logger.severe("Could not analyze image with Rust pipeline", e, s);diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rsindex 520400c6d1..2a161bcd10 100644--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs@@ -1,6 +1,6 @@use std::path::Path;-use ente_assets::AssetStore;+use ente_assets::{AssetStore, download::Error as DownloadError};use ente_ml::{assets, error::MlError, indexing, types};use flutter_rust_bridge::spawn_blocking_with;@@ -21,6 +21,7 @@ pub struct AnalyzeImageRequest {pub enum RustMlError {InvalidImage { message: String },CorruptModel { message: String },+ ModelDownloadNetwork { message: String },Other { message: String },}@@ -113,10 +114,15 @@ pub struct RunClipTextRequest {}#[derive(Clone, Debug)]-pub struct ModelDownloadProgress {- pub model: String,- pub downloaded_bytes: u64,- pub total_bytes: Option<u64>,+pub enum ModelDownloadEvent {+ Progress {+ model: String,+ downloaded_bytes: u64,+ total_bytes: Option<u64>,+ },+ Failed {+ error: RustMlError,+ },}#[derive(Clone, Debug)]@@ -216,7 +222,7 @@ pub async fn preload_ml_models(run_clip: bool,run_pets: bool,include_clip_text: bool,- progress: StreamSink<ModelDownloadProgress>,+ progress: StreamSink<ModelDownloadEvent>,) {let store = AssetStore::new(&assets_dir);let mut selected = assets::indexing_assets(run_faces, run_clip, run_pets);@@ -243,7 +249,7 @@ pub async fn preload_ml_models(}if let Err(error) =assets::ensure_mobile_models(&store, Path::new(&assets_dir), &selected, |update| {- let _ = progress.add(ModelDownloadProgress {+ let _ = progress.add(ModelDownloadEvent::Progress {model: labels[update.asset_index].to_string(),downloaded_bytes: update.asset_progress.downloaded_bytes,total_bytes: update.asset_progress.total_bytes,@@ -251,7 +257,9 @@ pub async fn preload_ml_models(}).await{- let _ = progress.add_error(error.to_string());+ let _ = progress.add(ModelDownloadEvent::Failed {+ error: error.into(),+ });}}@@ -283,6 +291,28 @@ impl From<MlError> for RustMlError {}}+impl From<DownloadError> for RustMlError {+ fn from(error: DownloadError) -> Self {+ let message = format!("model download failed: {error}");+ if is_expected_download_error(&error) {+ Self::ModelDownloadNetwork { message }+ } else {+ Self::Other { message }+ }+ }+}++fn is_expected_download_error(error: &DownloadError) -> bool {+ match error {+ DownloadError::Network(_) | DownloadError::Http(_) => true,+ DownloadError::Target { source, .. } => is_expected_download_error(source),+ DownloadError::Fallback { single, ranged } => {+ is_expected_download_error(single) && is_expected_download_error(ranged)+ }+ _ => false,+ }+}+fn to_api_analyze_image_result(result: indexing::AnalyzeImageResult) -> AnalyzeImageResult {AnalyzeImageResult {file_id: result.file_id,@@ -391,3 +421,94 @@ fn to_api_pet_body_result(result: types::PetBodyResult) -> RustPetBodyResult {.collect(),}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn network_and_http_download_failures_keep_their_classification() {+ for error in [+ DownloadError::Network("timed out".into()),+ DownloadError::Http(503),+ DownloadError::Target {+ label: "CLIP".into(),+ source: Box::new(DownloadError::Fallback {+ single: Box::new(DownloadError::Network("connection closed".into())),+ ranged: Box::new(DownloadError::Http(503)),+ }),+ },+ ] {+ let expected_message = format!("model download failed: {error}");+ let RustMlError::ModelDownloadNetwork { message } = RustMlError::from(error) else {+ panic!("expected a network download failure");+ };+ assert_eq!(message, expected_message);+ }+ }++ #[test]+ fn non_network_download_failures_remain_reportable() {+ for error in [+ DownloadError::Validation("checksum mismatch".into()),+ DownloadError::StorageFull,+ DownloadError::Io(std::io::Error::other("cannot import model")),+ DownloadError::Protocol("invalid range".into()),+ DownloadError::SizeMismatch {+ expected: 10,+ actual: 5,+ },+ DownloadError::InvalidTarget("invalid asset".into()),+ DownloadError::Cancelled,+ ] {+ let error = DownloadError::Target {+ label: "CLIP".into(),+ source: Box::new(error),+ };+ assert!(matches!(+ RustMlError::from(error),+ RustMlError::Other { .. }+ ));+ }+ }++ #[test]+ fn mixed_fallback_failures_remain_reportable() {+ for (single, ranged) in [+ (+ DownloadError::Validation("checksum mismatch".into()),+ DownloadError::Network("timed out".into()),+ ),+ (+ DownloadError::Network("timed out".into()),+ DownloadError::Protocol("invalid range".into()),+ ),+ ] {+ let error = DownloadError::Fallback {+ single: Box::new(single),+ ranged: Box::new(ranged),+ };+ assert!(matches!(+ RustMlError::from(error),+ RustMlError::Other { .. }+ ));+ }+ }++ #[test]+ fn native_runtime_and_inference_errors_remain_reportable() {+ for error in [+ MlError::Runtime("worker failed".into()),+ MlError::Ort("inference failed".into()),+ ] {+ assert!(matches!(+ RustMlError::from(error),+ RustMlError::Other { .. }+ ));+ }+ assert!(matches!(+ task_error("worker panicked"),+ RustMlError::Other { .. }+ ));+ }+}diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rsindex 88363f67d2..bffde907e4 100644--- a/rust/crates/assets/src/store.rs+++ b/rust/crates/assets/src/store.rs@@ -752,6 +752,7 @@ mod tests {}Err(error) => panic!("test server: {error}"),};+ stream.set_nonblocking(false).unwrap();stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap();diff --git a/rust/crates/ml/src/assets.rs b/rust/crates/ml/src/assets.rsindex ab80dea5d7..407697aac1 100644--- a/rust/crates/ml/src/assets.rs+++ b/rust/crates/ml/src/assets.rs@@ -2,10 +2,10 @@ use std::fs;use std::path::{Path, PathBuf};use ente_assets::{- Asset, AssetDownloadProgress, AssetFile, AssetStore, download::CancellationToken,+ Asset, AssetDownloadProgress, AssetFile, AssetStore,+ download::{CancellationToken, Error as DownloadError},};-use super::error::{MlError, MlResult};use super::models::{self, Model, ModelPaths};const MODELS: &str = "models";@@ -101,7 +101,7 @@ pub async fn ensure_indexing_models(run_faces: bool,run_clip: bool,run_pets: bool,-) -> MlResult<ModelPaths> {+) -> Result<ModelPaths, DownloadError> {ensure_mobile_models(store,legacy_dir,@@ -112,7 +112,10 @@ pub async fn ensure_indexing_models(Ok(indexing_model_paths(store, run_faces, run_clip, run_pets))}-pub async fn ensure_clip_text(store: &AssetStore, legacy_dir: &Path) -> MlResult<ClipTextPaths> {+pub async fn ensure_clip_text(+ store: &AssetStore,+ legacy_dir: &Path,+) -> Result<ClipTextPaths, DownloadError> {ensure_mobile_models(store, legacy_dir, &[clip_text_asset()], |_| {}).await?;Ok(clip_text_paths(store))}@@ -122,7 +125,7 @@ pub async fn ensure_mobile_models(legacy_dir: &Path,assets: &[Asset],on_progress: impl FnMut(AssetDownloadProgress) + Send,-) -> MlResult<()> {+) -> Result<(), DownloadError> {store.download_with_staged_import(assets,@@ -139,7 +142,6 @@ pub async fn ensure_mobile_models(},).await- .map_err(|error| MlError::Runtime(format!("model download failed: {error}")))}pub fn is_clip_text_available(store: &AssetStore, legacy_dir: &Path, include_vocab: bool) -> bool {