Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Preserve expected ML download errors and fix test sockets

laurenspriemc4a783e2bed5
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
index 2e53a26a10..fdf237e034 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
@@ -9,6 +9,7 @@ import "package:photos/db/ml/db.dart";
import "package:photos/models/ml/vector.dart";
import "package:photos/service_locator.dart" show isLocalGalleryMode;
import "package:photos/services/machine_learning/ml_constants.dart";
+import "package:photos/services/machine_learning/ml_exceptions.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
import "package:photos/services/machine_learning/semantic_search/query_result.dart";
import "package:photos/services/machine_learning/webgpu_execution_policy.dart";
@@ -94,6 +95,9 @@ class MLComputer extends SuperIsolate {
if (isolateResult is RustCorruptModelException) {
throw isolateResult;
}
+ if (isolateResult is ModelDownloadNetworkException) {
+ throw isolateResult;
+ }
final textEmbedding = isolateResult as List<double>;
return textEmbedding;
} on WiFiUnavailableError catch (e, s) {
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart b/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart
index 055553d580..794e93e700 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_exceptions.dart
@@ -1,5 +1,14 @@
import "package:photos/core/exceptions.dart";
+class ModelDownloadNetworkException implements Exception, LocallyHandledError {
+ const ModelDownloadNetworkException(this.message);
+
+ final String message;
+
+ @override
+ String toString() => "ModelDownloadNetworkException: $message";
+}
+
class ThumbnailRetrievalException implements Exception {
final String message;
final StackTrace stackTrace;
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
index 74b61fd297..f65527cd3f 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
@@ -3,6 +3,7 @@ import "package:logging/logging.dart";
import "package:photos/models/ml/ml_versions.dart"
show mlIndexFlagCoreML, mlIndexFlagRuntimeRust, mlIndexFlagWebGPU;
import "package:photos/service_locator.dart" show flagService, localSettings;
+import "package:photos/services/machine_learning/ml_exceptions.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/machine_learning/webgpu_execution_policy.dart";
@@ -68,6 +69,9 @@ class MLIndexingIsolate extends SuperIsolate {
shouldPauseIndexingAndClustering = true;
throw isolateResult;
}
+ if (isolateResult is ModelDownloadNetworkException) {
+ throw isolateResult;
+ }
final resultJsonString = isolateResult as String?;
if (resultJsonString == null) {
if (!shouldPauseIndexingAndClustering) {
@@ -103,10 +107,13 @@ class MLIndexingIsolate extends SuperIsolate {
final frozenRuntimeArgs = Map<String, dynamic>.unmodifiable(
rustRuntimeArgs,
);
- await runInIsolate(
+ final result = await runInIsolate(
IsolateOperation.prepareRustMlRuntime,
frozenRuntimeArgs,
);
+ if (result is ModelDownloadNetworkException) {
+ throw result;
+ }
_cachedRustRuntimeArgs = frozenRuntimeArgs;
});
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart
index ebbfb5dbf4..453c3b48ce 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart
@@ -6,6 +6,7 @@ import "package:path/path.dart" as p;
import "package:path_provider/path_provider.dart";
import "package:photos/service_locator.dart"
show flagService, hasGrantedMLConsent, isLocalGalleryMode, localSettings;
+import "package:photos/services/machine_learning/ml_exceptions.dart";
import "package:photos/src/rust/api/ml_indexing_api.dart" as rust_ml;
import "package:photos/utils/network_util.dart";
import "package:synchronized/synchronized.dart";
@@ -116,11 +117,20 @@ class MLModelDownloadService {
runPets: downloadIndexing && _shouldDownloadPetModels,
includeClipText: downloadText,
)) {
- _progressController.add((
- update.model,
- update.downloadedBytes.toInt(),
- update.totalBytes?.toInt() ?? 0,
- ));
+ switch (update) {
+ case rust_ml.ModelDownloadEvent_Progress():
+ _progressController.add((
+ update.model,
+ update.downloadedBytes.toInt(),
+ update.totalBytes?.toInt() ?? 0,
+ ));
+ case rust_ml.ModelDownloadEvent_Failed():
+ final error = update.error;
+ if (error is rust_ml.RustMlError_ModelDownloadNetwork) {
+ throw ModelDownloadNetworkException(error.message);
+ }
+ throw error;
+ }
}
} catch (e, s) {
_logger.warning(
diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
index a9c5051ce8..6f479072fe 100644
--- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
+++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
@@ -10,6 +10,7 @@ import "package:photos/models/ml/face/box.dart";
import "package:photos/models/ml/vector.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
import "package:photos/services/machine_learning/ml_constants.dart";
+import "package:photos/services/machine_learning/ml_exceptions.dart";
import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/machine_learning/semantic_search/query_result.dart";
import "package:photos/src/rust/api/image_processing_api.dart"
@@ -100,12 +101,18 @@ Future<dynamic> isolateFunction(
result = await analyzeImageRust(args);
} on rust_ml.RustMlError_CorruptModel catch (e) {
return RustCorruptModelException(e.message);
+ } on ModelDownloadNetworkException catch (e) {
+ return e;
}
return result.toJsonString();
case IsolateOperation.prepareRustMlRuntime:
await _ensureRustLoaded();
- await _ensureRustRuntimePrepared(args);
+ try {
+ await _ensureRustRuntimePrepared(args);
+ } on rust_ml.RustMlError_ModelDownloadNetwork catch (e) {
+ return ModelDownloadNetworkException(e.message);
+ }
return true;
case IsolateOperation.releaseRustMlRuntime:
@@ -153,6 +160,8 @@ Future<dynamic> isolateFunction(
);
} on rust_ml.RustMlError_CorruptModel catch (e) {
return RustCorruptModelException(e.message);
+ } on rust_ml.RustMlError_ModelDownloadNetwork catch (e) {
+ return ModelDownloadNetworkException(e.message);
}
return List<double>.from(result.embedding, growable: false);
diff --git a/mobile/apps/photos/lib/utils/ml_util.dart b/mobile/apps/photos/lib/utils/ml_util.dart
index 560f9581f3..2408bbde10 100644
--- a/mobile/apps/photos/lib/utils/ml_util.dart
+++ b/mobile/apps/photos/lib/utils/ml_util.dart
@@ -822,6 +822,8 @@ Future<MLResult> analyzeImageRust(Map args) async {
rust_ml.AnalyzeImageResult rustResult;
try {
rustResult = await runRustAnalyzeForPath(imagePath);
+ } on rust_ml.RustMlError_ModelDownloadNetwork catch (e) {
+ throw ModelDownloadNetworkException(e.message);
} catch (e, s) {
if (!_isRustImageIssue(e)) {
if (_isRustCorruptModelIssue(e)) {
@@ -950,7 +952,7 @@ Future<MLResult> analyzeImageRust(Map args) async {
return result;
} catch (e, s) {
- if (isExpectedMlSkipError(e)) {
+ if (e is ModelDownloadNetworkException || isExpectedMlSkipError(e)) {
rethrow;
}
_logger.severe("Could not analyze image with Rust pipeline", e, s);
diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
index 520400c6d1..2a161bcd10 100644
--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
@@ -1,6 +1,6 @@
use std::path::Path;
-use ente_assets::AssetStore;
+use ente_assets::{AssetStore, download::Error as DownloadError};
use ente_ml::{assets, error::MlError, indexing, types};
use flutter_rust_bridge::spawn_blocking_with;
@@ -21,6 +21,7 @@ pub struct AnalyzeImageRequest {
pub enum RustMlError {
InvalidImage { message: String },
CorruptModel { message: String },
+ ModelDownloadNetwork { message: String },
Other { message: String },
}
@@ -113,10 +114,15 @@ pub struct RunClipTextRequest {
}
#[derive(Clone, Debug)]
-pub struct ModelDownloadProgress {
- pub model: String,
- pub downloaded_bytes: u64,
- pub total_bytes: Option<u64>,
+pub enum ModelDownloadEvent {
+ Progress {
+ model: String,
+ downloaded_bytes: u64,
+ total_bytes: Option<u64>,
+ },
+ Failed {
+ error: RustMlError,
+ },
}
#[derive(Clone, Debug)]
@@ -216,7 +222,7 @@ pub async fn preload_ml_models(
run_clip: bool,
run_pets: bool,
include_clip_text: bool,
- progress: StreamSink<ModelDownloadProgress>,
+ progress: StreamSink<ModelDownloadEvent>,
) {
let store = AssetStore::new(&assets_dir);
let mut selected = assets::indexing_assets(run_faces, run_clip, run_pets);
@@ -243,7 +249,7 @@ pub async fn preload_ml_models(
}
if let Err(error) =
assets::ensure_mobile_models(&store, Path::new(&assets_dir), &selected, |update| {
- let _ = progress.add(ModelDownloadProgress {
+ let _ = progress.add(ModelDownloadEvent::Progress {
model: labels[update.asset_index].to_string(),
downloaded_bytes: update.asset_progress.downloaded_bytes,
total_bytes: update.asset_progress.total_bytes,
@@ -251,7 +257,9 @@ pub async fn preload_ml_models(
})
.await
{
- let _ = progress.add_error(error.to_string());
+ let _ = progress.add(ModelDownloadEvent::Failed {
+ error: error.into(),
+ });
}
}
@@ -283,6 +291,28 @@ impl From<MlError> for RustMlError {
}
}
+impl From<DownloadError> for RustMlError {
+ fn from(error: DownloadError) -> Self {
+ let message = format!("model download failed: {error}");
+ if is_expected_download_error(&error) {
+ Self::ModelDownloadNetwork { message }
+ } else {
+ Self::Other { message }
+ }
+ }
+}
+
+fn is_expected_download_error(error: &DownloadError) -> bool {
+ match error {
+ DownloadError::Network(_) | DownloadError::Http(_) => true,
+ DownloadError::Target { source, .. } => is_expected_download_error(source),
+ DownloadError::Fallback { single, ranged } => {
+ is_expected_download_error(single) && is_expected_download_error(ranged)
+ }
+ _ => false,
+ }
+}
+
fn to_api_analyze_image_result(result: indexing::AnalyzeImageResult) -> AnalyzeImageResult {
AnalyzeImageResult {
file_id: result.file_id,
@@ -391,3 +421,94 @@ fn to_api_pet_body_result(result: types::PetBodyResult) -> RustPetBodyResult {
.collect(),
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn network_and_http_download_failures_keep_their_classification() {
+ for error in [
+ DownloadError::Network("timed out".into()),
+ DownloadError::Http(503),
+ DownloadError::Target {
+ label: "CLIP".into(),
+ source: Box::new(DownloadError::Fallback {
+ single: Box::new(DownloadError::Network("connection closed".into())),
+ ranged: Box::new(DownloadError::Http(503)),
+ }),
+ },
+ ] {
+ let expected_message = format!("model download failed: {error}");
+ let RustMlError::ModelDownloadNetwork { message } = RustMlError::from(error) else {
+ panic!("expected a network download failure");
+ };
+ assert_eq!(message, expected_message);
+ }
+ }
+
+ #[test]
+ fn non_network_download_failures_remain_reportable() {
+ for error in [
+ DownloadError::Validation("checksum mismatch".into()),
+ DownloadError::StorageFull,
+ DownloadError::Io(std::io::Error::other("cannot import model")),
+ DownloadError::Protocol("invalid range".into()),
+ DownloadError::SizeMismatch {
+ expected: 10,
+ actual: 5,
+ },
+ DownloadError::InvalidTarget("invalid asset".into()),
+ DownloadError::Cancelled,
+ ] {
+ let error = DownloadError::Target {
+ label: "CLIP".into(),
+ source: Box::new(error),
+ };
+ assert!(matches!(
+ RustMlError::from(error),
+ RustMlError::Other { .. }
+ ));
+ }
+ }
+
+ #[test]
+ fn mixed_fallback_failures_remain_reportable() {
+ for (single, ranged) in [
+ (
+ DownloadError::Validation("checksum mismatch".into()),
+ DownloadError::Network("timed out".into()),
+ ),
+ (
+ DownloadError::Network("timed out".into()),
+ DownloadError::Protocol("invalid range".into()),
+ ),
+ ] {
+ let error = DownloadError::Fallback {
+ single: Box::new(single),
+ ranged: Box::new(ranged),
+ };
+ assert!(matches!(
+ RustMlError::from(error),
+ RustMlError::Other { .. }
+ ));
+ }
+ }
+
+ #[test]
+ fn native_runtime_and_inference_errors_remain_reportable() {
+ for error in [
+ MlError::Runtime("worker failed".into()),
+ MlError::Ort("inference failed".into()),
+ ] {
+ assert!(matches!(
+ RustMlError::from(error),
+ RustMlError::Other { .. }
+ ));
+ }
+ assert!(matches!(
+ task_error("worker panicked"),
+ RustMlError::Other { .. }
+ ));
+ }
+}
diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rs
index 88363f67d2..bffde907e4 100644
--- a/rust/crates/assets/src/store.rs
+++ b/rust/crates/assets/src/store.rs
@@ -752,6 +752,7 @@ mod tests {
}
Err(error) => panic!("test server: {error}"),
};
+ stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
diff --git a/rust/crates/ml/src/assets.rs b/rust/crates/ml/src/assets.rs
index ab80dea5d7..407697aac1 100644
--- a/rust/crates/ml/src/assets.rs
+++ b/rust/crates/ml/src/assets.rs
@@ -2,10 +2,10 @@ use std::fs;
use std::path::{Path, PathBuf};
use ente_assets::{
- Asset, AssetDownloadProgress, AssetFile, AssetStore, download::CancellationToken,
+ Asset, AssetDownloadProgress, AssetFile, AssetStore,
+ download::{CancellationToken, Error as DownloadError},
};
-use super::error::{MlError, MlResult};
use super::models::{self, Model, ModelPaths};
const MODELS: &str = "models";
@@ -101,7 +101,7 @@ pub async fn ensure_indexing_models(
run_faces: bool,
run_clip: bool,
run_pets: bool,
-) -> MlResult<ModelPaths> {
+) -> Result<ModelPaths, DownloadError> {
ensure_mobile_models(
store,
legacy_dir,
@@ -112,7 +112,10 @@ pub async fn ensure_indexing_models(
Ok(indexing_model_paths(store, run_faces, run_clip, run_pets))
}
-pub async fn ensure_clip_text(store: &AssetStore, legacy_dir: &Path) -> MlResult<ClipTextPaths> {
+pub async fn ensure_clip_text(
+ store: &AssetStore,
+ legacy_dir: &Path,
+) -> Result<ClipTextPaths, DownloadError> {
ensure_mobile_models(store, legacy_dir, &[clip_text_asset()], |_| {}).await?;
Ok(clip_text_paths(store))
}
@@ -122,7 +125,7 @@ pub async fn ensure_mobile_models(
legacy_dir: &Path,
assets: &[Asset],
on_progress: impl FnMut(AssetDownloadProgress) + Send,
-) -> MlResult<()> {
+) -> Result<(), DownloadError> {
store
.download_with_staged_import(
assets,
@@ -139,7 +142,6 @@ pub async fn ensure_mobile_models(
},
)
.await
- .map_err(|error| MlError::Runtime(format!("model download failed: {error}")))
}
pub fn is_clip_text_available(store: &AssetStore, legacy_dir: &Path, include_vocab: bool) -> bool {