π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Handle desktop ML indexing failures
laurenspriemfea5c8eca671
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.tsindex 9fc32fcbb0..88d975860c 100644--- a/desktop/src/main/services/ml-worker.ts+++ b/desktop/src/main/services/ml-worker.ts@@ -78,6 +78,27 @@ const parseInitData = (data: unknown) => {type MLNative = typeof MLNativeModule;+type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";++interface MLWorkerAnalyzeImageError {+ kind: MLWorkerAnalyzeImageErrorKind;+ message: string;+}++type MLWorkerAnalyzeImageResponse =+ | { ok: true; result: MLNativeModule.AnalyzeImageResult }+ | { ok: false; error: MLWorkerAnalyzeImageError };++class CategorizedMLWorkerError extends Error {+ constructor(+ public readonly kind: MLWorkerAnalyzeImageErrorKind,+ message: string,+ ) {+ super(message);+ this.name = "CategorizedMLWorkerError";+ }+}+let _native: MLNative | undefined;let _nativeLoadError: string | undefined;@@ -107,7 +128,10 @@ const loadMLNative = (paths: MLNativePaths) => {*/const mlNative = () => {if (_native) return _native;- throw new Error(`ML is unavailable: ${_nativeLoadError ?? "not loaded"}`);+ throw new CategorizedMLWorkerError(+ "init",+ `ML is unavailable: ${_nativeLoadError ?? "not loaded"}`,+ );};/**@@ -189,6 +213,9 @@ const modelSavePath = (modelName: string) =>*/const _verifiedModelPaths = new Map<string, Promise<string>>();+/** The single cleanup attempt shared by all model loads in this process. */+let _oldModelsCleanup: Promise<void> | undefined;+const modelPath = (spec: ModelSpec) => {let promise = _verifiedModelPaths.get(spec.name);if (!promise) {@@ -258,12 +285,18 @@ const computeFileSHA256 = (filePath: string) =>/*** Cleanup old models.*- * This code runs whenever we need to download a new model, which usually- * happens when we update a model, so this is a great time to go through the- * list of previously existent but now unused models, and delete them if they- * exist to clean up the user's disk space.+ * This code runs at most once in each utility process, before its first model+ * is resolved. This is a good time to delete previously existent but now+ * unused models and reclaim their disk space.*/-const cleanupOldModelsIfNeeded = async () => {+const cleanupOldModelsIfNeeded = () =>+ (_oldModelsCleanup ??= cleanupOldModels().catch((e: unknown) => {+ // Removing obsolete cache files is best effort and must not prevent the+ // current models from loading. Retry on the next process start.+ log.warn("Failed to clean up old ML models", e);+ }));++const cleanupOldModels = async () => {const oldModelNames = ["clip-image-vit-32-float32.onnx","clip-text-vit-32-uint8.onnx",@@ -281,7 +314,7 @@ const cleanupOldModelsIfNeeded = async () => {const modelPath = modelSavePath(modelName);if (existsSync(modelPath)) {log.info(`Removing unused ML model at ${modelPath}`);- await fs.rm(modelPath);+ await fs.rm(modelPath, { force: true });}}};@@ -409,7 +442,15 @@ export interface MLWorkerAnalyzeImageRequest {*/export const analyzeImage = async (req: MLWorkerAnalyzeImageRequest,-): Promise<MLNativeModule.AnalyzeImageResult> => {+): Promise<MLWorkerAnalyzeImageResponse> => {+ try {+ return { ok: true, result: await analyzeImageOrThrow(req) };+ } catch (e) {+ return { ok: false, error: categorizeAnalyzeImageError(e) };+ }+};++const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {const native = mlNative();try {return await analyzeImageOnce(native, req);@@ -427,6 +468,23 @@ export const analyzeImage = async (}};+const categorizeAnalyzeImageError = (e: unknown): MLWorkerAnalyzeImageError => {+ if (e instanceof CategorizedMLWorkerError) {+ return { kind: e.kind, message: e.message };+ }++ const message = e instanceof Error ? e.message : String(e);+ const kind: MLWorkerAnalyzeImageErrorKind =+ message.startsWith("Decode: ") || message.startsWith("Image: ")+ ? "image"+ : message.startsWith("Ort: ") ||+ message.startsWith("CorruptModel: ") ||+ message.startsWith("Runtime: ")+ ? "ort"+ : "misc";+ return { kind, message };+};+const analyzeImageOnce = async (native: MLNative,req: MLWorkerAnalyzeImageRequest,diff --git a/mobile/apps/photos/lib/utils/ml_util.dart b/mobile/apps/photos/lib/utils/ml_util.dartindex 5a6e4947a4..c98eb360a6 100644--- a/mobile/apps/photos/lib/utils/ml_util.dart+++ b/mobile/apps/photos/lib/utils/ml_util.dart@@ -859,21 +859,22 @@ Future<MLResult> analyzeImageRust(Map args) async {try {rustResult = await runRustAnalyzeForPath(imagePath);} catch (e, s) {- if (!_isRustDecodeIssue(e)) {+ if (!_isRustImageIssue(e)) {if (_isRustCorruptModelIssue(e)) {rethrow;}_logger.severe(- "Rust pipeline failed (non-decode) for fileID $enteFileID (format: $fileFormat)",+ "Rust pipeline failed (non-image) for fileID $enteFileID (format: $fileFormat)",e,s,);rethrow;}- // A Rust decode error may be a resource-limit rejection. Retrying it- // through an unbounded Dart decoder can exhaust the mobile process.- throw _asInvalidImageFormatExceptionForRustDecodeFailure(+ // A Rust image error is deterministic for this file. In particular, a+ // decode error may be a resource-limit rejection, so retrying it through+ // an unbounded Dart decoder can exhaust the mobile process.+ throw _asInvalidImageFormatExceptionForRustImageFailure(enteFileID: enteFileID,fileFormat: fileFormat,primaryError: e,@@ -1023,8 +1024,9 @@ String formatExpectedMlSkipReasonForLogs(Object error) {return firstLine.isEmpty ? "unknown ML skip reason" : firstLine;}-bool _isRustDecodeIssue(Object error) {- return error is rust_ml.RustMlError_Decode;+bool _isRustImageIssue(Object error) {+ return error is rust_ml.RustMlError_Decode ||+ error is rust_ml.RustMlError_Image;}bool _isRustCorruptModelIssue(Object error) {@@ -1043,13 +1045,13 @@ String _normalizedErrorMessage(Object error) {return error.toString().toLowerCase();}-Exception _asInvalidImageFormatExceptionForRustDecodeFailure({+Exception _asInvalidImageFormatExceptionForRustImageFailure({required int enteFileID,required String fileFormat,required Object primaryError,}) {final details = <String>[- "InvalidImageFormatException: Rust decode failed for fileID $enteFileID (format: $fileFormat)",+ "InvalidImageFormatException: Rust image processing failed for fileID $enteFileID (format: $fileFormat)","primary_error: $primaryError",];return Exception(details.join("; "));diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rsindex 3a3d6a787d..659de09f4e 100644--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs@@ -31,6 +31,7 @@ pub struct AnalyzeImageRequest {pub enum RustMlError {InvalidRequest(String),Decode(String),+ Image(String),Preprocess(String),Ort(String),CorruptModel(String),@@ -229,6 +230,7 @@ impl From<SharedMlError> for RustMlError {match value {SharedMlError::InvalidRequest(message) => RustMlError::InvalidRequest(message),SharedMlError::Decode(message) => RustMlError::Decode(message),+ SharedMlError::Image(message) => RustMlError::Image(message),SharedMlError::Preprocess(message) => RustMlError::Preprocess(message),SharedMlError::Ort(message) => RustMlError::Ort(message),SharedMlError::CorruptModel(message) => RustMlError::CorruptModel(message),diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex d48dc45128..50def15c90 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -276,6 +276,7 @@ fn ml_error_to_napi(error: MlError) -> Error {let kind = match &error {MlError::InvalidRequest(_) => "InvalidRequest",MlError::Decode(_) => "Decode",+ MlError::Image(_) => "Image",MlError::Preprocess(_) => "Preprocess",MlError::Ort(_) => "Ort",MlError::CorruptModel(_) => "CorruptModel",@@ -285,6 +286,7 @@ fn ml_error_to_napi(error: MlError) -> Error {let detail = match error {MlError::InvalidRequest(message)| MlError::Decode(message)+ | MlError::Image(message)| MlError::Preprocess(message)| MlError::Ort(message)| MlError::CorruptModel(message)diff --git a/rust/crates/photos/src/ml/error.rs b/rust/crates/photos/src/ml/error.rsindex afb3dcdf44..1427c03126 100644--- a/rust/crates/photos/src/ml/error.rs+++ b/rust/crates/photos/src/ml/error.rs@@ -8,6 +8,11 @@ pub enum MlError {InvalidRequest(String),#[error("decode error: {0}")]Decode(String),+ /// A deterministic failure caused by this image's decoded contents or by+ /// image-derived geometry. Unlike generic preprocessing/postprocessing+ /// failures, callers may safely stop retrying this particular image.+ #[error("image processing error: {0}")]+ Image(String),#[error("preprocess error: {0}")]Preprocess(String),#[error("onnx runtime error: {0}")]@@ -36,7 +41,7 @@ impl From<ente_image::ImageError> for MlError {fn from(value: ente_image::ImageError) -> Self {match value {ente_image::ImageError::Decode(message) => MlError::Decode(message),- ente_image::ImageError::Postprocess(message) => MlError::Postprocess(message),+ ente_image::ImageError::Postprocess(message) => MlError::Image(message),}}}diff --git a/rust/crates/photos/src/ml/face/align.rs b/rust/crates/photos/src/ml/face/align.rsindex 83fcddd5d6..472523c4ae 100644--- a/rust/crates/photos/src/ml/face/align.rs+++ b/rust/crates/photos/src/ml/face/align.rs@@ -76,9 +76,9 @@ fn take_rgb_image(decoded: &mut DecodedImage) -> MlResult<RgbImage> {let expected_len = (decoded.dimensions.width as usize).checked_mul(decoded.dimensions.height as usize).and_then(|pixels| pixels.checked_mul(3))- .ok_or_else(|| MlError::Preprocess("RGB source dimensions overflow".to_string()))?;+ .ok_or_else(|| MlError::Image("RGB source dimensions overflow".to_string()))?;if decoded.rgb.len() != expected_len {- return Err(MlError::Preprocess(format!(+ return Err(MlError::Image(format!("invalid RGB source length: expected {expected_len}, got {}",decoded.rgb.len())));@@ -143,7 +143,7 @@ fn estimate_similarity_transform(src_points: &[[f32; 2]; 5]) -> MlResult<Alignmelet rank = singular.iter().filter(|v| **v > 1e-6).count();if rank == 0 {- return Err(MlError::Postprocess(+ return Err(MlError::Image("failed to estimate similarity transform (rank=0)".to_string(),));}@@ -236,7 +236,7 @@ fn warp_face_image(source: &RgbImage, affine_matrix: &[[f32; 3]; 3]) -> MlResulttransform[2][1],transform[2][2],])- .ok_or_else(|| MlError::Postprocess("invalid affine matrix projection".to_string()))?;+ .ok_or_else(|| MlError::Image("invalid affine matrix projection".to_string()))?;let mut output = RgbImage::from_pixel(FACE_INPUT_SIZE, FACE_INPUT_SIZE, Rgb([114, 114, 114]));warp_into(@@ -405,7 +405,10 @@ fn variance_2d(matrix: &[i32], rows: usize, cols: usize) -> f32 {#[cfg(test)]mod tests {use super::run_face_alignment;- use crate::ml::types::{DecodedImage, Dimensions};+ use crate::ml::{+ error::MlError,+ types::{DecodedImage, Dimensions, FaceDetection},+ };#[test]fn face_alignment_restores_the_decoded_rgb_allocation() {@@ -425,4 +428,25 @@ mod tests {assert_eq!(decoded.rgb, (0..12).collect::<Vec<_>>());assert_eq!(decoded.rgb.as_ptr(), allocation);}++ #[test]+ fn degenerate_landmarks_are_classified_as_an_image_error() {+ let mut decoded = DecodedImage {+ dimensions: Dimensions {+ width: 2,+ height: 2,+ },+ rgb: (0..12).collect(),+ };+ let detection = FaceDetection {+ score: 1.0,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ keypoints: [[0.5, 0.5]; 5],+ };++ let result = run_face_alignment(1, &mut decoded, vec![detection]);++ assert!(matches!(result, Err(MlError::Image(_))));+ assert_eq!(decoded.rgb, (0..12).collect::<Vec<_>>());+ }}diff --git a/rust/crates/photos/src/ml/face/thumbnail.rs b/rust/crates/photos/src/ml/face/thumbnail.rsindex d87c9dfdc7..0513eadd8b 100644--- a/rust/crates/photos/src/ml/face/thumbnail.rs+++ b/rust/crates/photos/src/ml/face/thumbnail.rs@@ -54,9 +54,8 @@ pub fn generate_face_thumbnails(let mut results = Vec::with_capacity(face_boxes.len());for (index, face_box) in face_boxes.iter().enumerate() {- let crop = compute_crop_rect(face_box, image_width, image_height).map_err(|e| {- MlError::InvalidRequest(format!("invalid face box at index {index}: {e}",))- })?;+ let crop = compute_crop_rect(face_box, image_width, image_height)+ .map_err(|e| MlError::Image(format!("invalid face box at index {index}: {e}")))?;let (target_width, target_height) =face_thumbnail_dimensions(crop.output_width, crop.output_height)?;let resized =@@ -164,7 +163,7 @@ fn resize_crop_with_fir(.resize_alg(ResizeAlg::Convolution(filter));resizer.resize(source, &mut resized, Some(&options))- .map_err(|e| MlError::Postprocess(format!("failed to resize face thumbnail crop: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to resize face thumbnail crop: {e}")))?;Ok(resized)}@@ -191,7 +190,7 @@ fn select_resize_filter(crop: &CropRect, target_width: u32, target_height: u32)fn face_thumbnail_dimensions(width: u32, height: u32) -> MlResult<(u32, u32)> {if width == 0 || height == 0 {- return Err(MlError::Postprocess(+ return Err(MlError::Image("cannot compute resize dimensions for zero-sized crop".to_string(),));}@@ -227,7 +226,7 @@ fn face_thumbnail_dimensions(width: u32, height: u32) -> MlResult<(u32, u32)> {fn scaled_dimension(value: u32, original_dimension: u32, target_dimension: u32) -> MlResult<u32> {if original_dimension == 0 {- return Err(MlError::Postprocess(+ return Err(MlError::Image("cannot scale from a zero-sized dimension".to_string(),));}@@ -237,7 +236,7 @@ fn scaled_dimension(value: u32, original_dimension: u32, target_dimension: u32)let rounded = (numerator + (denominator / 2)) / denominator;u32::try_from(rounded.max(1)).map_err(|_| {- MlError::Postprocess(format!(+ MlError::Image(format!("scaled dimension exceeds u32 for value={value}, original={original_dimension}, target={target_dimension}",))})@@ -252,7 +251,10 @@ mod tests {FaceBox, compute_crop_rect, face_thumbnail_dimensions, generate_face_thumbnails,select_resize_filter,};- use crate::ml::types::{DecodedImage, Dimensions};+ use crate::ml::{+ error::MlError,+ types::{DecodedImage, Dimensions},+ };#[test]fn compute_crop_rect_matches_canvas_math_for_center_box() {@@ -330,9 +332,10 @@ mod tests {height: 0.3,}];- let result = generate_face_thumbnails(&decoded, &face_boxes);-- assert!(result.is_err());+ assert!(matches!(+ generate_face_thumbnails(&decoded, &face_boxes),+ Err(MlError::Image(_))+ ));}#[test]diff --git a/rust/crates/photos/src/ml/pet/align.rs b/rust/crates/photos/src/ml/pet/align.rsindex acc51ced87..fab60bd089 100644--- a/rust/crates/photos/src/ml/pet/align.rs+++ b/rust/crates/photos/src/ml/pet/align.rs@@ -244,7 +244,7 @@ fn crop_and_resize_rgb<'a>(let clamped_w = w.min(src_w.saturating_sub(x));let clamped_h = h.min(src_h.saturating_sub(y));if clamped_w == 0 || clamped_h == 0 {- return Err(MlError::Preprocess(+ return Err(MlError::Image("crop_and_resize_rgb: crop region extends beyond source image".to_string(),));}@@ -313,16 +313,16 @@ impl<'a> RgbRegion<'a> {|| x > decoded.dimensions.width.saturating_sub(width)|| y > decoded.dimensions.height.saturating_sub(height){- return Err(MlError::Preprocess(+ return Err(MlError::Image("RGB region extends beyond decoded image".to_string(),));}let expected_len = (decoded.dimensions.width as usize).checked_mul(decoded.dimensions.height as usize).and_then(|pixels| pixels.checked_mul(3))- .ok_or_else(|| MlError::Preprocess("decoded RGB dimensions overflow".to_string()))?;+ .ok_or_else(|| MlError::Image("decoded RGB dimensions overflow".to_string()))?;if decoded.rgb.len() < expected_len {- return Err(MlError::Preprocess(+ return Err(MlError::Image("decoded RGB buffer is shorter than its dimensions".to_string(),));}diff --git a/rust/crates/photos/src/ml/pet/preprocess.rs b/rust/crates/photos/src/ml/pet/preprocess.rsindex 44e6a2d124..bff5d5cf51 100644--- a/rust/crates/photos/src/ml/pet/preprocess.rs+++ b/rust/crates/photos/src/ml/pet/preprocess.rs@@ -130,7 +130,7 @@ fn relative_crop(decoded: &DecodedImage, box_xyxy: &[f32; 4]) -> MlResult<PixelClet height = y2.saturating_sub(y1);if width == 0 || height == 0 {- return Err(MlError::Preprocess("crop region has zero area".to_string()));+ return Err(MlError::Image("crop region has zero area".to_string()));}Ok(PixelCrop {@@ -166,25 +166,23 @@ impl RgbCropResizer {crop: PixelCrop,) -> MlResult<&[u8]> {if crop.width == 0 || crop.height == 0 {- return Err(MlError::Preprocess(- "crop dimensions cannot be zero".to_string(),- ));+ return Err(MlError::Image("crop dimensions cannot be zero".to_string()));}if crop.x > source_width.saturating_sub(crop.width)|| crop.y > source_height.saturating_sub(crop.height){- return Err(MlError::Preprocess(+ return Err(MlError::Image("crop region extends beyond source image".to_string(),));}let source = FirImageRef::new(source_width, source_height, rgb, PixelType::U8x3)- .map_err(|e| MlError::Preprocess(format!("failed to create FIR source image: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to create FIR source image: {e}")))?;let source = CroppedImage::new(&source, crop.x, crop.y, crop.width, crop.height)- .map_err(|e| MlError::Preprocess(format!("failed to create FIR crop view: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to create FIR crop view: {e}")))?;self.resizer.resize(&source, &mut self.resized, Some(&self.options))- .map_err(|e| MlError::Preprocess(format!("failed to resize RGB crop: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to resize RGB crop: {e}")))?;Ok(self.resized.buffer())}diff --git a/rust/crates/photos/src/ml/preprocess.rs b/rust/crates/photos/src/ml/preprocess.rsindex c4c5919c85..1b45c8c38a 100644--- a/rust/crates/photos/src/ml/preprocess.rs+++ b/rust/crates/photos/src/ml/preprocess.rs@@ -73,7 +73,7 @@ impl YoloInput {pub(crate) fn preprocess_yolo(decoded: &DecodedImage) -> MlResult<YoloInput> {if decoded.dimensions.width == 0 || decoded.dimensions.height == 0 {- return Err(MlError::Preprocess(+ return Err(MlError::Image("image dimensions cannot be zero".to_string(),));}@@ -90,14 +90,14 @@ pub(crate) fn preprocess_yolo(decoded: &DecodedImage) -> MlResult<YoloInput> {decoded.rgb.as_slice(),PixelType::U8x3,)- .map_err(|e| MlError::Preprocess(format!("failed to create FIR source image: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to create FIR source image: {e}")))?;let mut resized_image = FirImage::new(scaled_width, scaled_height, PixelType::U8x3);let mut resizer = Resizer::new();let options = ResizeOptions::new().resize_alg(ResizeAlg::Interpolation(FilterType::Bilinear));resizer.resize(&src_image, &mut resized_image, Some(&options))- .map_err(|e| MlError::Preprocess(format!("failed to resize YOLO image input: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to resize YOLO image input: {e}")))?;let scaled_width_usize = scaled_width as usize;let scaled_height_usize = scaled_height as usize;@@ -134,7 +134,7 @@ pub(crate) fn preprocess_yolo(decoded: &DecodedImage) -> MlResult<YoloInput> {pub(crate) fn preprocess_clip(decoded: &DecodedImage) -> MlResult<Vec<f32>> {if decoded.dimensions.width == 0 || decoded.dimensions.height == 0 {- return Err(MlError::Preprocess(+ return Err(MlError::Image("image dimensions cannot be zero".to_string(),));}@@ -151,7 +151,7 @@ pub(crate) fn preprocess_clip(decoded: &DecodedImage) -> MlResult<Vec<f32>> {decoded.rgb.as_slice(),PixelType::U8x3,)- .map_err(|e| MlError::Preprocess(format!("failed to create FIR source image: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to create FIR source image: {e}")))?;let start_x = (scaled_width.saturating_sub(CLIP_IMAGE_INPUT_SIZE as u32) / 2) as f64;let start_y = (scaled_height.saturating_sub(CLIP_IMAGE_INPUT_SIZE as u32) / 2) as f64;@@ -173,7 +173,7 @@ pub(crate) fn preprocess_clip(decoded: &DecodedImage) -> MlResult<Vec<f32>> {.crop(crop_left, crop_top, crop_width, crop_height);resizer.resize(&src_image, &mut resized_image, Some(&options))- .map_err(|e| MlError::Preprocess(format!("failed to resize CLIP image input: {e}")))?;+ .map_err(|e| MlError::Image(format!("failed to resize CLIP image input: {e}")))?;let resized = resized_image.buffer();@@ -201,7 +201,7 @@ mod tests {images::{Image as FirImage, ImageRef as FirImageRef},};- use super::{YOLO_INPUT_SIZE, YoloInput, preprocess_clip};+ use super::{YOLO_INPUT_SIZE, YoloInput, preprocess_clip, preprocess_yolo};use crate::ml::{clip::CLIP_IMAGE_INPUT_SIZE,error::{MlError, MlResult},@@ -209,6 +209,14 @@ mod tests {types::{DecodedImage, Dimensions},};+ #[test]+ fn zero_sized_images_are_classified_as_image_errors() {+ let decoded = test_image(0, 0);++ assert!(matches!(preprocess_yolo(&decoded), Err(MlError::Image(_))));+ assert!(matches!(preprocess_clip(&decoded), Err(MlError::Image(_))));+ }+#[test]fn yolo_geometry_matches_the_previous_letterbox_correction() {let input = YoloInput {diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.tsindex 125cd68479..2cbe1c6269 100644--- a/web/packages/base/types/ipc.ts+++ b/web/packages/base/types/ipc.ts@@ -743,7 +743,7 @@ export interface ElectronMLWorker {*/analyzeImage: (request: MLWorkerAnalyzeImageRequest,- ) => Promise<MLWorkerAnalyzeImageResult>;+ ) => Promise<MLWorkerAnalyzeImageResponse>;/*** Release the (pinned) native ML sessions.@@ -809,6 +809,32 @@ export interface MLWorkerAnalyzeImageRequest {generateFaceCrops: boolean;}+/**+ * Broad category for an error encountered while analyzing an image.+ *+ * Only `image` indicates a permanent problem inherent to the image. `init`+ * blocks indexing for the lifetime of the utility process, while `ort` and+ * `misc` remain retryable. None of those other kinds should permanently mark+ * the file as unindexable.+ */+export type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";++/** An error returned by the Electron ML utility process. */+export interface MLWorkerAnalyzeImageError {+ kind: MLWorkerAnalyzeImageErrorKind;+ message: string;+}++/**+ * Result envelope for an image analysis request.+ *+ * This explicit envelope is used instead of throwing a custom `Error` because+ * Comlink only preserves an error's name, message and stack.+ */+export type MLWorkerAnalyzeImageResponse =+ | { ok: true; result: MLWorkerAnalyzeImageResult }+ | { ok: false; error: MLWorkerAnalyzeImageError };+/*** A face detected by {@link ElectronMLWorker.analyzeImage}.*diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.tsindex 5b2cbdf350..283459e127 100644--- a/web/packages/new/photos/services/ml/worker.ts+++ b/web/packages/new/photos/services/ml/worker.ts@@ -7,6 +7,7 @@ import log from "ente-base/log";import { logUnhandledErrorsAndRejectionsInWorker } from "ente-base/log-web";import type {ElectronMLWorker,+ MLWorkerAnalyzeImageErrorKind,MLWorkerAnalyzeImageRequest,MLWorkerAnalyzeImageResult,} from "ente-base/types/ipc";@@ -186,6 +187,8 @@ export class MLWorker {* closes while we're indexing.*/index() {+ if (_blockingMLInitError) return Promise.resolve(0);+const nextIdle = new Promise<number>((resolve, reject) =>this.onNextIdles.push({ resolve, reject }),);@@ -217,6 +220,8 @@ export class MLWorker {* us and won't need to download the file from remote.*/onUpload(file: EnteFile, processableUploadItem: ProcessableUploadItem) {+ if (_blockingMLInitError) return;+// Add the recently uploaded file to the live indexing queue.this.liveQ.push({file,@@ -265,16 +270,20 @@ export class MLWorker {return [];});- this.countSinceLastIdle += items.length;-// If there is items remaining,if (items.length > 0) {// Index them.- const indexedCount = await indexNextBatch(+ const { indexedCount, blockingInitError } = await indexNextBatch(items,this.electron!,this.delegate,);+ if (blockingInitError) {+ this.suspendForMLInitError(blockingInitError);+ return;+ }++ this.countSinceLastIdle += items.length;if (indexedCount > 0) {// We made some progress, so there are no complete blockers// (e.g. network being offline). Reset the idle duration and@@ -308,7 +317,24 @@ export class MLWorker {log.warn("Failed to release the native ML runtime", e);});- // Resolve any awaiting promises returned from `index`.+ this.resolvePendingIndexRequests();+ }++ /** Stop indexing until this web worker (and its utility process) is replaced. */+ private suspendForMLInitError(error: MLAnalyzeError) {+ log.error(+ "Suspending ML indexing because the native runtime failed to initialize",+ error,+ );+ this.liveQ = [];+ this.state = "idle";+ this.idleTimeout = undefined;+ this.delegate?.workerDidUpdateStatus();+ this.resolvePendingIndexRequests();+ }++ /** Resolve callers waiting for the worker to next become idle. */+ private resolvePendingIndexRequests() {const onNextIdles = this.onNextIdles;const countSinceLastIdle = this.countSinceLastIdle;this.onNextIdles = [];@@ -395,7 +421,7 @@ logUnhandledErrorsAndRejectionsInWorker();/*** Index the given batch of items.*- * @returns the count of items which were indexed.+ * @returns the number of indexed items and any process-blocking init error.*/const indexNextBatch = async (items: IndexableItem[],@@ -407,7 +433,7 @@ const indexNextBatch = async (// were able to upload just a bit ago but don't have network now.if (!self.navigator.onLine) {log.info("Skipping ML indexing since we are not online");- return 0;+ return { indexedCount: 0 };}// Keep track if any of the items failed.@@ -417,7 +443,7 @@ const indexNextBatch = async (const tasks = new Array<Promise<void> | undefined>(4).fill(undefined);let i = 0;- while (i < items.length) {+ while (i < items.length && !_blockingMLInitError) {for (let j = 0; j < tasks.length; j++) {if (i < items.length && !tasks[j]) {// Use an IIFE to capture the value of j at the time of@@ -430,6 +456,9 @@ const indexNextBatch = async (.catch((e: unknown) => {const f = fileLogID(item.file);log.error(`Failed to index ${f}`, e);+ if (isMLInitError(e)) {+ _blockingMLInitError ??= e;+ }failureCount++;tasks[j] = undefined;}))(items[i++]!, j);@@ -454,16 +483,16 @@ const indexNextBatch = async (// Clear any cached CLIP indexes, since now we might have new ones.clearCachedCLIPIndexes();- const indexedCount = items.length - failureCount;+ const attemptedCount = i;+ const indexedCount = attemptedCount - failureCount;log.info(failureCount > 0? `Indexed ${indexedCount} files (${failureCount} failed)`- : `Indexed ${items.length} files`,+ : `Indexed ${attemptedCount} files`,);- // Return the count of indexed files.- return indexedCount;+ return { indexedCount, blockingInitError: _blockingMLInitError };};/**@@ -626,8 +655,11 @@ const index = async (electron,);} catch (e) {+ // Only failures inherent to the image (including the JPEG conversion+ // fallback below) are permanent. Runtime and infrastructure failures+ // should be retried after the underlying issue is resolved.// See: [Note: Transient and permanent indexing failures]- await markIndexingFailed(fileID);+ if (isMLImageError(e)) await markIndexingFailed(fileID);throw e;}@@ -747,19 +779,52 @@ const analyzeImageWithConversionFallback = async (`Native decode of ${fileLogID(file)} failed, retrying with a JPEG conversion`,e,);- const blob = await fetchRenderableBlob(- file,- processableUploadItem,- electron,- );- const bytes = new Uint8Array(await blob.arrayBuffer());+ let bytes: Uint8Array;+ try {+ const blob = await fetchRenderableBlob(+ file,+ processableUploadItem,+ electron,+ );+ bytes = new Uint8Array(await blob.arrayBuffer());+ } catch (conversionError) {+ // Fetching the source for conversion can still fail transiently.+ // Do not turn such a download error into a permanent image error.+ if (isNetworkDownloadError(conversionError)) throw conversionError;++ const message =+ conversionError instanceof Error+ ? conversionError.message+ : String(conversionError);+ throw new MLAnalyzeError(+ "image",+ `JPEG conversion fallback failed: ${message}`,+ );+ }return await enqueueAnalyzeImage(electron, { ...request, bytes });}};-/** `true` if the error looks like the native side failing to decode an image. */+/** `true` for a permanent failure inherent to the image being analyzed. */+const isMLImageError = (e: unknown): e is MLAnalyzeError =>+ e instanceof MLAnalyzeError && e.kind == "image";++/** `true` specifically for the Rust image decoder failure that permits fallback. */const isMLDecodeError = (e: unknown) =>- e instanceof Error && e.message.startsWith("Decode: ");+ isMLImageError(e) && e.message.startsWith("Decode: ");++const isMLInitError = (e: unknown): e is MLAnalyzeError =>+ e instanceof MLAnalyzeError && e.kind == "init";++class MLAnalyzeError extends Error {+ constructor(+ public readonly kind: MLWorkerAnalyzeImageErrorKind,+ message: string,+ ) {+ super(message);+ this.name = "MLAnalyzeError";+ }+}/*** A promise queue serializing calls to {@link ElectronMLWorker.analyzeImage}.@@ -772,13 +837,23 @@ const isMLDecodeError = (e: unknown) =>*/let _analyzeImageQueue: Promise<unknown> = Promise.resolve();+/** The native addon is loaded only once, so this error cannot recover here. */+let _blockingMLInitError: MLAnalyzeError | undefined;+const enqueueAnalyzeImage = (electron: ElectronMLWorker,request: MLWorkerAnalyzeImageRequest,): Promise<MLWorkerAnalyzeImageResult> => {- const result = _analyzeImageQueue.then(() =>- electron.analyzeImage(request),- );+ const result = _analyzeImageQueue.then(async () => {+ const response = await electron.analyzeImage(request);+ if (!response.ok) {+ throw new MLAnalyzeError(+ response.error.kind,+ response.error.message,+ );+ }+ return response.result;+ });_analyzeImageQueue = result.catch(() => undefined);return result;};