Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Handle desktop ML indexing failures

laurenspriemfea5c8eca671
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts
index 9fc32fcbb0..88d975860c 100644
--- a/desktop/src/main/services/ml-worker.ts
+++ b/desktop/src/main/services/ml-worker.ts
@@ -78,6 +78,27 @@ const parseInitData = (data: unknown) => {
type MLNative = typeof MLNativeModule;
+type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";
+
+interface MLWorkerAnalyzeImageError {
+ kind: MLWorkerAnalyzeImageErrorKind;
+ message: string;
+}
+
+type MLWorkerAnalyzeImageResponse =
+ | { ok: true; result: MLNativeModule.AnalyzeImageResult }
+ | { ok: false; error: MLWorkerAnalyzeImageError };
+
+class CategorizedMLWorkerError extends Error {
+ constructor(
+ public readonly kind: MLWorkerAnalyzeImageErrorKind,
+ message: string,
+ ) {
+ super(message);
+ this.name = "CategorizedMLWorkerError";
+ }
+}
+
let _native: MLNative | undefined;
let _nativeLoadError: string | undefined;
@@ -107,7 +128,10 @@ const loadMLNative = (paths: MLNativePaths) => {
*/
const mlNative = () => {
if (_native) return _native;
- throw new Error(`ML is unavailable: ${_nativeLoadError ?? "not loaded"}`);
+ throw new CategorizedMLWorkerError(
+ "init",
+ `ML is unavailable: ${_nativeLoadError ?? "not loaded"}`,
+ );
};
/**
@@ -189,6 +213,9 @@ const modelSavePath = (modelName: string) =>
*/
const _verifiedModelPaths = new Map<string, Promise<string>>();
+/** The single cleanup attempt shared by all model loads in this process. */
+let _oldModelsCleanup: Promise<void> | undefined;
+
const modelPath = (spec: ModelSpec) => {
let promise = _verifiedModelPaths.get(spec.name);
if (!promise) {
@@ -258,12 +285,18 @@ const computeFileSHA256 = (filePath: string) =>
/**
* Cleanup old models.
*
- * This code runs whenever we need to download a new model, which usually
- * happens when we update a model, so this is a great time to go through the
- * list of previously existent but now unused models, and delete them if they
- * exist to clean up the user's disk space.
+ * This code runs at most once in each utility process, before its first model
+ * is resolved. This is a good time to delete previously existent but now
+ * unused models and reclaim their disk space.
*/
-const cleanupOldModelsIfNeeded = async () => {
+const cleanupOldModelsIfNeeded = () =>
+ (_oldModelsCleanup ??= cleanupOldModels().catch((e: unknown) => {
+ // Removing obsolete cache files is best effort and must not prevent the
+ // current models from loading. Retry on the next process start.
+ log.warn("Failed to clean up old ML models", e);
+ }));
+
+const cleanupOldModels = async () => {
const oldModelNames = [
"clip-image-vit-32-float32.onnx",
"clip-text-vit-32-uint8.onnx",
@@ -281,7 +314,7 @@ const cleanupOldModelsIfNeeded = async () => {
const modelPath = modelSavePath(modelName);
if (existsSync(modelPath)) {
log.info(`Removing unused ML model at ${modelPath}`);
- await fs.rm(modelPath);
+ await fs.rm(modelPath, { force: true });
}
}
};
@@ -409,7 +442,15 @@ export interface MLWorkerAnalyzeImageRequest {
*/
export const analyzeImage = async (
req: MLWorkerAnalyzeImageRequest,
-): Promise<MLNativeModule.AnalyzeImageResult> => {
+): Promise<MLWorkerAnalyzeImageResponse> => {
+ try {
+ return { ok: true, result: await analyzeImageOrThrow(req) };
+ } catch (e) {
+ return { ok: false, error: categorizeAnalyzeImageError(e) };
+ }
+};
+
+const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {
const native = mlNative();
try {
return await analyzeImageOnce(native, req);
@@ -427,6 +468,23 @@ export const analyzeImage = async (
}
};
+const categorizeAnalyzeImageError = (e: unknown): MLWorkerAnalyzeImageError => {
+ if (e instanceof CategorizedMLWorkerError) {
+ return { kind: e.kind, message: e.message };
+ }
+
+ const message = e instanceof Error ? e.message : String(e);
+ const kind: MLWorkerAnalyzeImageErrorKind =
+ message.startsWith("Decode: ") || message.startsWith("Image: ")
+ ? "image"
+ : message.startsWith("Ort: ") ||
+ message.startsWith("CorruptModel: ") ||
+ message.startsWith("Runtime: ")
+ ? "ort"
+ : "misc";
+ return { kind, message };
+};
+
const analyzeImageOnce = async (
native: MLNative,
req: MLWorkerAnalyzeImageRequest,
diff --git a/mobile/apps/photos/lib/utils/ml_util.dart b/mobile/apps/photos/lib/utils/ml_util.dart
index 5a6e4947a4..c98eb360a6 100644
--- a/mobile/apps/photos/lib/utils/ml_util.dart
+++ b/mobile/apps/photos/lib/utils/ml_util.dart
@@ -859,21 +859,22 @@ Future<MLResult> analyzeImageRust(Map args) async {
try {
rustResult = await runRustAnalyzeForPath(imagePath);
} catch (e, s) {
- if (!_isRustDecodeIssue(e)) {
+ if (!_isRustImageIssue(e)) {
if (_isRustCorruptModelIssue(e)) {
rethrow;
}
_logger.severe(
- "Rust pipeline failed (non-decode) for fileID $enteFileID (format: $fileFormat)",
+ "Rust pipeline failed (non-image) for fileID $enteFileID (format: $fileFormat)",
e,
s,
);
rethrow;
}
- // A Rust decode error may be a resource-limit rejection. Retrying it
- // through an unbounded Dart decoder can exhaust the mobile process.
- throw _asInvalidImageFormatExceptionForRustDecodeFailure(
+ // A Rust image error is deterministic for this file. In particular, a
+ // decode error may be a resource-limit rejection, so retrying it through
+ // an unbounded Dart decoder can exhaust the mobile process.
+ throw _asInvalidImageFormatExceptionForRustImageFailure(
enteFileID: enteFileID,
fileFormat: fileFormat,
primaryError: e,
@@ -1023,8 +1024,9 @@ String formatExpectedMlSkipReasonForLogs(Object error) {
return firstLine.isEmpty ? "unknown ML skip reason" : firstLine;
}
-bool _isRustDecodeIssue(Object error) {
- return error is rust_ml.RustMlError_Decode;
+bool _isRustImageIssue(Object error) {
+ return error is rust_ml.RustMlError_Decode ||
+ error is rust_ml.RustMlError_Image;
}
bool _isRustCorruptModelIssue(Object error) {
@@ -1043,13 +1045,13 @@ String _normalizedErrorMessage(Object error) {
return error.toString().toLowerCase();
}
-Exception _asInvalidImageFormatExceptionForRustDecodeFailure({
+Exception _asInvalidImageFormatExceptionForRustImageFailure({
required int enteFileID,
required String fileFormat,
required Object primaryError,
}) {
final details = <String>[
- "InvalidImageFormatException: Rust decode failed for fileID $enteFileID (format: $fileFormat)",
+ "InvalidImageFormatException: Rust image processing failed for fileID $enteFileID (format: $fileFormat)",
"primary_error: $primaryError",
];
return Exception(details.join("; "));
diff --git a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
index 3a3d6a787d..659de09f4e 100644
--- a/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_indexing_api.rs
@@ -31,6 +31,7 @@ pub struct AnalyzeImageRequest {
pub enum RustMlError {
InvalidRequest(String),
Decode(String),
+ Image(String),
Preprocess(String),
Ort(String),
CorruptModel(String),
@@ -229,6 +230,7 @@ impl From<SharedMlError> for RustMlError {
match value {
SharedMlError::InvalidRequest(message) => RustMlError::InvalidRequest(message),
SharedMlError::Decode(message) => RustMlError::Decode(message),
+ SharedMlError::Image(message) => RustMlError::Image(message),
SharedMlError::Preprocess(message) => RustMlError::Preprocess(message),
SharedMlError::Ort(message) => RustMlError::Ort(message),
SharedMlError::CorruptModel(message) => RustMlError::CorruptModel(message),
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index d48dc45128..50def15c90 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -276,6 +276,7 @@ fn ml_error_to_napi(error: MlError) -> Error {
let kind = match &error {
MlError::InvalidRequest(_) => "InvalidRequest",
MlError::Decode(_) => "Decode",
+ MlError::Image(_) => "Image",
MlError::Preprocess(_) => "Preprocess",
MlError::Ort(_) => "Ort",
MlError::CorruptModel(_) => "CorruptModel",
@@ -285,6 +286,7 @@ fn ml_error_to_napi(error: MlError) -> Error {
let detail = match error {
MlError::InvalidRequest(message)
| MlError::Decode(message)
+ | MlError::Image(message)
| MlError::Preprocess(message)
| MlError::Ort(message)
| MlError::CorruptModel(message)
diff --git a/rust/crates/photos/src/ml/error.rs b/rust/crates/photos/src/ml/error.rs
index afb3dcdf44..1427c03126 100644
--- a/rust/crates/photos/src/ml/error.rs
+++ b/rust/crates/photos/src/ml/error.rs
@@ -8,6 +8,11 @@ pub enum MlError {
InvalidRequest(String),
#[error("decode error: {0}")]
Decode(String),
+ /// A deterministic failure caused by this image's decoded contents or by
+ /// image-derived geometry. Unlike generic preprocessing/postprocessing
+ /// failures, callers may safely stop retrying this particular image.
+ #[error("image processing error: {0}")]
+ Image(String),
#[error("preprocess error: {0}")]
Preprocess(String),
#[error("onnx runtime error: {0}")]
@@ -36,7 +41,7 @@ impl From<ente_image::ImageError> for MlError {
fn from(value: ente_image::ImageError) -> Self {
match value {
ente_image::ImageError::Decode(message) => MlError::Decode(message),
- ente_image::ImageError::Postprocess(message) => MlError::Postprocess(message),
+ ente_image::ImageError::Postprocess(message) => MlError::Image(message),
}
}
}
diff --git a/rust/crates/photos/src/ml/face/align.rs b/rust/crates/photos/src/ml/face/align.rs
index 83fcddd5d6..472523c4ae 100644
--- a/rust/crates/photos/src/ml/face/align.rs
+++ b/rust/crates/photos/src/ml/face/align.rs
@@ -76,9 +76,9 @@ fn take_rgb_image(decoded: &mut DecodedImage) -> MlResult<RgbImage> {
let expected_len = (decoded.dimensions.width as usize)
.checked_mul(decoded.dimensions.height as usize)
.and_then(|pixels| pixels.checked_mul(3))
- .ok_or_else(|| MlError::Preprocess("RGB source dimensions overflow".to_string()))?;
+ .ok_or_else(|| MlError::Image("RGB source dimensions overflow".to_string()))?;
if decoded.rgb.len() != expected_len {
- return Err(MlError::Preprocess(format!(
+ return Err(MlError::Image(format!(
"invalid RGB source length: expected {expected_len}, got {}",
decoded.rgb.len()
)));
@@ -143,7 +143,7 @@ fn estimate_similarity_transform(src_points: &[[f32; 2]; 5]) -> MlResult<Alignme
let rank = singular.iter().filter(|v| **v > 1e-6).count();
if rank == 0 {
- return Err(MlError::Postprocess(
+ return Err(MlError::Image(
"failed to estimate similarity transform (rank=0)".to_string(),
));
}
@@ -236,7 +236,7 @@ fn warp_face_image(source: &RgbImage, affine_matrix: &[[f32; 3]; 3]) -> MlResult
transform[2][1],
transform[2][2],
])
- .ok_or_else(|| MlError::Postprocess("invalid affine matrix projection".to_string()))?;
+ .ok_or_else(|| MlError::Image("invalid affine matrix projection".to_string()))?;
let mut output = RgbImage::from_pixel(FACE_INPUT_SIZE, FACE_INPUT_SIZE, Rgb([114, 114, 114]));
warp_into(
@@ -405,7 +405,10 @@ fn variance_2d(matrix: &[i32], rows: usize, cols: usize) -> f32 {
#[cfg(test)]
mod tests {
use super::run_face_alignment;
- use crate::ml::types::{DecodedImage, Dimensions};
+ use crate::ml::{
+ error::MlError,
+ types::{DecodedImage, Dimensions, FaceDetection},
+ };
#[test]
fn face_alignment_restores_the_decoded_rgb_allocation() {
@@ -425,4 +428,25 @@ mod tests {
assert_eq!(decoded.rgb, (0..12).collect::<Vec<_>>());
assert_eq!(decoded.rgb.as_ptr(), allocation);
}
+
+ #[test]
+ fn degenerate_landmarks_are_classified_as_an_image_error() {
+ let mut decoded = DecodedImage {
+ dimensions: Dimensions {
+ width: 2,
+ height: 2,
+ },
+ rgb: (0..12).collect(),
+ };
+ let detection = FaceDetection {
+ score: 1.0,
+ box_xyxy: [0.0, 0.0, 1.0, 1.0],
+ keypoints: [[0.5, 0.5]; 5],
+ };
+
+ let result = run_face_alignment(1, &mut decoded, vec![detection]);
+
+ assert!(matches!(result, Err(MlError::Image(_))));
+ assert_eq!(decoded.rgb, (0..12).collect::<Vec<_>>());
+ }
}
diff --git a/rust/crates/photos/src/ml/face/thumbnail.rs b/rust/crates/photos/src/ml/face/thumbnail.rs
index d87c9dfdc7..0513eadd8b 100644
--- a/rust/crates/photos/src/ml/face/thumbnail.rs
+++ b/rust/crates/photos/src/ml/face/thumbnail.rs
@@ -54,9 +54,8 @@ pub fn generate_face_thumbnails(
let mut results = Vec::with_capacity(face_boxes.len());
for (index, face_box) in face_boxes.iter().enumerate() {
- let crop = compute_crop_rect(face_box, image_width, image_height).map_err(|e| {
- MlError::InvalidRequest(format!("invalid face box at index {index}: {e}",))
- })?;
+ let crop = compute_crop_rect(face_box, image_width, image_height)
+ .map_err(|e| MlError::Image(format!("invalid face box at index {index}: {e}")))?;
let (target_width, target_height) =
face_thumbnail_dimensions(crop.output_width, crop.output_height)?;
let resized =
@@ -164,7 +163,7 @@ fn resize_crop_with_fir(
.resize_alg(ResizeAlg::Convolution(filter));
resizer
.resize(source, &mut resized, Some(&options))
- .map_err(|e| MlError::Postprocess(format!("failed to resize face thumbnail crop: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to resize face thumbnail crop: {e}")))?;
Ok(resized)
}
@@ -191,7 +190,7 @@ fn select_resize_filter(crop: &CropRect, target_width: u32, target_height: u32)
fn face_thumbnail_dimensions(width: u32, height: u32) -> MlResult<(u32, u32)> {
if width == 0 || height == 0 {
- return Err(MlError::Postprocess(
+ return Err(MlError::Image(
"cannot compute resize dimensions for zero-sized crop".to_string(),
));
}
@@ -227,7 +226,7 @@ fn face_thumbnail_dimensions(width: u32, height: u32) -> MlResult<(u32, u32)> {
fn scaled_dimension(value: u32, original_dimension: u32, target_dimension: u32) -> MlResult<u32> {
if original_dimension == 0 {
- return Err(MlError::Postprocess(
+ return Err(MlError::Image(
"cannot scale from a zero-sized dimension".to_string(),
));
}
@@ -237,7 +236,7 @@ fn scaled_dimension(value: u32, original_dimension: u32, target_dimension: u32)
let rounded = (numerator + (denominator / 2)) / denominator;
u32::try_from(rounded.max(1)).map_err(|_| {
- MlError::Postprocess(format!(
+ MlError::Image(format!(
"scaled dimension exceeds u32 for value={value}, original={original_dimension}, target={target_dimension}",
))
})
@@ -252,7 +251,10 @@ mod tests {
FaceBox, compute_crop_rect, face_thumbnail_dimensions, generate_face_thumbnails,
select_resize_filter,
};
- use crate::ml::types::{DecodedImage, Dimensions};
+ use crate::ml::{
+ error::MlError,
+ types::{DecodedImage, Dimensions},
+ };
#[test]
fn compute_crop_rect_matches_canvas_math_for_center_box() {
@@ -330,9 +332,10 @@ mod tests {
height: 0.3,
}];
- let result = generate_face_thumbnails(&decoded, &face_boxes);
-
- assert!(result.is_err());
+ assert!(matches!(
+ generate_face_thumbnails(&decoded, &face_boxes),
+ Err(MlError::Image(_))
+ ));
}
#[test]
diff --git a/rust/crates/photos/src/ml/pet/align.rs b/rust/crates/photos/src/ml/pet/align.rs
index acc51ced87..fab60bd089 100644
--- a/rust/crates/photos/src/ml/pet/align.rs
+++ b/rust/crates/photos/src/ml/pet/align.rs
@@ -244,7 +244,7 @@ fn crop_and_resize_rgb<'a>(
let clamped_w = w.min(src_w.saturating_sub(x));
let clamped_h = h.min(src_h.saturating_sub(y));
if clamped_w == 0 || clamped_h == 0 {
- return Err(MlError::Preprocess(
+ return Err(MlError::Image(
"crop_and_resize_rgb: crop region extends beyond source image".to_string(),
));
}
@@ -313,16 +313,16 @@ impl<'a> RgbRegion<'a> {
|| x > decoded.dimensions.width.saturating_sub(width)
|| y > decoded.dimensions.height.saturating_sub(height)
{
- return Err(MlError::Preprocess(
+ return Err(MlError::Image(
"RGB region extends beyond decoded image".to_string(),
));
}
let expected_len = (decoded.dimensions.width as usize)
.checked_mul(decoded.dimensions.height as usize)
.and_then(|pixels| pixels.checked_mul(3))
- .ok_or_else(|| MlError::Preprocess("decoded RGB dimensions overflow".to_string()))?;
+ .ok_or_else(|| MlError::Image("decoded RGB dimensions overflow".to_string()))?;
if decoded.rgb.len() < expected_len {
- return Err(MlError::Preprocess(
+ return Err(MlError::Image(
"decoded RGB buffer is shorter than its dimensions".to_string(),
));
}
diff --git a/rust/crates/photos/src/ml/pet/preprocess.rs b/rust/crates/photos/src/ml/pet/preprocess.rs
index 44e6a2d124..bff5d5cf51 100644
--- a/rust/crates/photos/src/ml/pet/preprocess.rs
+++ b/rust/crates/photos/src/ml/pet/preprocess.rs
@@ -130,7 +130,7 @@ fn relative_crop(decoded: &DecodedImage, box_xyxy: &[f32; 4]) -> MlResult<PixelC
let height = y2.saturating_sub(y1);
if width == 0 || height == 0 {
- return Err(MlError::Preprocess("crop region has zero area".to_string()));
+ return Err(MlError::Image("crop region has zero area".to_string()));
}
Ok(PixelCrop {
@@ -166,25 +166,23 @@ impl RgbCropResizer {
crop: PixelCrop,
) -> MlResult<&[u8]> {
if crop.width == 0 || crop.height == 0 {
- return Err(MlError::Preprocess(
- "crop dimensions cannot be zero".to_string(),
- ));
+ return Err(MlError::Image("crop dimensions cannot be zero".to_string()));
}
if crop.x > source_width.saturating_sub(crop.width)
|| crop.y > source_height.saturating_sub(crop.height)
{
- return Err(MlError::Preprocess(
+ return Err(MlError::Image(
"crop region extends beyond source image".to_string(),
));
}
let source = FirImageRef::new(source_width, source_height, rgb, PixelType::U8x3)
- .map_err(|e| MlError::Preprocess(format!("failed to create FIR source image: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to create FIR source image: {e}")))?;
let source = CroppedImage::new(&source, crop.x, crop.y, crop.width, crop.height)
- .map_err(|e| MlError::Preprocess(format!("failed to create FIR crop view: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to create FIR crop view: {e}")))?;
self.resizer
.resize(&source, &mut self.resized, Some(&self.options))
- .map_err(|e| MlError::Preprocess(format!("failed to resize RGB crop: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to resize RGB crop: {e}")))?;
Ok(self.resized.buffer())
}
diff --git a/rust/crates/photos/src/ml/preprocess.rs b/rust/crates/photos/src/ml/preprocess.rs
index c4c5919c85..1b45c8c38a 100644
--- a/rust/crates/photos/src/ml/preprocess.rs
+++ b/rust/crates/photos/src/ml/preprocess.rs
@@ -73,7 +73,7 @@ impl YoloInput {
pub(crate) fn preprocess_yolo(decoded: &DecodedImage) -> MlResult<YoloInput> {
if decoded.dimensions.width == 0 || decoded.dimensions.height == 0 {
- return Err(MlError::Preprocess(
+ return Err(MlError::Image(
"image dimensions cannot be zero".to_string(),
));
}
@@ -90,14 +90,14 @@ pub(crate) fn preprocess_yolo(decoded: &DecodedImage) -> MlResult<YoloInput> {
decoded.rgb.as_slice(),
PixelType::U8x3,
)
- .map_err(|e| MlError::Preprocess(format!("failed to create FIR source image: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to create FIR source image: {e}")))?;
let mut resized_image = FirImage::new(scaled_width, scaled_height, PixelType::U8x3);
let mut resizer = Resizer::new();
let options = ResizeOptions::new().resize_alg(ResizeAlg::Interpolation(FilterType::Bilinear));
resizer
.resize(&src_image, &mut resized_image, Some(&options))
- .map_err(|e| MlError::Preprocess(format!("failed to resize YOLO image input: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to resize YOLO image input: {e}")))?;
let scaled_width_usize = scaled_width as usize;
let scaled_height_usize = scaled_height as usize;
@@ -134,7 +134,7 @@ pub(crate) fn preprocess_yolo(decoded: &DecodedImage) -> MlResult<YoloInput> {
pub(crate) fn preprocess_clip(decoded: &DecodedImage) -> MlResult<Vec<f32>> {
if decoded.dimensions.width == 0 || decoded.dimensions.height == 0 {
- return Err(MlError::Preprocess(
+ return Err(MlError::Image(
"image dimensions cannot be zero".to_string(),
));
}
@@ -151,7 +151,7 @@ pub(crate) fn preprocess_clip(decoded: &DecodedImage) -> MlResult<Vec<f32>> {
decoded.rgb.as_slice(),
PixelType::U8x3,
)
- .map_err(|e| MlError::Preprocess(format!("failed to create FIR source image: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to create FIR source image: {e}")))?;
let start_x = (scaled_width.saturating_sub(CLIP_IMAGE_INPUT_SIZE as u32) / 2) as f64;
let start_y = (scaled_height.saturating_sub(CLIP_IMAGE_INPUT_SIZE as u32) / 2) as f64;
@@ -173,7 +173,7 @@ pub(crate) fn preprocess_clip(decoded: &DecodedImage) -> MlResult<Vec<f32>> {
.crop(crop_left, crop_top, crop_width, crop_height);
resizer
.resize(&src_image, &mut resized_image, Some(&options))
- .map_err(|e| MlError::Preprocess(format!("failed to resize CLIP image input: {e}")))?;
+ .map_err(|e| MlError::Image(format!("failed to resize CLIP image input: {e}")))?;
let resized = resized_image.buffer();
@@ -201,7 +201,7 @@ mod tests {
images::{Image as FirImage, ImageRef as FirImageRef},
};
- use super::{YOLO_INPUT_SIZE, YoloInput, preprocess_clip};
+ use super::{YOLO_INPUT_SIZE, YoloInput, preprocess_clip, preprocess_yolo};
use crate::ml::{
clip::CLIP_IMAGE_INPUT_SIZE,
error::{MlError, MlResult},
@@ -209,6 +209,14 @@ mod tests {
types::{DecodedImage, Dimensions},
};
+ #[test]
+ fn zero_sized_images_are_classified_as_image_errors() {
+ let decoded = test_image(0, 0);
+
+ assert!(matches!(preprocess_yolo(&decoded), Err(MlError::Image(_))));
+ assert!(matches!(preprocess_clip(&decoded), Err(MlError::Image(_))));
+ }
+
#[test]
fn yolo_geometry_matches_the_previous_letterbox_correction() {
let input = YoloInput {
diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.ts
index 125cd68479..2cbe1c6269 100644
--- a/web/packages/base/types/ipc.ts
+++ b/web/packages/base/types/ipc.ts
@@ -743,7 +743,7 @@ export interface ElectronMLWorker {
*/
analyzeImage: (
request: MLWorkerAnalyzeImageRequest,
- ) => Promise<MLWorkerAnalyzeImageResult>;
+ ) => Promise<MLWorkerAnalyzeImageResponse>;
/**
* Release the (pinned) native ML sessions.
@@ -809,6 +809,32 @@ export interface MLWorkerAnalyzeImageRequest {
generateFaceCrops: boolean;
}
+/**
+ * Broad category for an error encountered while analyzing an image.
+ *
+ * Only `image` indicates a permanent problem inherent to the image. `init`
+ * blocks indexing for the lifetime of the utility process, while `ort` and
+ * `misc` remain retryable. None of those other kinds should permanently mark
+ * the file as unindexable.
+ */
+export type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";
+
+/** An error returned by the Electron ML utility process. */
+export interface MLWorkerAnalyzeImageError {
+ kind: MLWorkerAnalyzeImageErrorKind;
+ message: string;
+}
+
+/**
+ * Result envelope for an image analysis request.
+ *
+ * This explicit envelope is used instead of throwing a custom `Error` because
+ * Comlink only preserves an error's name, message and stack.
+ */
+export type MLWorkerAnalyzeImageResponse =
+ | { ok: true; result: MLWorkerAnalyzeImageResult }
+ | { ok: false; error: MLWorkerAnalyzeImageError };
+
/**
* A face detected by {@link ElectronMLWorker.analyzeImage}.
*
diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts
index 5b2cbdf350..283459e127 100644
--- a/web/packages/new/photos/services/ml/worker.ts
+++ b/web/packages/new/photos/services/ml/worker.ts
@@ -7,6 +7,7 @@ import log from "ente-base/log";
import { logUnhandledErrorsAndRejectionsInWorker } from "ente-base/log-web";
import type {
ElectronMLWorker,
+ MLWorkerAnalyzeImageErrorKind,
MLWorkerAnalyzeImageRequest,
MLWorkerAnalyzeImageResult,
} from "ente-base/types/ipc";
@@ -186,6 +187,8 @@ export class MLWorker {
* closes while we're indexing.
*/
index() {
+ if (_blockingMLInitError) return Promise.resolve(0);
+
const nextIdle = new Promise<number>((resolve, reject) =>
this.onNextIdles.push({ resolve, reject }),
);
@@ -217,6 +220,8 @@ export class MLWorker {
* us and won't need to download the file from remote.
*/
onUpload(file: EnteFile, processableUploadItem: ProcessableUploadItem) {
+ if (_blockingMLInitError) return;
+
// Add the recently uploaded file to the live indexing queue.
this.liveQ.push({
file,
@@ -265,16 +270,20 @@ export class MLWorker {
return [];
});
- this.countSinceLastIdle += items.length;
-
// If there is items remaining,
if (items.length > 0) {
// Index them.
- const indexedCount = await indexNextBatch(
+ const { indexedCount, blockingInitError } = await indexNextBatch(
items,
this.electron!,
this.delegate,
);
+ if (blockingInitError) {
+ this.suspendForMLInitError(blockingInitError);
+ return;
+ }
+
+ this.countSinceLastIdle += items.length;
if (indexedCount > 0) {
// We made some progress, so there are no complete blockers
// (e.g. network being offline). Reset the idle duration and
@@ -308,7 +317,24 @@ export class MLWorker {
log.warn("Failed to release the native ML runtime", e);
});
- // Resolve any awaiting promises returned from `index`.
+ this.resolvePendingIndexRequests();
+ }
+
+ /** Stop indexing until this web worker (and its utility process) is replaced. */
+ private suspendForMLInitError(error: MLAnalyzeError) {
+ log.error(
+ "Suspending ML indexing because the native runtime failed to initialize",
+ error,
+ );
+ this.liveQ = [];
+ this.state = "idle";
+ this.idleTimeout = undefined;
+ this.delegate?.workerDidUpdateStatus();
+ this.resolvePendingIndexRequests();
+ }
+
+ /** Resolve callers waiting for the worker to next become idle. */
+ private resolvePendingIndexRequests() {
const onNextIdles = this.onNextIdles;
const countSinceLastIdle = this.countSinceLastIdle;
this.onNextIdles = [];
@@ -395,7 +421,7 @@ logUnhandledErrorsAndRejectionsInWorker();
/**
* Index the given batch of items.
*
- * @returns the count of items which were indexed.
+ * @returns the number of indexed items and any process-blocking init error.
*/
const indexNextBatch = async (
items: IndexableItem[],
@@ -407,7 +433,7 @@ const indexNextBatch = async (
// were able to upload just a bit ago but don't have network now.
if (!self.navigator.onLine) {
log.info("Skipping ML indexing since we are not online");
- return 0;
+ return { indexedCount: 0 };
}
// Keep track if any of the items failed.
@@ -417,7 +443,7 @@ const indexNextBatch = async (
const tasks = new Array<Promise<void> | undefined>(4).fill(undefined);
let i = 0;
- while (i < items.length) {
+ while (i < items.length && !_blockingMLInitError) {
for (let j = 0; j < tasks.length; j++) {
if (i < items.length && !tasks[j]) {
// Use an IIFE to capture the value of j at the time of
@@ -430,6 +456,9 @@ const indexNextBatch = async (
.catch((e: unknown) => {
const f = fileLogID(item.file);
log.error(`Failed to index ${f}`, e);
+ if (isMLInitError(e)) {
+ _blockingMLInitError ??= e;
+ }
failureCount++;
tasks[j] = undefined;
}))(items[i++]!, j);
@@ -454,16 +483,16 @@ const indexNextBatch = async (
// Clear any cached CLIP indexes, since now we might have new ones.
clearCachedCLIPIndexes();
- const indexedCount = items.length - failureCount;
+ const attemptedCount = i;
+ const indexedCount = attemptedCount - failureCount;
log.info(
failureCount > 0
? `Indexed ${indexedCount} files (${failureCount} failed)`
- : `Indexed ${items.length} files`,
+ : `Indexed ${attemptedCount} files`,
);
- // Return the count of indexed files.
- return indexedCount;
+ return { indexedCount, blockingInitError: _blockingMLInitError };
};
/**
@@ -626,8 +655,11 @@ const index = async (
electron,
);
} catch (e) {
+ // Only failures inherent to the image (including the JPEG conversion
+ // fallback below) are permanent. Runtime and infrastructure failures
+ // should be retried after the underlying issue is resolved.
// See: [Note: Transient and permanent indexing failures]
- await markIndexingFailed(fileID);
+ if (isMLImageError(e)) await markIndexingFailed(fileID);
throw e;
}
@@ -747,19 +779,52 @@ const analyzeImageWithConversionFallback = async (
`Native decode of ${fileLogID(file)} failed, retrying with a JPEG conversion`,
e,
);
- const blob = await fetchRenderableBlob(
- file,
- processableUploadItem,
- electron,
- );
- const bytes = new Uint8Array(await blob.arrayBuffer());
+ let bytes: Uint8Array;
+ try {
+ const blob = await fetchRenderableBlob(
+ file,
+ processableUploadItem,
+ electron,
+ );
+ bytes = new Uint8Array(await blob.arrayBuffer());
+ } catch (conversionError) {
+ // Fetching the source for conversion can still fail transiently.
+ // Do not turn such a download error into a permanent image error.
+ if (isNetworkDownloadError(conversionError)) throw conversionError;
+
+ const message =
+ conversionError instanceof Error
+ ? conversionError.message
+ : String(conversionError);
+ throw new MLAnalyzeError(
+ "image",
+ `JPEG conversion fallback failed: ${message}`,
+ );
+ }
return await enqueueAnalyzeImage(electron, { ...request, bytes });
}
};
-/** `true` if the error looks like the native side failing to decode an image. */
+/** `true` for a permanent failure inherent to the image being analyzed. */
+const isMLImageError = (e: unknown): e is MLAnalyzeError =>
+ e instanceof MLAnalyzeError && e.kind == "image";
+
+/** `true` specifically for the Rust image decoder failure that permits fallback. */
const isMLDecodeError = (e: unknown) =>
- e instanceof Error && e.message.startsWith("Decode: ");
+ isMLImageError(e) && e.message.startsWith("Decode: ");
+
+const isMLInitError = (e: unknown): e is MLAnalyzeError =>
+ e instanceof MLAnalyzeError && e.kind == "init";
+
+class MLAnalyzeError extends Error {
+ constructor(
+ public readonly kind: MLWorkerAnalyzeImageErrorKind,
+ message: string,
+ ) {
+ super(message);
+ this.name = "MLAnalyzeError";
+ }
+}
/**
* A promise queue serializing calls to {@link ElectronMLWorker.analyzeImage}.
@@ -772,13 +837,23 @@ const isMLDecodeError = (e: unknown) =>
*/
let _analyzeImageQueue: Promise<unknown> = Promise.resolve();
+/** The native addon is loaded only once, so this error cannot recover here. */
+let _blockingMLInitError: MLAnalyzeError | undefined;
+
const enqueueAnalyzeImage = (
electron: ElectronMLWorker,
request: MLWorkerAnalyzeImageRequest,
): Promise<MLWorkerAnalyzeImageResult> => {
- const result = _analyzeImageQueue.then(() =>
- electron.analyzeImage(request),
- );
+ const result = _analyzeImageQueue.then(async () => {
+ const response = await electron.analyzeImage(request);
+ if (!response.ok) {
+ throw new MLAnalyzeError(
+ response.error.kind,
+ response.error.message,
+ );
+ }
+ return response.result;
+ });
_analyzeImageQueue = result.catch(() => undefined);
return result;
};