π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Don't fail full indexing on just face crop issue
laurenspriem8c5420040e95
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex c79e947f23..13a4f8bc85 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -129,8 +129,9 @@ pub struct AnalyzeImageResult {pub decoded_image_size: Dimensions,pub faces: Option<Vec<FaceResult>>,/// JPEG bytes, index-aligned with `faces`; present only when requested- /// via `generate_face_crops`.- pub face_crops: Option<Vec<Buffer>>,+ /// via `generate_face_crops`. Crop generation is best effort: a face+ /// whose crop could not be generated has a null slot.+ pub face_crops: Option<Vec<Option<Buffer>>>,pub clip: Option<ClipResult>,pub pet_faces: Option<Vec<PetFaceResult>>,pub pet_bodies: Option<Vec<PetBodyResult>>,@@ -316,9 +317,12 @@ fn to_napi_analyze_image_result(result: shared_indexing::AnalyzeImageResult) ->faces: result.faces.map(|faces| faces.into_iter().map(to_napi_face_result).collect()),- face_crops: result- .face_crops- .map(|crops| crops.into_iter().map(Buffer::from).collect()),+ face_crops: result.face_crops.map(|crops| {+ crops+ .into_iter()+ .map(|crop| crop.map(Buffer::from))+ .collect()+ }),clip: result.clip.map(|clip| ClipResult {embedding: clip.embedding.into(),}),diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex 8664d4b529..9b81e1b913 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -1,6 +1,7 @@use crate::ml::{clip::{run_clip_image, run_clip_text_query, tokenize_clip_text as tokenize_clip_text_impl},error::{MlError, MlResult},+ events,face::{run_face_alignment, run_face_detection, run_face_embedding,thumbnail::{FaceBox, generate_face_thumbnails},@@ -41,8 +42,9 @@ pub struct AnalyzeImageResult {pub decoded_image_size: Dimensions,pub faces: Option<Vec<FaceResult>>,/// Index-aligned with `faces`; present only when requested via- /// `generate_face_crops`.- pub face_crops: Option<Vec<Vec<u8>>>,+ /// `generate_face_crops`. Crop generation is best effort: a face whose+ /// crop could not be generated has `None` in its slot.+ pub face_crops: Option<Vec<Option<Vec<u8>>>>,pub clip: Option<ClipResult>,pub pet_faces: Option<Vec<PetFaceResult>>,pub pet_bodies: Option<Vec<PetBodyResult>>,@@ -125,25 +127,37 @@ pub fn analyze_image(req: AnalyzeImageRequest) -> MlResult<AnalyzeImageResult> {None};+ // Crops are a cosmetic side artifact of indexing, so their generation+ // is best effort: a face whose crop fails is skipped (with a `None`+ // slot to keep the index alignment) instead of failing the analysis.let face_crops = if generate_face_crops {- faces- .as_ref()- .map(|face_results| {- let face_boxes = face_results- .iter()- .map(|face| {- let [x_min, y_min, x_max, y_max] = face.detection.box_xyxy;- FaceBox {- x: x_min,- y: y_min,- width: x_max - x_min,- height: y_max - y_min,+ faces.as_ref().map(|face_results| {+ face_results+ .iter()+ .map(|face| {+ let [x_min, y_min, x_max, y_max] = face.detection.box_xyxy;+ let face_box = FaceBox {+ x: x_min,+ y: y_min,+ width: x_max - x_min,+ height: y_max - y_min,+ };+ match generate_face_thumbnails(&decoded, std::slice::from_ref(&face_box)) {+ Ok(mut crops) => crops.pop(),+ Err(error) => {+ events::record(+ events::Severity::Warning,+ format!(+ "skipping face crop for face {}: {error}",+ face.face_id+ ),+ );+ None}- })- .collect::<Vec<_>>();- generate_face_thumbnails(&decoded, &face_boxes)- })- .transpose()?+ }+ })+ .collect::<Vec<_>>()+ })} else {None};diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rsindex 62cf6d9159..acf450a814 100644--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@ -292,7 +292,12 @@ impl MlIndexingTestContext {match analyze_image(req) {Ok(result) => {let face_count = result.faces.as_ref().map_or(0, Vec::len);- let crop_count = result.face_crops.as_ref().map_or(0, Vec::len);+ // Count generated crops only; a `None` slot (best-effort+ // skip) must not happen for the test corpus.+ let crop_count = result+ .face_crops+ .as_ref()+ .map_or(0, |crops| crops.iter().flatten().count());if crop_count != face_count {failures.push(format!("{file_id}: face crop count {crop_count} does not match face count {face_count}"diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.tsindex 27f17ff944..e5b3423e01 100644--- a/web/packages/base/types/ipc.ts+++ b/web/packages/base/types/ipc.ts@@ -877,9 +877,10 @@ export interface MLWorkerAnalyzeImageResult {/*** JPEG blobs of the crops of detected faces, index-aligned with* {@link faces}. Present when the request had {@link generateFaceCrops}- * set.+ * set. Crop generation is best effort: a face whose crop could not be+ * generated has a `null` in its slot.*/- faceCrops?: Uint8Array<ArrayBuffer>[];+ faceCrops?: (Uint8Array<ArrayBuffer> | null)[];/** Present when the request had {@link runClip} set. */clip?: {/** The (normalized) CLIP image embedding (512 floating point values). */diff --git a/web/packages/new/photos/services/ml/crop.ts b/web/packages/new/photos/services/ml/crop.tsindex 9eac4d2d11..50f59a903f 100644--- a/web/packages/new/photos/services/ml/crop.ts+++ b/web/packages/new/photos/services/ml/crop.ts@@ -40,7 +40,8 @@ export const regenerateFaceCrops = async (* indexing an image.** @param faceCrops JPEG blobs of the face crops, index-aligned with the faces- * in {@link faceIndex}.+ * in {@link faceIndex}. Faces whose crop the native side could not generate+ * have a `null` in their slot, and are skipped.** @param faceIndex The {@link FaceIndex} containing information about the* faces detected in the given image.@@ -49,7 +50,7 @@ export const regenerateFaceCrops = async (* from the {@link BlobCache} named "face-crops".*/export const saveFaceCropBlobs = async (- faceCrops: Uint8Array<ArrayBuffer>[],+ faceCrops: (Uint8Array<ArrayBuffer> | null)[],faceIndex: FaceIndex,) => {const cache = await blobCache("face-crops");