Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Don't fail full indexing on just face crop issue

laurenspriem8c5420040e95
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index c79e947f23..13a4f8bc85 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -129,8 +129,9 @@ pub struct AnalyzeImageResult {
pub decoded_image_size: Dimensions,
pub faces: Option<Vec<FaceResult>>,
/// JPEG bytes, index-aligned with `faces`; present only when requested
- /// via `generate_face_crops`.
- pub face_crops: Option<Vec<Buffer>>,
+ /// via `generate_face_crops`. Crop generation is best effort: a face
+ /// whose crop could not be generated has a null slot.
+ pub face_crops: Option<Vec<Option<Buffer>>>,
pub clip: Option<ClipResult>,
pub pet_faces: Option<Vec<PetFaceResult>>,
pub pet_bodies: Option<Vec<PetBodyResult>>,
@@ -316,9 +317,12 @@ fn to_napi_analyze_image_result(result: shared_indexing::AnalyzeImageResult) ->
faces: result
.faces
.map(|faces| faces.into_iter().map(to_napi_face_result).collect()),
- face_crops: result
- .face_crops
- .map(|crops| crops.into_iter().map(Buffer::from).collect()),
+ face_crops: result.face_crops.map(|crops| {
+ crops
+ .into_iter()
+ .map(|crop| crop.map(Buffer::from))
+ .collect()
+ }),
clip: result.clip.map(|clip| ClipResult {
embedding: clip.embedding.into(),
}),
diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rs
index 8664d4b529..9b81e1b913 100644
--- a/rust/crates/photos/src/ml/indexing.rs
+++ b/rust/crates/photos/src/ml/indexing.rs
@@ -1,6 +1,7 @@
use crate::ml::{
clip::{run_clip_image, run_clip_text_query, tokenize_clip_text as tokenize_clip_text_impl},
error::{MlError, MlResult},
+ events,
face::{
run_face_alignment, run_face_detection, run_face_embedding,
thumbnail::{FaceBox, generate_face_thumbnails},
@@ -41,8 +42,9 @@ pub struct AnalyzeImageResult {
pub decoded_image_size: Dimensions,
pub faces: Option<Vec<FaceResult>>,
/// Index-aligned with `faces`; present only when requested via
- /// `generate_face_crops`.
- pub face_crops: Option<Vec<Vec<u8>>>,
+ /// `generate_face_crops`. Crop generation is best effort: a face whose
+ /// crop could not be generated has `None` in its slot.
+ pub face_crops: Option<Vec<Option<Vec<u8>>>>,
pub clip: Option<ClipResult>,
pub pet_faces: Option<Vec<PetFaceResult>>,
pub pet_bodies: Option<Vec<PetBodyResult>>,
@@ -125,25 +127,37 @@ pub fn analyze_image(req: AnalyzeImageRequest) -> MlResult<AnalyzeImageResult> {
None
};
+ // Crops are a cosmetic side artifact of indexing, so their generation
+ // is best effort: a face whose crop fails is skipped (with a `None`
+ // slot to keep the index alignment) instead of failing the analysis.
let face_crops = if generate_face_crops {
- faces
- .as_ref()
- .map(|face_results| {
- let face_boxes = face_results
- .iter()
- .map(|face| {
- let [x_min, y_min, x_max, y_max] = face.detection.box_xyxy;
- FaceBox {
- x: x_min,
- y: y_min,
- width: x_max - x_min,
- height: y_max - y_min,
+ faces.as_ref().map(|face_results| {
+ face_results
+ .iter()
+ .map(|face| {
+ let [x_min, y_min, x_max, y_max] = face.detection.box_xyxy;
+ let face_box = FaceBox {
+ x: x_min,
+ y: y_min,
+ width: x_max - x_min,
+ height: y_max - y_min,
+ };
+ match generate_face_thumbnails(&decoded, std::slice::from_ref(&face_box)) {
+ Ok(mut crops) => crops.pop(),
+ Err(error) => {
+ events::record(
+ events::Severity::Warning,
+ format!(
+ "skipping face crop for face {}: {error}",
+ face.face_id
+ ),
+ );
+ None
}
- })
- .collect::<Vec<_>>();
- generate_face_thumbnails(&decoded, &face_boxes)
- })
- .transpose()?
+ }
+ })
+ .collect::<Vec<_>>()
+ })
} else {
None
};
diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rs
index 62cf6d9159..acf450a814 100644
--- a/rust/crates/photos/tests/support/ml_indexing.rs
+++ b/rust/crates/photos/tests/support/ml_indexing.rs
@@ -292,7 +292,12 @@ impl MlIndexingTestContext {
match analyze_image(req) {
Ok(result) => {
let face_count = result.faces.as_ref().map_or(0, Vec::len);
- let crop_count = result.face_crops.as_ref().map_or(0, Vec::len);
+ // Count generated crops only; a `None` slot (best-effort
+ // skip) must not happen for the test corpus.
+ let crop_count = result
+ .face_crops
+ .as_ref()
+ .map_or(0, |crops| crops.iter().flatten().count());
if crop_count != face_count {
failures.push(format!(
"{file_id}: face crop count {crop_count} does not match face count {face_count}"
diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.ts
index 27f17ff944..e5b3423e01 100644
--- a/web/packages/base/types/ipc.ts
+++ b/web/packages/base/types/ipc.ts
@@ -877,9 +877,10 @@ export interface MLWorkerAnalyzeImageResult {
/**
* JPEG blobs of the crops of detected faces, index-aligned with
* {@link faces}. Present when the request had {@link generateFaceCrops}
- * set.
+ * set. Crop generation is best effort: a face whose crop could not be
+ * generated has a `null` in its slot.
*/
- faceCrops?: Uint8Array<ArrayBuffer>[];
+ faceCrops?: (Uint8Array<ArrayBuffer> | null)[];
/** Present when the request had {@link runClip} set. */
clip?: {
/** The (normalized) CLIP image embedding (512 floating point values). */
diff --git a/web/packages/new/photos/services/ml/crop.ts b/web/packages/new/photos/services/ml/crop.ts
index 9eac4d2d11..50f59a903f 100644
--- a/web/packages/new/photos/services/ml/crop.ts
+++ b/web/packages/new/photos/services/ml/crop.ts
@@ -40,7 +40,8 @@ export const regenerateFaceCrops = async (
* indexing an image.
*
* @param faceCrops JPEG blobs of the face crops, index-aligned with the faces
- * in {@link faceIndex}.
+ * in {@link faceIndex}. Faces whose crop the native side could not generate
+ * have a `null` in their slot, and are skipped.
*
* @param faceIndex The {@link FaceIndex} containing information about the
* faces detected in the given image.
@@ -49,7 +50,7 @@ export const regenerateFaceCrops = async (
* from the {@link BlobCache} named "face-crops".
*/
export const saveFaceCropBlobs = async (
- faceCrops: Uint8Array<ArrayBuffer>[],
+ faceCrops: (Uint8Array<ArrayBuffer> | null)[],
faceIndex: FaceIndex,
) => {
const cache = await blobCache("face-crops");