π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Cap ML detections after greedy NMS
laurenspriem8994776b2d0d
diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rsindex 61ff675c3a..b01fe3aae3 100644--- a/rust/crates/photos/src/ml/face/detect.rs+++ b/rust/crates/photos/src/ml/face/detect.rs@@ -1,6 +1,7 @@use crate::ml::{error::{MlError, MlResult},onnx,+ postprocess::MAX_DETECTIONS_PER_IMAGE,preprocess::{YOLO_INPUT_SIZE, YoloInput},runtime::MlRuntimeView,types::FaceDetection,@@ -99,37 +100,31 @@ fn postprocess_face_tensor<T: onnx::FloatTensorData>(});}- Ok(naive_non_max_suppression(detections, IOU_THRESHOLD))+ Ok(greedy_non_max_suppression(detections, IOU_THRESHOLD))}-fn naive_non_max_suppression(+fn greedy_non_max_suppression(mut detections: Vec<FaceDetection>,iou_threshold: f32,) -> Vec<FaceDetection> {detections.sort_by(|a, b| b.score.total_cmp(&a.score));- let mut suppressed = vec![false; detections.len()];- for i in 0..detections.len() {- if suppressed[i] {+ let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));+ for detection in detections {+ if retained+ .iter()+ .any(|existing| calculate_iou(existing, &detection) >= iou_threshold)+ {continue;}- for j in (i + 1)..detections.len() {- if suppressed[j] {- continue;- }- let iou = calculate_iou(&detections[i], &detections[j]);- if iou >= iou_threshold {- suppressed[j] = true;- }+ retained.push(detection);+ if retained.len() == MAX_DETECTIONS_PER_IMAGE {+ break;}}- detections- .into_iter()- .enumerate()- .filter_map(|(index, detection)| (!suppressed[index]).then_some(detection))- .collect()+ retained}fn calculate_iou(a: &FaceDetection, b: &FaceDetection) -> f32 {@@ -156,3 +151,54 @@ fn calculate_iou(a: &FaceDetection, b: &FaceDetection) -> f32 {}intersection_area / union_area}++#[cfg(test)]+mod tests {+ use super::{+ FaceDetection, IOU_THRESHOLD, MAX_DETECTIONS_PER_IMAGE, greedy_non_max_suppression,+ };++ #[test]+ fn face_nms_retains_the_highest_scoring_hundred_detections() {+ let detections = (0..=MAX_DETECTIONS_PER_IMAGE)+ .map(|index| FaceDetection {+ score: index as f32,+ box_xyxy: separated_box(index),+ keypoints: [[0.0; 2]; 5],+ })+ .collect();++ let retained = greedy_non_max_suppression(detections, IOU_THRESHOLD);++ assert_eq!(retained.len(), MAX_DETECTIONS_PER_IMAGE);+ assert_eq!(retained.first().unwrap().score, 100.0);+ assert_eq!(retained.last().unwrap().score, 1.0);+ }++ #[test]+ fn face_nms_suppresses_lower_scoring_overlaps() {+ let retained = greedy_non_max_suppression(+ vec![+ FaceDetection {+ score: 0.8,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ keypoints: [[0.0; 2]; 5],+ },+ FaceDetection {+ score: 0.9,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ keypoints: [[0.0; 2]; 5],+ },+ ],+ IOU_THRESHOLD,+ );++ assert_eq!(retained.len(), 1);+ assert_eq!(retained[0].score, 0.9);+ }++ fn separated_box(index: usize) -> [f32; 4] {+ let x = index as f32 * 2.0;+ [x, 0.0, x + 1.0, 1.0]+ }+}diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rsindex b7ec7c2abf..89f77c479c 100644--- a/rust/crates/photos/src/ml/pet/detect.rs+++ b/rust/crates/photos/src/ml/pet/detect.rs@@ -1,6 +1,7 @@use crate::ml::{error::{MlError, MlResult},onnx,+ postprocess::MAX_DETECTIONS_PER_IMAGE,preprocess::{YOLO_INPUT_SIZE, YoloInput},runtime::MlRuntimeView,types::{PetBodyDetection, PetFaceDetection},@@ -163,7 +164,7 @@ fn postprocess_pet_face_tensor<T: onnx::FloatTensorData>(});}- Ok(naive_nms_pet_face(detections, PET_FACE_IOU_THRESHOLD))+ Ok(greedy_nms_pet_face(detections, PET_FACE_IOU_THRESHOLD))}pub(crate) fn run_pet_body_detection(@@ -233,7 +234,7 @@ fn postprocess_pet_body_tensor<T: onnx::FloatTensorData>(});}- Ok(naive_nms_pet_body(detections, BODY_IOU_THRESHOLD))+ Ok(greedy_nms_pet_body(detections, BODY_IOU_THRESHOLD))}fn winning_pet_body_class<T: onnx::FloatTensorData>(@@ -291,69 +292,152 @@ fn calculate_iou_4(a: &[f32; 4], b: &[f32; 4]) -> f32 {if union <= 0.0 { 0.0 } else { inter / union }}-fn naive_nms_pet_face(+fn greedy_nms_pet_face(mut detections: Vec<PetFaceDetection>,iou_threshold: f32,) -> Vec<PetFaceDetection> {detections.sort_by(|a, b| b.score.total_cmp(&a.score));- let n = detections.len();- let mut suppressed = vec![false; n];- for i in 0..n {- if suppressed[i] {++ let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));+ for detection in detections {+ if retained.iter().any(|existing: &PetFaceDetection| {+ existing.class_id == detection.class_id+ && calculate_iou_4(&existing.box_xyxy, &detection.box_xyxy) >= iou_threshold+ }) {continue;}- for j in (i + 1)..n {- if suppressed[j] {- continue;- }- if detections[i].class_id == detections[j].class_id- && calculate_iou_4(&detections[i].box_xyxy, &detections[j].box_xyxy)- >= iou_threshold- {- suppressed[j] = true;- }++ retained.push(detection);+ if retained.len() == MAX_DETECTIONS_PER_IMAGE {+ break;}}- detections- .into_iter()- .zip(suppressed)- .filter_map(|(d, s)| if s { None } else { Some(d) })- .collect()++ retained}-fn naive_nms_pet_body(+fn greedy_nms_pet_body(mut detections: Vec<PetBodyDetection>,iou_threshold: f32,) -> Vec<PetBodyDetection> {detections.sort_by(|a, b| b.score.total_cmp(&a.score));- let n = detections.len();- let mut suppressed = vec![false; n];- for i in 0..n {- if suppressed[i] {++ let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));+ for detection in detections {+ if retained.iter().any(|existing: &PetBodyDetection| {+ existing.coco_class == detection.coco_class+ && calculate_iou_4(&existing.box_xyxy, &detection.box_xyxy) >= iou_threshold+ }) {continue;}- for j in (i + 1)..n {- if suppressed[j] {- continue;- }- if detections[i].coco_class == detections[j].coco_class- && calculate_iou_4(&detections[i].box_xyxy, &detections[j].box_xyxy)- >= iou_threshold- {- suppressed[j] = true;- }++ retained.push(detection);+ if retained.len() == MAX_DETECTIONS_PER_IMAGE {+ break;}}- detections- .into_iter()- .zip(suppressed)- .filter_map(|(d, s)| if s { None } else { Some(d) })- .collect()++ retained}#[cfg(test)]mod tests {- use super::{BODY_MIN_SCORE, COCO_CAT, COCO_DOG, winning_pet_body_class};+ use super::{+ BODY_IOU_THRESHOLD, BODY_MIN_SCORE, COCO_CAT, COCO_DOG, MAX_DETECTIONS_PER_IMAGE,+ PET_FACE_IOU_THRESHOLD, PET_SPECIES_CAT, PET_SPECIES_DOG, PetBodyDetection,+ PetFaceDetection, greedy_nms_pet_body, greedy_nms_pet_face, winning_pet_body_class,+ };++ #[test]+ fn pet_nms_retains_the_highest_scoring_hundred_detections() {+ let faces = (0..=MAX_DETECTIONS_PER_IMAGE)+ .map(|index| PetFaceDetection {+ score: index as f32,+ box_xyxy: separated_box(index),+ keypoints: [[0.0; 2]; 3],+ class_id: PET_SPECIES_DOG,+ })+ .collect();+ let bodies = (0..=MAX_DETECTIONS_PER_IMAGE)+ .map(|index| PetBodyDetection {+ score: index as f32,+ box_xyxy: separated_box(index),+ coco_class: COCO_DOG,+ })+ .collect();++ let retained_faces = greedy_nms_pet_face(faces, PET_FACE_IOU_THRESHOLD);+ let retained_bodies = greedy_nms_pet_body(bodies, BODY_IOU_THRESHOLD);++ assert_eq!(retained_faces.len(), MAX_DETECTIONS_PER_IMAGE);+ assert_eq!(retained_faces.first().unwrap().score, 100.0);+ assert_eq!(retained_faces.last().unwrap().score, 1.0);+ assert_eq!(retained_bodies.len(), MAX_DETECTIONS_PER_IMAGE);+ assert_eq!(retained_bodies.first().unwrap().score, 100.0);+ assert_eq!(retained_bodies.last().unwrap().score, 1.0);+ }++ #[test]+ fn pet_nms_only_suppresses_overlaps_within_the_same_class() {+ let retained_faces = greedy_nms_pet_face(+ vec![+ PetFaceDetection {+ score: 0.8,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ keypoints: [[0.0; 2]; 3],+ class_id: PET_SPECIES_DOG,+ },+ PetFaceDetection {+ score: 0.9,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ keypoints: [[0.0; 2]; 3],+ class_id: PET_SPECIES_DOG,+ },+ PetFaceDetection {+ score: 0.7,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ keypoints: [[0.0; 2]; 3],+ class_id: PET_SPECIES_CAT,+ },+ ],+ PET_FACE_IOU_THRESHOLD,+ );+ let retained_bodies = greedy_nms_pet_body(+ vec![+ PetBodyDetection {+ score: 0.8,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ coco_class: COCO_DOG,+ },+ PetBodyDetection {+ score: 0.9,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ coco_class: COCO_DOG,+ },+ PetBodyDetection {+ score: 0.7,+ box_xyxy: [0.0, 0.0, 1.0, 1.0],+ coco_class: COCO_CAT,+ },+ ],+ BODY_IOU_THRESHOLD,+ );++ assert_eq!(+ retained_faces+ .iter()+ .map(|detection| (detection.score, detection.class_id))+ .collect::<Vec<_>>(),+ vec![(0.9, PET_SPECIES_DOG), (0.7, PET_SPECIES_CAT)]+ );+ assert_eq!(+ retained_bodies+ .iter()+ .map(|detection| (detection.score, detection.coco_class))+ .collect::<Vec<_>>(),+ vec![(0.9, COCO_DOG), (0.7, COCO_CAT)]+ );+ }#[test]fn pet_body_class_prefilter_matches_full_scan() {@@ -411,4 +495,9 @@ mod tests {let score = best_logit * row[4];(score >= BODY_MIN_SCORE).then_some((best_class, score))}++ fn separated_box(index: usize) -> [f32; 4] {+ let x = index as f32 * 2.0;+ [x, 0.0, x + 1.0, 1.0]+ }}diff --git a/rust/crates/photos/src/ml/postprocess.rs b/rust/crates/photos/src/ml/postprocess.rsindex 3fe9d6009b..e70c71437c 100644--- a/rust/crates/photos/src/ml/postprocess.rs+++ b/rust/crates/photos/src/ml/postprocess.rs@@ -1,3 +1,5 @@+pub(crate) const MAX_DETECTIONS_PER_IMAGE: usize = 100;+pub(crate) fn l2_normalize(embedding: &mut [f32], zero_threshold: f32) {let mut norm = 0.0f32;for value in embedding.iter() {