π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust][photos] Consolidate ML detection suppression
laurenspriemc3533df21126
diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rsindex 6a4da38d7e..71ab73c31a 100644--- a/rust/crates/photos/src/ml/face/detect.rs+++ b/rust/crates/photos/src/ml/face/detect.rs@@ -1,7 +1,7 @@use crate::ml::{error::{MlError, MlResult},onnx,- postprocess::MAX_DETECTIONS_PER_IMAGE,+ postprocess::{NmsDetection, greedy_non_max_suppression},preprocess::{YOLO_INPUT_SIZE, YoloInput},runtime::MlRuntimeView,types::FaceDetection,@@ -10,6 +10,16 @@ use crate::ml::{const IOU_THRESHOLD: f32 = 0.4;const MIN_SCORE_THRESHOLD: f32 = 0.5;+impl NmsDetection for FaceDetection {+ fn score(&self) -> f32 {+ self.score+ }++ fn box_xyxy(&self) -> &[f32; 4] {+ &self.box_xyxy+ }+}+pub(crate) fn run_face_detection(runtime: &MlRuntimeView<'_>,input: &YoloInput,@@ -104,60 +114,11 @@ fn postprocess_face_tensor<T: onnx::FloatTensorData>(Ok(greedy_non_max_suppression(detections, IOU_THRESHOLD))}-fn greedy_non_max_suppression(- mut detections: Vec<FaceDetection>,- iou_threshold: f32,-) -> Vec<FaceDetection> {- detections.sort_by(|a, b| b.score.total_cmp(&a.score));-- let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));- for detection in detections {- if retained- .iter()- .any(|existing| calculate_iou(existing, &detection) >= iou_threshold)- {- continue;- }-- retained.push(detection);- if retained.len() == MAX_DETECTIONS_PER_IMAGE {- break;- }- }-- retained-}--fn calculate_iou(a: &FaceDetection, b: &FaceDetection) -> f32 {- let area_a =- (a.box_xyxy[2] - a.box_xyxy[0]).max(0.0) * (a.box_xyxy[3] - a.box_xyxy[1]).max(0.0);- let area_b =- (b.box_xyxy[2] - b.box_xyxy[0]).max(0.0) * (b.box_xyxy[3] - b.box_xyxy[1]).max(0.0);-- let intersection_min_x = a.box_xyxy[0].max(b.box_xyxy[0]);- let intersection_min_y = a.box_xyxy[1].max(b.box_xyxy[1]);- let intersection_max_x = a.box_xyxy[2].min(b.box_xyxy[2]);- let intersection_max_y = a.box_xyxy[3].min(b.box_xyxy[3]);-- let intersection_width = intersection_max_x - intersection_min_x;- let intersection_height = intersection_max_y - intersection_min_y;- if intersection_width < 0.0 || intersection_height < 0.0 {- return 0.0;- }-- let intersection_area = intersection_width * intersection_height;- let union_area = area_a + area_b - intersection_area;- if union_area <= 0.0 {- return 0.0;- }- intersection_area / union_area-}-#[cfg(test)]mod tests {- use super::{- FaceDetection, IOU_THRESHOLD, MAX_DETECTIONS_PER_IMAGE, greedy_non_max_suppression,- };+ use crate::ml::postprocess::MAX_DETECTIONS_PER_IMAGE;++ use super::{FaceDetection, IOU_THRESHOLD, greedy_non_max_suppression};#[test]fn face_nms_retains_the_highest_scoring_hundred_detections() {diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rsindex 0124541f2c..071869b595 100644--- a/rust/crates/photos/src/ml/pet/detect.rs+++ b/rust/crates/photos/src/ml/pet/detect.rs@@ -1,7 +1,7 @@use crate::ml::{error::{MlError, MlResult},onnx,- postprocess::MAX_DETECTIONS_PER_IMAGE,+ postprocess::{NmsDetection, greedy_non_max_suppression},preprocess::{YOLO_INPUT_SIZE, YoloInput},runtime::MlRuntimeView,types::{PetBodyDetection, PetFaceDetection},@@ -16,6 +16,34 @@ const PET_FACE_MIN_SCORE: f32 = 0.3;const BODY_IOU_THRESHOLD: f32 = 0.5;const BODY_MIN_SCORE: f32 = 0.3;+impl NmsDetection for PetFaceDetection {+ fn score(&self) -> f32 {+ self.score+ }++ fn box_xyxy(&self) -> &[f32; 4] {+ &self.box_xyxy+ }++ fn same_class(&self, other: &Self) -> bool {+ self.class_id == other.class_id+ }+}++impl NmsDetection for PetBodyDetection {+ fn score(&self) -> f32 {+ self.score+ }++ fn box_xyxy(&self) -> &[f32; 4] {+ &self.box_xyxy+ }++ fn same_class(&self, other: &Self) -> bool {+ self.coco_class == other.coco_class+ }+}+// Species IDs used across both Rust and Dart:// 0 = dog (face detection class_id=0, COCO_DOG=16)// 1 = cat (face detection class_id=1, COCO_CAT=15)@@ -165,7 +193,10 @@ fn postprocess_pet_face_tensor<T: onnx::FloatTensorData>(});}- Ok(greedy_nms_pet_face(detections, PET_FACE_IOU_THRESHOLD))+ Ok(greedy_non_max_suppression(+ detections,+ PET_FACE_IOU_THRESHOLD,+ ))}pub(crate) fn run_pet_body_detection(@@ -236,7 +267,7 @@ fn postprocess_pet_body_tensor<T: onnx::FloatTensorData>(});}- Ok(greedy_nms_pet_body(detections, BODY_IOU_THRESHOLD))+ Ok(greedy_non_max_suppression(detections, BODY_IOU_THRESHOLD))}fn winning_pet_body_class<T: onnx::FloatTensorData>(@@ -274,80 +305,14 @@ fn winning_pet_body_class<T: onnx::FloatTensorData>(Some((best_cls, best_logit * obj_conf))}-fn calculate_iou_4(a: &[f32; 4], b: &[f32; 4]) -> f32 {- let area_a = (a[2] - a[0]).max(0.0) * (a[3] - a[1]).max(0.0);- let area_b = (b[2] - b[0]).max(0.0) * (b[3] - b[1]).max(0.0);-- let ix1 = a[0].max(b[0]);- let iy1 = a[1].max(b[1]);- let ix2 = a[2].min(b[2]);- let iy2 = a[3].min(b[3]);-- let iw = ix2 - ix1;- let ih = iy2 - iy1;- if iw < 0.0 || ih < 0.0 {- return 0.0;- }-- let inter = iw * ih;- let union = area_a + area_b - inter;- if union <= 0.0 { 0.0 } else { inter / union }-}--fn greedy_nms_pet_face(- mut detections: Vec<PetFaceDetection>,- iou_threshold: f32,-) -> Vec<PetFaceDetection> {- detections.sort_by(|a, b| b.score.total_cmp(&a.score));-- let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));- for detection in detections {- if retained.iter().any(|existing: &PetFaceDetection| {- existing.class_id == detection.class_id- && calculate_iou_4(&existing.box_xyxy, &detection.box_xyxy) >= iou_threshold- }) {- continue;- }-- retained.push(detection);- if retained.len() == MAX_DETECTIONS_PER_IMAGE {- break;- }- }-- retained-}--fn greedy_nms_pet_body(- mut detections: Vec<PetBodyDetection>,- iou_threshold: f32,-) -> Vec<PetBodyDetection> {- detections.sort_by(|a, b| b.score.total_cmp(&a.score));-- let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));- for detection in detections {- if retained.iter().any(|existing: &PetBodyDetection| {- existing.coco_class == detection.coco_class- && calculate_iou_4(&existing.box_xyxy, &detection.box_xyxy) >= iou_threshold- }) {- continue;- }-- retained.push(detection);- if retained.len() == MAX_DETECTIONS_PER_IMAGE {- break;- }- }-- retained-}-#[cfg(test)]mod tests {+ use crate::ml::postprocess::{MAX_DETECTIONS_PER_IMAGE, greedy_non_max_suppression};+use super::{- BODY_IOU_THRESHOLD, BODY_MIN_SCORE, COCO_CAT, COCO_DOG, MAX_DETECTIONS_PER_IMAGE,- PET_FACE_IOU_THRESHOLD, PET_SPECIES_CAT, PET_SPECIES_DOG, PetBodyDetection,- PetFaceDetection, greedy_nms_pet_body, greedy_nms_pet_face, winning_pet_body_class,+ BODY_IOU_THRESHOLD, BODY_MIN_SCORE, COCO_CAT, COCO_DOG, PET_FACE_IOU_THRESHOLD,+ PET_SPECIES_CAT, PET_SPECIES_DOG, PetBodyDetection, PetFaceDetection,+ winning_pet_body_class,};#[test]@@ -368,8 +333,8 @@ mod tests {}).collect();- let retained_faces = greedy_nms_pet_face(faces, PET_FACE_IOU_THRESHOLD);- let retained_bodies = greedy_nms_pet_body(bodies, BODY_IOU_THRESHOLD);+ let retained_faces = greedy_non_max_suppression(faces, PET_FACE_IOU_THRESHOLD);+ let retained_bodies = greedy_non_max_suppression(bodies, BODY_IOU_THRESHOLD);assert_eq!(retained_faces.len(), MAX_DETECTIONS_PER_IMAGE);assert_eq!(retained_faces.first().unwrap().score, 100.0);@@ -381,7 +346,7 @@ mod tests {#[test]fn pet_nms_only_suppresses_overlaps_within_the_same_class() {- let retained_faces = greedy_nms_pet_face(+ let retained_faces = greedy_non_max_suppression(vec![PetFaceDetection {score: 0.8,@@ -404,7 +369,7 @@ mod tests {],PET_FACE_IOU_THRESHOLD,);- let retained_bodies = greedy_nms_pet_body(+ let retained_bodies = greedy_non_max_suppression(vec![PetBodyDetection {score: 0.8,diff --git a/rust/crates/photos/src/ml/postprocess.rs b/rust/crates/photos/src/ml/postprocess.rsindex e70c71437c..620a3ecf6e 100644--- a/rust/crates/photos/src/ml/postprocess.rs+++ b/rust/crates/photos/src/ml/postprocess.rs@@ -1,5 +1,60 @@pub(crate) const MAX_DETECTIONS_PER_IMAGE: usize = 100;+pub(crate) trait NmsDetection {+ fn score(&self) -> f32;+ fn box_xyxy(&self) -> &[f32; 4];+ fn same_class(&self, _other: &Self) -> bool {+ true+ }+}++pub(crate) fn greedy_non_max_suppression<D: NmsDetection>(+ mut detections: Vec<D>,+ iou_threshold: f32,+) -> Vec<D> {+ detections.sort_by(|a, b| b.score().total_cmp(&a.score()));++ let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));+ for detection in detections {+ if retained.iter().any(|existing: &D| {+ existing.same_class(&detection)+ && calculate_iou(existing.box_xyxy(), detection.box_xyxy()) >= iou_threshold+ }) {+ continue;+ }++ retained.push(detection);+ if retained.len() == MAX_DETECTIONS_PER_IMAGE {+ break;+ }+ }++ retained+}++fn calculate_iou(a: &[f32; 4], b: &[f32; 4]) -> f32 {+ let area_a = (a[2] - a[0]).max(0.0) * (a[3] - a[1]).max(0.0);+ let area_b = (b[2] - b[0]).max(0.0) * (b[3] - b[1]).max(0.0);++ let intersection_min_x = a[0].max(b[0]);+ let intersection_min_y = a[1].max(b[1]);+ let intersection_max_x = a[2].min(b[2]);+ let intersection_max_y = a[3].min(b[3]);++ let intersection_width = intersection_max_x - intersection_min_x;+ let intersection_height = intersection_max_y - intersection_min_y;+ if intersection_width < 0.0 || intersection_height < 0.0 {+ return 0.0;+ }++ let intersection_area = intersection_width * intersection_height;+ let union_area = area_a + area_b - intersection_area;+ if union_area <= 0.0 {+ return 0.0;+ }+ intersection_area / union_area+}+pub(crate) fn l2_normalize(embedding: &mut [f32], zero_threshold: f32) {let mut norm = 0.0f32;for value in embedding.iter() {