Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][photos] Consolidate ML detection suppression

laurenspriemc3533df21126
diff --git a/rust/crates/photos/src/ml/face/detect.rs b/rust/crates/photos/src/ml/face/detect.rs
index 6a4da38d7e..71ab73c31a 100644
--- a/rust/crates/photos/src/ml/face/detect.rs
+++ b/rust/crates/photos/src/ml/face/detect.rs
@@ -1,7 +1,7 @@
use crate::ml::{
error::{MlError, MlResult},
onnx,
- postprocess::MAX_DETECTIONS_PER_IMAGE,
+ postprocess::{NmsDetection, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
runtime::MlRuntimeView,
types::FaceDetection,
@@ -10,6 +10,16 @@ use crate::ml::{
const IOU_THRESHOLD: f32 = 0.4;
const MIN_SCORE_THRESHOLD: f32 = 0.5;
+impl NmsDetection for FaceDetection {
+ fn score(&self) -> f32 {
+ self.score
+ }
+
+ fn box_xyxy(&self) -> &[f32; 4] {
+ &self.box_xyxy
+ }
+}
+
pub(crate) fn run_face_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
@@ -104,60 +114,11 @@ fn postprocess_face_tensor<T: onnx::FloatTensorData>(
Ok(greedy_non_max_suppression(detections, IOU_THRESHOLD))
}
-fn greedy_non_max_suppression(
- mut detections: Vec<FaceDetection>,
- iou_threshold: f32,
-) -> Vec<FaceDetection> {
- detections.sort_by(|a, b| b.score.total_cmp(&a.score));
-
- let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));
- for detection in detections {
- if retained
- .iter()
- .any(|existing| calculate_iou(existing, &detection) >= iou_threshold)
- {
- continue;
- }
-
- retained.push(detection);
- if retained.len() == MAX_DETECTIONS_PER_IMAGE {
- break;
- }
- }
-
- retained
-}
-
-fn calculate_iou(a: &FaceDetection, b: &FaceDetection) -> f32 {
- let area_a =
- (a.box_xyxy[2] - a.box_xyxy[0]).max(0.0) * (a.box_xyxy[3] - a.box_xyxy[1]).max(0.0);
- let area_b =
- (b.box_xyxy[2] - b.box_xyxy[0]).max(0.0) * (b.box_xyxy[3] - b.box_xyxy[1]).max(0.0);
-
- let intersection_min_x = a.box_xyxy[0].max(b.box_xyxy[0]);
- let intersection_min_y = a.box_xyxy[1].max(b.box_xyxy[1]);
- let intersection_max_x = a.box_xyxy[2].min(b.box_xyxy[2]);
- let intersection_max_y = a.box_xyxy[3].min(b.box_xyxy[3]);
-
- let intersection_width = intersection_max_x - intersection_min_x;
- let intersection_height = intersection_max_y - intersection_min_y;
- if intersection_width < 0.0 || intersection_height < 0.0 {
- return 0.0;
- }
-
- let intersection_area = intersection_width * intersection_height;
- let union_area = area_a + area_b - intersection_area;
- if union_area <= 0.0 {
- return 0.0;
- }
- intersection_area / union_area
-}
-
#[cfg(test)]
mod tests {
- use super::{
- FaceDetection, IOU_THRESHOLD, MAX_DETECTIONS_PER_IMAGE, greedy_non_max_suppression,
- };
+ use crate::ml::postprocess::MAX_DETECTIONS_PER_IMAGE;
+
+ use super::{FaceDetection, IOU_THRESHOLD, greedy_non_max_suppression};
#[test]
fn face_nms_retains_the_highest_scoring_hundred_detections() {
diff --git a/rust/crates/photos/src/ml/pet/detect.rs b/rust/crates/photos/src/ml/pet/detect.rs
index 0124541f2c..071869b595 100644
--- a/rust/crates/photos/src/ml/pet/detect.rs
+++ b/rust/crates/photos/src/ml/pet/detect.rs
@@ -1,7 +1,7 @@
use crate::ml::{
error::{MlError, MlResult},
onnx,
- postprocess::MAX_DETECTIONS_PER_IMAGE,
+ postprocess::{NmsDetection, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
runtime::MlRuntimeView,
types::{PetBodyDetection, PetFaceDetection},
@@ -16,6 +16,34 @@ const PET_FACE_MIN_SCORE: f32 = 0.3;
const BODY_IOU_THRESHOLD: f32 = 0.5;
const BODY_MIN_SCORE: f32 = 0.3;
+impl NmsDetection for PetFaceDetection {
+ fn score(&self) -> f32 {
+ self.score
+ }
+
+ fn box_xyxy(&self) -> &[f32; 4] {
+ &self.box_xyxy
+ }
+
+ fn same_class(&self, other: &Self) -> bool {
+ self.class_id == other.class_id
+ }
+}
+
+impl NmsDetection for PetBodyDetection {
+ fn score(&self) -> f32 {
+ self.score
+ }
+
+ fn box_xyxy(&self) -> &[f32; 4] {
+ &self.box_xyxy
+ }
+
+ fn same_class(&self, other: &Self) -> bool {
+ self.coco_class == other.coco_class
+ }
+}
+
// Species IDs used across both Rust and Dart:
// 0 = dog (face detection class_id=0, COCO_DOG=16)
// 1 = cat (face detection class_id=1, COCO_CAT=15)
@@ -165,7 +193,10 @@ fn postprocess_pet_face_tensor<T: onnx::FloatTensorData>(
});
}
- Ok(greedy_nms_pet_face(detections, PET_FACE_IOU_THRESHOLD))
+ Ok(greedy_non_max_suppression(
+ detections,
+ PET_FACE_IOU_THRESHOLD,
+ ))
}
pub(crate) fn run_pet_body_detection(
@@ -236,7 +267,7 @@ fn postprocess_pet_body_tensor<T: onnx::FloatTensorData>(
});
}
- Ok(greedy_nms_pet_body(detections, BODY_IOU_THRESHOLD))
+ Ok(greedy_non_max_suppression(detections, BODY_IOU_THRESHOLD))
}
fn winning_pet_body_class<T: onnx::FloatTensorData>(
@@ -274,80 +305,14 @@ fn winning_pet_body_class<T: onnx::FloatTensorData>(
Some((best_cls, best_logit * obj_conf))
}
-fn calculate_iou_4(a: &[f32; 4], b: &[f32; 4]) -> f32 {
- let area_a = (a[2] - a[0]).max(0.0) * (a[3] - a[1]).max(0.0);
- let area_b = (b[2] - b[0]).max(0.0) * (b[3] - b[1]).max(0.0);
-
- let ix1 = a[0].max(b[0]);
- let iy1 = a[1].max(b[1]);
- let ix2 = a[2].min(b[2]);
- let iy2 = a[3].min(b[3]);
-
- let iw = ix2 - ix1;
- let ih = iy2 - iy1;
- if iw < 0.0 || ih < 0.0 {
- return 0.0;
- }
-
- let inter = iw * ih;
- let union = area_a + area_b - inter;
- if union <= 0.0 { 0.0 } else { inter / union }
-}
-
-fn greedy_nms_pet_face(
- mut detections: Vec<PetFaceDetection>,
- iou_threshold: f32,
-) -> Vec<PetFaceDetection> {
- detections.sort_by(|a, b| b.score.total_cmp(&a.score));
-
- let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));
- for detection in detections {
- if retained.iter().any(|existing: &PetFaceDetection| {
- existing.class_id == detection.class_id
- && calculate_iou_4(&existing.box_xyxy, &detection.box_xyxy) >= iou_threshold
- }) {
- continue;
- }
-
- retained.push(detection);
- if retained.len() == MAX_DETECTIONS_PER_IMAGE {
- break;
- }
- }
-
- retained
-}
-
-fn greedy_nms_pet_body(
- mut detections: Vec<PetBodyDetection>,
- iou_threshold: f32,
-) -> Vec<PetBodyDetection> {
- detections.sort_by(|a, b| b.score.total_cmp(&a.score));
-
- let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));
- for detection in detections {
- if retained.iter().any(|existing: &PetBodyDetection| {
- existing.coco_class == detection.coco_class
- && calculate_iou_4(&existing.box_xyxy, &detection.box_xyxy) >= iou_threshold
- }) {
- continue;
- }
-
- retained.push(detection);
- if retained.len() == MAX_DETECTIONS_PER_IMAGE {
- break;
- }
- }
-
- retained
-}
-
#[cfg(test)]
mod tests {
+ use crate::ml::postprocess::{MAX_DETECTIONS_PER_IMAGE, greedy_non_max_suppression};
+
use super::{
- BODY_IOU_THRESHOLD, BODY_MIN_SCORE, COCO_CAT, COCO_DOG, MAX_DETECTIONS_PER_IMAGE,
- PET_FACE_IOU_THRESHOLD, PET_SPECIES_CAT, PET_SPECIES_DOG, PetBodyDetection,
- PetFaceDetection, greedy_nms_pet_body, greedy_nms_pet_face, winning_pet_body_class,
+ BODY_IOU_THRESHOLD, BODY_MIN_SCORE, COCO_CAT, COCO_DOG, PET_FACE_IOU_THRESHOLD,
+ PET_SPECIES_CAT, PET_SPECIES_DOG, PetBodyDetection, PetFaceDetection,
+ winning_pet_body_class,
};
#[test]
@@ -368,8 +333,8 @@ mod tests {
})
.collect();
- let retained_faces = greedy_nms_pet_face(faces, PET_FACE_IOU_THRESHOLD);
- let retained_bodies = greedy_nms_pet_body(bodies, BODY_IOU_THRESHOLD);
+ let retained_faces = greedy_non_max_suppression(faces, PET_FACE_IOU_THRESHOLD);
+ let retained_bodies = greedy_non_max_suppression(bodies, BODY_IOU_THRESHOLD);
assert_eq!(retained_faces.len(), MAX_DETECTIONS_PER_IMAGE);
assert_eq!(retained_faces.first().unwrap().score, 100.0);
@@ -381,7 +346,7 @@ mod tests {
#[test]
fn pet_nms_only_suppresses_overlaps_within_the_same_class() {
- let retained_faces = greedy_nms_pet_face(
+ let retained_faces = greedy_non_max_suppression(
vec![
PetFaceDetection {
score: 0.8,
@@ -404,7 +369,7 @@ mod tests {
],
PET_FACE_IOU_THRESHOLD,
);
- let retained_bodies = greedy_nms_pet_body(
+ let retained_bodies = greedy_non_max_suppression(
vec![
PetBodyDetection {
score: 0.8,
diff --git a/rust/crates/photos/src/ml/postprocess.rs b/rust/crates/photos/src/ml/postprocess.rs
index e70c71437c..620a3ecf6e 100644
--- a/rust/crates/photos/src/ml/postprocess.rs
+++ b/rust/crates/photos/src/ml/postprocess.rs
@@ -1,5 +1,60 @@
pub(crate) const MAX_DETECTIONS_PER_IMAGE: usize = 100;
+pub(crate) trait NmsDetection {
+ fn score(&self) -> f32;
+ fn box_xyxy(&self) -> &[f32; 4];
+ fn same_class(&self, _other: &Self) -> bool {
+ true
+ }
+}
+
+pub(crate) fn greedy_non_max_suppression<D: NmsDetection>(
+ mut detections: Vec<D>,
+ iou_threshold: f32,
+) -> Vec<D> {
+ detections.sort_by(|a, b| b.score().total_cmp(&a.score()));
+
+ let mut retained = Vec::with_capacity(detections.len().min(MAX_DETECTIONS_PER_IMAGE));
+ for detection in detections {
+ if retained.iter().any(|existing: &D| {
+ existing.same_class(&detection)
+ && calculate_iou(existing.box_xyxy(), detection.box_xyxy()) >= iou_threshold
+ }) {
+ continue;
+ }
+
+ retained.push(detection);
+ if retained.len() == MAX_DETECTIONS_PER_IMAGE {
+ break;
+ }
+ }
+
+ retained
+}
+
+fn calculate_iou(a: &[f32; 4], b: &[f32; 4]) -> f32 {
+ let area_a = (a[2] - a[0]).max(0.0) * (a[3] - a[1]).max(0.0);
+ let area_b = (b[2] - b[0]).max(0.0) * (b[3] - b[1]).max(0.0);
+
+ let intersection_min_x = a[0].max(b[0]);
+ let intersection_min_y = a[1].max(b[1]);
+ let intersection_max_x = a[2].min(b[2]);
+ let intersection_max_y = a[3].min(b[3]);
+
+ let intersection_width = intersection_max_x - intersection_min_x;
+ let intersection_height = intersection_max_y - intersection_min_y;
+ if intersection_width < 0.0 || intersection_height < 0.0 {
+ return 0.0;
+ }
+
+ let intersection_area = intersection_width * intersection_height;
+ let union_area = area_a + area_b - intersection_area;
+ if union_area <= 0.0 {
+ return 0.0;
+ }
+ intersection_area / union_area
+}
+
pub(crate) fn l2_normalize(embedding: &mut [f32], zero_threshold: f32) {
let mut norm = 0.0f32;
for value in embedding.iter() {