π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
use crate::{
error::{MlError, MlResult},
models::Model,
onnx,
postprocess::{NmsDetection, YoloOutputRow, greedy_non_max_suppression},
preprocess::{YOLO_INPUT_SIZE, YoloInput},
runtime::MlRuntimeView,
types::{PetBodyDetection, PetFaceDetection},
};
use super::{COCO_CAT, COCO_DOG, PET_SPECIES_CAT, PET_SPECIES_DOG};
// These thresholds match the Python pipeline.
const PET_FACE_IOU_THRESHOLD: f32 = 0.5;
const PET_FACE_MIN_SCORE: f32 = 0.3;
const BODY_IOU_THRESHOLD: f32 = 0.5;
const BODY_MIN_SCORE: f32 = 0.3;
impl NmsDetection for PetFaceDetection {
fn score(&self) -> f32 {
self.score
}
fn box_xyxy(&self) -> &[f32; 4] {
&self.box_xyxy
}
fn same_class(&self, other: &Self) -> bool {
self.class_id == other.class_id
}
}
impl NmsDetection for PetBodyDetection {
fn score(&self) -> f32 {
self.score
}
fn box_xyxy(&self) -> &[f32; 4] {
&self.box_xyxy
}
fn same_class(&self, other: &Self) -> bool {
self.coco_class == other.coco_class
}
}
pub(crate) fn run_pet_face_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetFaceDetection>> {
runtime.run(Model::PetFaceDetection, |session| {
onnx::with_prepared_float_output(
session,
&input.tensor,
[1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
|output_shape, output_data| {
postprocess_pet_face_detections(output_shape, output_data, input)
},
)
})
}
fn postprocess_pet_face_detections(
output_shape: &[i64],
output_data: onnx::BorrowedFloatTensor<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetFaceDetection>> {
match output_data {
onnx::BorrowedFloatTensor::F32(data) => {
postprocess_pet_face_tensor(output_shape, data, input)
}
onnx::BorrowedFloatTensor::F16(data) => {
postprocess_pet_face_tensor(output_shape, data, input)
}
}
}
fn postprocess_pet_face_tensor<T: onnx::FloatTensorData>(
output_shape: &[i64],
output_data: T,
input: &YoloInput,
) -> MlResult<Vec<PetFaceDetection>> {
let row_len = if let [_, .., last] = output_shape {
*last as usize
} else if output_shape.len() == 1 {
let total = output_data.len();
if total.is_multiple_of(13) {
13
} else if total.is_multiple_of(12) {
12
} else if total.is_multiple_of(11) {
11
} else {
return Err(MlError::Postprocess(format!(
"unexpected pet face detector output size: {total} (shape: {output_shape:?})"
)));
}
} else {
return Err(MlError::Postprocess(
"pet face detector output shape is empty".to_string(),
));
};
if row_len < 11 || output_data.len() < row_len {
return Err(MlError::Postprocess(format!(
"pet face detector row_len={} too small or output too short (len={})",
row_len,
output_data.len()
)));
}
let detection_rows = output_data.len() / row_len;
let mut detections = Vec::new();
// Row: x, y, width, height, score, left eye, right eye, nose, classes.
for i in 0..detection_rows {
let start = i * row_len;
let row = YoloOutputRow::new(output_data, start);
let score = row.confidence();
if score < PET_FACE_MIN_SCORE {
continue;
}
let mut box_xyxy = row.box_xyxy();
let mut keypoints = [row.keypoint(5), row.keypoint(7), row.keypoint(9)];
input.correct_box_and_keypoints(&mut box_xyxy, &mut keypoints);
// Two-class rows are [cat, dog]; one-class rows are dog-only.
let class_id: u8 = if row_len >= 13 {
if row.value(12) > row.value(11) {
PET_SPECIES_DOG
} else {
PET_SPECIES_CAT
}
} else {
PET_SPECIES_DOG
};
detections.push(PetFaceDetection {
score,
box_xyxy,
keypoints,
class_id,
});
}
Ok(greedy_non_max_suppression(
detections,
PET_FACE_IOU_THRESHOLD,
))
}
pub(crate) fn run_pet_body_detection(
runtime: &MlRuntimeView<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetBodyDetection>> {
runtime.run(Model::PetBodyDetection, |session| {
onnx::with_prepared_float_output(
session,
&input.tensor,
[1, 3, YOLO_INPUT_SIZE as i64, YOLO_INPUT_SIZE as i64],
|_output_shape, output_data| postprocess_pet_body_detections(output_data, input),
)
})
}
fn postprocess_pet_body_detections(
output_data: onnx::BorrowedFloatTensor<'_>,
input: &YoloInput,
) -> MlResult<Vec<PetBodyDetection>> {
match output_data {
onnx::BorrowedFloatTensor::F32(data) => postprocess_pet_body_tensor(data, input),
onnx::BorrowedFloatTensor::F16(data) => postprocess_pet_body_tensor(data, input),
}
}
fn postprocess_pet_body_tensor<T: onnx::FloatTensorData>(
output_data: T,
input: &YoloInput,
) -> MlResult<Vec<PetBodyDetection>> {
// YOLOv5 output format: [x, y, w, h, obj_conf, cls0, cls1, ..., cls79]
let row_len = 85usize;
if output_data.len() < row_len {
return Ok(Vec::new());
}
let detection_rows = output_data.len() / row_len;
let mut detections = Vec::new();
for i in 0..detection_rows {
let start = i * row_len;
let row = YoloOutputRow::new(output_data, start);
let Some((class_id, class_score)) = winning_pet_body_class(row) else {
continue;
};
let mut box_xyxy = row.box_xyxy();
input.correct_box(&mut box_xyxy);
detections.push(PetBodyDetection {
score: class_score,
box_xyxy,
coco_class: class_id,
});
}
Ok(greedy_non_max_suppression(detections, BODY_IOU_THRESHOLD))
}
fn winning_pet_body_class<T: onnx::FloatTensorData>(row: YoloOutputRow<T>) -> Option<(u8, f32)> {
let obj_conf = row.confidence();
let cat_logit = row.value(5 + COCO_CAT as usize);
let dog_logit = row.value(5 + COCO_DOG as usize);
let best_pet_logit = if dog_logit.total_cmp(&cat_logit).is_ge() {
dog_logit
} else {
cat_logit
};
if best_pet_logit * obj_conf < BODY_MIN_SCORE {
return None;
}
// A sufficiently strong pet score still needs to win against every other
// COCO class. Keep the original total ordering and tie behaviour here.
let mut best_cls = 0u8;
let mut best_logit = row.value(5);
for class in 1u8..80 {
let logit = row.value(5 + class as usize);
if logit.total_cmp(&best_logit).is_ge() {
best_cls = class;
best_logit = logit;
}
}
if best_cls != COCO_CAT && best_cls != COCO_DOG {
return None;
}
Some((best_cls, best_logit * obj_conf))
}
#[cfg(test)]
mod tests {
use crate::postprocess::{MAX_DETECTIONS_PER_IMAGE, YoloOutputRow, greedy_non_max_suppression};
use super::{
BODY_IOU_THRESHOLD, BODY_MIN_SCORE, COCO_CAT, COCO_DOG, PET_FACE_IOU_THRESHOLD,
PET_SPECIES_CAT, PET_SPECIES_DOG, PetBodyDetection, PetFaceDetection,
winning_pet_body_class,
};
#[test]
fn pet_nms_retains_the_highest_scoring_hundred_detections() {
let faces = (0..=MAX_DETECTIONS_PER_IMAGE)
.map(|index| PetFaceDetection {
score: index as f32,
box_xyxy: separated_box(index),
keypoints: [[0.0; 2]; 3],
class_id: PET_SPECIES_DOG,
})
.collect();
let bodies = (0..=MAX_DETECTIONS_PER_IMAGE)
.map(|index| PetBodyDetection {
score: index as f32,
box_xyxy: separated_box(index),
coco_class: COCO_DOG,
})
.collect();
let retained_faces = greedy_non_max_suppression(faces, PET_FACE_IOU_THRESHOLD);
let retained_bodies = greedy_non_max_suppression(bodies, BODY_IOU_THRESHOLD);
assert_eq!(retained_faces.len(), MAX_DETECTIONS_PER_IMAGE);
assert_eq!(retained_faces.first().unwrap().score, 100.0);
assert_eq!(retained_faces.last().unwrap().score, 1.0);
assert_eq!(retained_bodies.len(), MAX_DETECTIONS_PER_IMAGE);
assert_eq!(retained_bodies.first().unwrap().score, 100.0);
assert_eq!(retained_bodies.last().unwrap().score, 1.0);
}
#[test]
fn pet_nms_only_suppresses_overlaps_within_the_same_class() {
let retained_faces = greedy_non_max_suppression(
vec![
PetFaceDetection {
score: 0.8,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
keypoints: [[0.0; 2]; 3],
class_id: PET_SPECIES_DOG,
},
PetFaceDetection {
score: 0.9,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
keypoints: [[0.0; 2]; 3],
class_id: PET_SPECIES_DOG,
},
PetFaceDetection {
score: 0.7,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
keypoints: [[0.0; 2]; 3],
class_id: PET_SPECIES_CAT,
},
],
PET_FACE_IOU_THRESHOLD,
);
let retained_bodies = greedy_non_max_suppression(
vec![
PetBodyDetection {
score: 0.8,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
coco_class: COCO_DOG,
},
PetBodyDetection {
score: 0.9,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
coco_class: COCO_DOG,
},
PetBodyDetection {
score: 0.7,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
coco_class: COCO_CAT,
},
],
BODY_IOU_THRESHOLD,
);
assert_eq!(
retained_faces
.iter()
.map(|detection| (detection.score, detection.class_id))
.collect::<Vec<_>>(),
vec![(0.9, PET_SPECIES_DOG), (0.7, PET_SPECIES_CAT)]
);
assert_eq!(
retained_bodies
.iter()
.map(|detection| (detection.score, detection.coco_class))
.collect::<Vec<_>>(),
vec![(0.9, COCO_DOG), (0.7, COCO_CAT)]
);
}
#[test]
fn pet_body_class_prefilter_matches_full_scan() {
let mut rows = Vec::new();
rows.push(body_row(0.9, &[(COCO_CAT, 0.2), (COCO_DOG, 0.1)]));
rows.push(body_row(0.9, &[(COCO_CAT, 0.8), (COCO_DOG, 0.7)]));
rows.push(body_row(0.9, &[(COCO_CAT, 0.8), (3, 0.9)]));
rows.push(body_row(1.0, &[(COCO_CAT, BODY_MIN_SCORE)]));
rows.push(body_row(
1.0,
&[(COCO_CAT, BODY_MIN_SCORE), (COCO_DOG, BODY_MIN_SCORE)],
));
rows.push(body_row(1.0, &[(COCO_DOG, 0.8), (79, 0.8)]));
for row in rows {
assert_eq!(
winning_pet_body_class(YoloOutputRow::new(row.as_slice(), 0)),
full_scan(&row)
);
let f16_row = row
.iter()
.copied()
.map(half::f16::from_f32)
.collect::<Vec<_>>();
let converted_row = f16_row
.iter()
.map(|value| value.to_f32())
.collect::<Vec<_>>();
assert_eq!(
winning_pet_body_class(YoloOutputRow::new(f16_row.as_slice(), 0)),
full_scan(&converted_row),
);
}
}
fn body_row(object_confidence: f32, scores: &[(u8, f32)]) -> Vec<f32> {
let mut row = vec![0.0; 85];
row[4] = object_confidence;
for &(class, score) in scores {
row[5 + class as usize] = score;
}
row
}
fn full_scan(row: &[f32]) -> Option<(u8, f32)> {
let (best_class, best_logit) = row[5..85]
.iter()
.enumerate()
.max_by(|a, b| a.1.total_cmp(b.1))
.unwrap();
let best_class = best_class as u8;
if best_class != COCO_CAT && best_class != COCO_DOG {
return None;
}
let score = best_logit * row[4];
(score >= BODY_MIN_SCORE).then_some((best_class, score))
}
fn separated_box(index: usize) -> [f32; 4] {
let x = index as f32 * 2.0;
[x, 0.0, x + 1.0, 1.0]
}
}