π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
use image::{ImageBuffer, Rgb, RgbImage};
use imageproc::geometric_transformations::{Border, Interpolation, Projection, warp_into};
use nalgebra::{Matrix2, Matrix3, Vector2};
use crate::{
error::{MlError, MlResult},
types::{AlignmentResult, DecodedImage, FaceDetection, FaceResult, to_face_id},
};
use super::FACE_INPUT_SIZE;
const LAPLACIAN_HARD_THRESHOLD: f32 = 10.0;
const REMOVE_SIDE_COLUMNS: usize = 56;
const MOBILEFACENET_NORMALIZATION_SCALE: f32 = 1.0 / 127.5;
const MOBILEFACENET_IDEAL_5_LANDMARKS: [[f32; 2]; 5] = [
[38.2946 / 112.0, 51.6963 / 112.0],
[73.5318 / 112.0, 51.5014 / 112.0],
[56.0252 / 112.0, 71.7366 / 112.0],
[41.5493 / 112.0, 92.3655 / 112.0],
[70.7299 / 112.0, 92.2041 / 112.0],
];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum FaceDirection {
Left,
Right,
Straight,
}
#[derive(Clone, Debug)]
struct FaceDetectionAbsolute {
keypoints: [[f32; 2]; 5],
}
pub(crate) fn run_face_alignment(
file_id: i64,
decoded: &mut DecodedImage,
detections: Vec<FaceDetection>,
) -> MlResult<(Vec<Vec<f32>>, Vec<FaceResult>)> {
let image_width = decoded.dimensions.width;
let image_height = decoded.dimensions.height;
let source = take_rgb_image(decoded)?;
let mut aligned_face_inputs = Vec::with_capacity(detections.len());
let mut face_results = Vec::with_capacity(detections.len());
let result = (|| {
for detection in detections {
let absolute_detection = to_absolute_detection(&detection, image_width, image_height);
let alignment = estimate_similarity_transform(&absolute_detection.keypoints)?;
let aligned = warp_face_image(&source, &alignment.affine_matrix)?;
let normalized = normalize_face_rgb_for_mobilefacenet(&aligned);
let blur_value = compute_blur_value(&aligned, face_direction(&absolute_detection));
let face_id = to_face_id(file_id, detection.box_xyxy);
aligned_face_inputs.push(normalized);
face_results.push(FaceResult {
detection,
blur_value,
alignment,
embedding: Vec::new(),
face_id,
});
}
Ok((aligned_face_inputs, face_results))
})();
// `imageproc::warp_into` requires an owned `RgbImage`. Put its allocation
// back so later indexing stages continue to borrow the decoded pixels.
decoded.rgb = source.into_raw();
result
}
fn take_rgb_image(decoded: &mut DecodedImage) -> MlResult<RgbImage> {
let expected_len = (decoded.dimensions.width as usize)
.checked_mul(decoded.dimensions.height as usize)
.and_then(|pixels| pixels.checked_mul(3))
.ok_or_else(|| MlError::Image("RGB source dimensions overflow".to_string()))?;
if decoded.rgb.len() != expected_len {
return Err(MlError::Image(format!(
"invalid RGB source length: expected {expected_len}, got {}",
decoded.rgb.len()
)));
}
let rgb = std::mem::take(&mut decoded.rgb);
#[expect(
clippy::expect_used,
reason = "RGB dimensions and exact buffer length are checked before conversion"
)]
Ok(ImageBuffer::<Rgb<u8>, _>::from_raw(
decoded.dimensions.width,
decoded.dimensions.height,
rgb,
)
.expect("validated RGB source dimensions and length"))
}
fn to_absolute_detection(
detection: &FaceDetection,
image_width: u32,
image_height: u32,
) -> FaceDetectionAbsolute {
let width = image_width as f32;
let height = image_height as f32;
let mut keypoints = [[0.0f32; 2]; 5];
for (i, point) in detection.keypoints.iter().enumerate() {
keypoints[i] = [point[0] * width, point[1] * height];
}
FaceDetectionAbsolute { keypoints }
}
fn estimate_similarity_transform(src_points: &[[f32; 2]; 5]) -> MlResult<AlignmentResult> {
let src_mean = mean_2d(src_points);
let dst_mean = mean_2d(&MOBILEFACENET_IDEAL_5_LANDMARKS);
let n = src_points.len() as f32;
let mut a = Matrix2::<f32>::zeros();
let mut src_var_sum = 0.0f32;
for (src, dst) in src_points
.iter()
.zip(MOBILEFACENET_IDEAL_5_LANDMARKS.iter())
{
let src_d = Vector2::new(src[0] - src_mean.x, src[1] - src_mean.y);
let dst_d = Vector2::new(dst[0] - dst_mean.x, dst[1] - dst_mean.y);
a += dst_d * src_d.transpose();
src_var_sum += src_d.dot(&src_d);
}
a /= n;
src_var_sum /= n;
let mut d = Vector2::new(1.0f32, 1.0f32);
if a.determinant() < 0.0 {
d[1] = -1.0;
}
let svd = a.svd(true, true);
let u = svd
.u
.ok_or_else(|| MlError::Postprocess("SVD failed to produce U".to_string()))?;
let v_t = svd
.v_t
.ok_or_else(|| MlError::Postprocess("SVD failed to produce V^T".to_string()))?;
let singular = svd.singular_values;
let rank = singular.iter().filter(|v| **v > 1e-6).count();
if rank == 0 {
return Err(MlError::Image(
"failed to estimate similarity transform (rank=0)".to_string(),
));
}
let r;
if rank == 1 {
if u.determinant() * v_t.transpose().determinant() > 0.0 {
r = u * v_t;
} else {
let mut d_local = d;
let last = d_local[1];
d_local[1] = -1.0;
r = u * Matrix2::new(d_local[0], 0.0, 0.0, d_local[1]) * v_t;
d[1] = last;
}
} else {
r = u * Matrix2::new(d[0], 0.0, 0.0, d[1]) * v_t;
}
let scale = if src_var_sum <= f32::EPSILON {
1.0
} else {
(singular[0] * d[0] + singular[1] * d[1]) / src_var_sum
};
let src_mean_v = Vector2::new(src_mean.x, src_mean.y);
let dst_mean_v = Vector2::new(dst_mean.x, dst_mean.y);
let translation = dst_mean_v - (r * src_mean_v) * scale;
let mut t = Matrix3::<f32>::identity();
let r_scaled = r * scale;
t[(0, 0)] = r_scaled[(0, 0)];
t[(0, 1)] = r_scaled[(0, 1)];
t[(1, 0)] = r_scaled[(1, 0)];
t[(1, 1)] = r_scaled[(1, 1)];
t[(0, 2)] = translation[0];
t[(1, 2)] = translation[1];
let size = if scale.abs() < f32::EPSILON {
1.0
} else {
1.0 / scale
};
let rotation = r[(1, 0)].atan2(r[(0, 0)]);
let mean_translation = (dst_mean_v - Vector2::new(0.5, 0.5)) * size;
let center = src_mean_v - mean_translation;
Ok(AlignmentResult {
affine_matrix: [
[t[(0, 0)], t[(0, 1)], t[(0, 2)]],
[t[(1, 0)], t[(1, 1)], t[(1, 2)]],
[t[(2, 0)], t[(2, 1)], t[(2, 2)]],
],
center: [center[0], center[1]],
size,
rotation,
})
}
fn mean_2d(points: &[[f32; 2]]) -> Vector2<f32> {
let mut sum = Vector2::new(0.0f32, 0.0f32);
for point in points {
sum[0] += point[0];
sum[1] += point[1];
}
sum / points.len() as f32
}
fn warp_face_image(source: &RgbImage, affine_matrix: &[[f32; 3]; 3]) -> MlResult<RgbImage> {
let mut transform = [[0.0f32; 3]; 3];
for row in 0..3 {
for col in 0..3 {
let value = affine_matrix[row][col];
transform[row][col] = if (value - 1.0).abs() <= f32::EPSILON {
1.0
} else {
value * FACE_INPUT_SIZE as f32
};
}
}
let projection = Projection::from_matrix([
transform[0][0],
transform[0][1],
transform[0][2],
transform[1][0],
transform[1][1],
transform[1][2],
transform[2][0],
transform[2][1],
transform[2][2],
])
.ok_or_else(|| MlError::Image("invalid affine matrix projection".to_string()))?;
let mut output = RgbImage::from_pixel(FACE_INPUT_SIZE, FACE_INPUT_SIZE, Rgb([114, 114, 114]));
let inverse = projection.invert();
warp_into(
source,
projection,
Interpolation::Bicubic,
Border::Constant(Rgb([114, 114, 114])),
&mut output,
);
for (x, y, pixel) in output.enumerate_pixels_mut() {
let (source_x, source_y) = inverse * (x as f32, y as f32);
let left = source_x.floor() - 1.0;
let top = source_y.floor() - 1.0;
if !(left >= 0.0
&& left + 4.0 < source.width() as f32
&& top >= 0.0
&& top + 4.0 < source.height() as f32)
{
*pixel = Rgb([114, 114, 114]);
}
}
Ok(output)
}
fn normalize_face_rgb_for_mobilefacenet(face_image: &RgbImage) -> Vec<f32> {
let mut output = Vec::with_capacity((FACE_INPUT_SIZE * FACE_INPUT_SIZE * 3) as usize);
for y in 0..FACE_INPUT_SIZE {
for x in 0..FACE_INPUT_SIZE {
let px = face_image.get_pixel(x, y).0;
output.push(px[0] as f32 * MOBILEFACENET_NORMALIZATION_SCALE - 1.0);
output.push(px[1] as f32 * MOBILEFACENET_NORMALIZATION_SCALE - 1.0);
output.push(px[2] as f32 * MOBILEFACENET_NORMALIZATION_SCALE - 1.0);
}
}
output
}
fn face_direction(detection: &FaceDetectionAbsolute) -> FaceDirection {
let left_eye = detection.keypoints[0];
let right_eye = detection.keypoints[1];
let nose = detection.keypoints[2];
let left_mouth = detection.keypoints[3];
let right_mouth = detection.keypoints[4];
let eye_distance_x = (right_eye[0] - left_eye[0]).abs();
let eye_distance_y = (right_eye[1] - left_eye[1]).abs();
let mouth_distance_y = (right_mouth[1] - left_mouth[1]).abs();
let face_is_upright = left_eye[1].max(right_eye[1]) + 0.5 * eye_distance_y < nose[1]
&& nose[1] + 0.5 * mouth_distance_y < left_mouth[1].min(right_mouth[1]);
let nose_sticking_out_left =
nose[0] < left_eye[0].min(right_eye[0]) && nose[0] < left_mouth[0].min(right_mouth[0]);
let nose_sticking_out_right =
nose[0] > left_eye[0].max(right_eye[0]) && nose[0] > left_mouth[0].max(right_mouth[0]);
let nose_close_to_left_eye = (nose[0] - left_eye[0]).abs() < 0.2 * eye_distance_x;
let nose_close_to_right_eye = (nose[0] - right_eye[0]).abs() < 0.2 * eye_distance_x;
if nose_sticking_out_left || (face_is_upright && nose_close_to_left_eye) {
FaceDirection::Left
} else if nose_sticking_out_right || (face_is_upright && nose_close_to_right_eye) {
FaceDirection::Right
} else {
FaceDirection::Straight
}
}
fn compute_blur_value(face_image: &RgbImage, direction: FaceDirection) -> f32 {
let (gray, gray_rows, gray_cols) = to_grayscale_buffer(face_image);
let (padded, padded_rows, padded_cols) =
pad_image_for_direction(&gray, gray_rows, gray_cols, direction);
let (laplacian, lap_rows, lap_cols) = apply_laplacian(&padded, padded_rows, padded_cols);
let variance = variance_2d(&laplacian, lap_rows, lap_cols);
if variance.is_finite() {
variance
} else {
LAPLACIAN_HARD_THRESHOLD + 1.0
}
}
fn to_grayscale_buffer(face_image: &RgbImage) -> (Vec<i32>, usize, usize) {
let w = face_image.width() as usize;
let h = face_image.height() as usize;
let mut gray = vec![0i32; w * h];
for y in 0..h {
for x in 0..w {
let px = face_image.get_pixel(x as u32, y as u32).0;
gray[y * w + x] = (0.299 * px[0] as f32 + 0.587 * px[1] as f32 + 0.114 * px[2] as f32)
.round()
.clamp(0.0, 255.0) as i32;
}
}
(gray, h, w)
}
fn apply_laplacian(
padded: &[i32],
padded_rows: usize,
padded_cols: usize,
) -> (Vec<i32>, usize, usize) {
let rows = padded_rows.saturating_sub(2);
let cols = padded_cols.saturating_sub(2);
let mut out = vec![0i32; rows * cols];
for i in 0..rows {
for j in 0..cols {
let top = padded[i * padded_cols + (j + 1)];
let left = padded[(i + 1) * padded_cols + j];
let center = padded[(i + 1) * padded_cols + (j + 1)];
let right = padded[(i + 1) * padded_cols + (j + 2)];
let bottom = padded[(i + 2) * padded_cols + (j + 1)];
out[i * cols + j] = top + left - (4 * center) + right + bottom;
}
}
(out, rows, cols)
}
fn pad_image_for_direction(
image: &[i32],
rows: usize,
cols: usize,
direction: FaceDirection,
) -> (Vec<i32>, usize, usize) {
let padded_cols = cols + 2 - REMOVE_SIDE_COLUMNS;
let padded_rows = rows + 2;
let mut padded = vec![0i32; padded_rows * padded_cols];
let start_col = match direction {
FaceDirection::Straight => REMOVE_SIDE_COLUMNS / 2,
FaceDirection::Left => REMOVE_SIDE_COLUMNS,
FaceDirection::Right => 0,
};
let copy_cols = padded_cols.saturating_sub(2);
for i in 0..rows {
for j in 0..copy_cols {
padded[(i + 1) * padded_cols + (j + 1)] = image[i * cols + (j + start_col)];
}
}
if copy_cols > 0 {
for col in 0..copy_cols {
padded[1 + col] = padded[2 * padded_cols + 1 + col];
padded[(rows + 1) * padded_cols + 1 + col] = padded[(rows - 1) * padded_cols + 1 + col];
}
}
for row in 0..(rows + 2) {
let row_start = row * padded_cols;
padded[row_start] = padded[row_start + 2];
padded[row_start + padded_cols - 1] = padded[row_start + padded_cols - 3];
}
(padded, padded_rows, padded_cols)
}
fn variance_2d(matrix: &[i32], rows: usize, cols: usize) -> f32 {
if rows == 0 || cols == 0 {
return 0.0;
}
let total = (rows * cols) as f32;
let mut mean = 0.0f32;
for value in matrix {
mean += *value as f32;
}
mean /= total;
let mut variance = 0.0f32;
for value in matrix {
let diff = *value as f32 - mean;
variance += diff * diff;
}
variance / total
}
#[cfg(test)]
mod tests {
use super::{
MOBILEFACENET_IDEAL_5_LANDMARKS, estimate_similarity_transform, run_face_alignment,
warp_face_image,
};
use crate::{
error::MlError,
types::{DecodedImage, Dimensions, FaceDetection},
};
use image::{Rgb, RgbImage};
#[test]
fn face_alignment_restores_the_decoded_rgb_allocation() {
let mut decoded = DecodedImage {
dimensions: Dimensions {
width: 2,
height: 2,
},
rgb: (0..12).collect(),
};
let allocation = decoded.rgb.as_ptr();
let (aligned, results) = run_face_alignment(1, &mut decoded, Vec::new()).unwrap();
assert!(aligned.is_empty());
assert!(results.is_empty());
assert_eq!(decoded.rgb, (0..12).collect::<Vec<_>>());
assert_eq!(decoded.rgb.as_ptr(), allocation);
}
#[test]
fn degenerate_landmarks_are_classified_as_an_image_error() {
let mut decoded = DecodedImage {
dimensions: Dimensions {
width: 2,
height: 2,
},
rgb: (0..12).collect(),
};
let detection = FaceDetection {
score: 1.0,
box_xyxy: [0.0, 0.0, 1.0, 1.0],
keypoints: [[0.5, 0.5]; 5],
};
let result = run_face_alignment(1, &mut decoded, vec![detection]);
assert!(matches!(result, Err(MlError::Image(_))));
assert_eq!(decoded.rgb, (0..12).collect::<Vec<_>>());
}
#[test]
fn face_warp_retains_the_previous_bicubic_border_fill() {
let source = RgbImage::from_pixel(112, 112, Rgb([255, 0, 0]));
let identity = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
let output = warp_face_image(&source, &identity).unwrap();
assert_eq!(*output.get_pixel(0, 0), Rgb([114, 114, 114]));
assert_eq!(*output.get_pixel(1, 1), Rgb([255, 0, 0]));
assert_eq!(*output.get_pixel(109, 109), Rgb([114, 114, 114]));
}
#[test]
fn similarity_transform_recovers_known_scale_and_translation() {
let source = MOBILEFACENET_IDEAL_5_LANDMARKS.map(|[x, y]| [2.0 * x + 0.1, 2.0 * y - 0.2]);
let transform = estimate_similarity_transform(&source)
.unwrap()
.affine_matrix;
let expected = [[0.5, 0.0, -0.05], [0.0, 0.5, 0.1], [0.0, 0.0, 1.0]];
for (actual_row, expected_row) in transform.iter().zip(expected) {
for (actual, expected) in actual_row.iter().zip(expected_row) {
assert!((actual - expected).abs() < 1e-5);
}
}
}
}