π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Convert only the YUV samples the model actually reads
laurenspriemcbe10cdfeab3
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rsindex 54b705f027..0a3af0430c 100644--- a/rust/crates/ml/src/scan/scanner.rs+++ b/rust/crates/ml/src/scan/scanner.rs@@ -90,7 +90,8 @@ impl ScannerSession {) -> Result<Option<Quad>, ScanError> {let bgr =rgba_to_bgr(rgba, to_i32(width)?, to_i32(height)?).map_err(ScanError::InvalidInput)?;- self.live_detect(&bgr, rotation_degrees)+ let frame = ImageSize::new(bgr.width as f64, bgr.height as f64);+ self.live_detect(&bgr, frame, rotation_degrees)}pub fn live_detect_bgra(@@ -103,7 +104,8 @@ impl ScannerSession {) -> Result<Option<Quad>, ScanError> {let bgr = bgra_to_bgr(bgra, to_i32(row_stride)?, to_i32(width)?, to_i32(height)?).map_err(ScanError::InvalidInput)?;- self.live_detect(&bgr, rotation_degrees)+ let frame = ImageSize::new(bgr.width as f64, bgr.height as f64);+ self.live_detect(&bgr, frame, rotation_degrees)}pub fn live_detect_yuv420(@@ -114,15 +116,21 @@ impl ScannerSession {layout: PlaneLayout,rotation_degrees: i32,) -> Result<Option<Quad>, ScanError> {- let bgr = yuv420_to_bgr(y, u, v, layout).map_err(ScanError::InvalidInput)?;- self.live_detect(&bgr, rotation_degrees)+ let bgr = yuv420_to_bgr(y, u, v, layout, MASK_SIDE, MASK_SIDE)+ .map_err(ScanError::InvalidInput)?;+ let frame = ImageSize::new(layout.width as f64, layout.height as f64);+ self.live_detect(&bgr, frame, rotation_degrees)}- fn live_detect(&self, bgr: &ImageU8, rotation_degrees: i32) -> Result<Option<Quad>, ScanError> {+ fn live_detect(+ &self,+ bgr: &ImageU8,+ frame_size: ImageSize,+ rotation_degrees: i32,+ ) -> Result<Option<Quad>, ScanError> {let mask = self.segment(bgr)?;- let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);let quad_in_mask =- detect_document_quad(&mask, original_size, true).map_err(ScanError::Pipeline)?;+ detect_document_quad(&mask, frame_size, true).map_err(ScanError::Pipeline)?;let mask_size = ImageSize::new(mask.width as f64, mask.height as f64);Ok(quad_in_mask.map(|q| q.rotate90(rotation_degrees / 90, mask_size)))}diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rsindex d75d8740df..3f1ddfaa5a 100644--- a/rust/crates/ml/src/scan/yuv.rs+++ b/rust/crates/ml/src/scan/yuv.rs@@ -3,6 +3,7 @@//! apply when converting frames to RGBA, so YUV and RGBA frames yield the//! same quad.+use crate::cv;use crate::cv::image::ImageU8;const YG: i32 = 18997; // round(1.164 * 64 * 256 * 256 / 257)@@ -50,12 +51,12 @@ fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> usize {(rows - 1) as usize * row_stride as usize + last_row_bytes as usize}-pub(crate) fn yuv420_to_bgr(+fn check_layout(y_plane: &[u8],u_plane: &[u8],v_plane: &[u8],layout: PlaneLayout,-) -> Result<ImageU8, String> {+) -> Result<(i32, i32), String> {let PlaneLayout {width,height,@@ -104,25 +105,71 @@ pub(crate) fn yuv420_to_bgr(v_plane.len()));}+ Ok((cw, ch))+}- let mut out = vec![0u8; width as usize * height as usize * 3];- for row in 0..height {- let y_base = row as usize * y_row_stride as usize;- let uv_base = (row / 2) as usize * uv_row_stride as usize;- let out_base = row as usize * width as usize * 3;- for col in 0..width {- let uv_index = uv_base + (col / 2) as usize * uv_pixel_stride as usize;- let (b, g, r) = yuv_to_bgr(- y_plane[y_base + col as usize],- u_plane[uv_index],- v_plane[uv_index],- );- let o = out_base + col as usize * 3;- out[o] = b;- out[o + 1] = g;- out[o + 2] = r;+fn plane_to_image(+ plane: &[u8],+ width: i32,+ height: i32,+ row_stride: i32,+ pixel_stride: i32,+) -> Result<ImageU8, String> {+ let (w, h) = (width as usize, height as usize);+ let (row_stride, pixel_stride) = (row_stride as usize, pixel_stride as usize);+ if pixel_stride == 1 && row_stride == w {+ return ImageU8::new(width, height, 1, plane[..w * h].to_vec());+ }+ let mut out = vec![0u8; w * h];+ for (row, dst) in out.chunks_exact_mut(w).enumerate() {+ let base = row * row_stride;+ for (col, sample) in dst.iter_mut().enumerate() {+ *sample = plane[base + col * pixel_stride];}}+ ImageU8::new(width, height, 1, out)+}++pub(crate) fn yuv420_to_bgr(+ y_plane: &[u8],+ u_plane: &[u8],+ v_plane: &[u8],+ layout: PlaneLayout,+ width: i32,+ height: i32,+) -> Result<ImageU8, String> {+ let (cw, ch) = check_layout(y_plane, u_plane, v_plane, layout)?;++ let y = plane_to_image(y_plane, layout.width, layout.height, layout.y_row_stride, 1)?;+ let u = plane_to_image(+ u_plane,+ cw,+ ch,+ layout.uv_row_stride,+ layout.uv_pixel_stride,+ )?;+ let v = plane_to_image(+ v_plane,+ cw,+ ch,+ layout.uv_row_stride,+ layout.uv_pixel_stride,+ )?;++ let y = cv::resize_u8(&y, width, height, cv::Interp::Bilinear)?;+ let u = cv::resize_u8(&u, width, height, cv::Interp::Bilinear)?;+ let v = cv::resize_u8(&v, width, height, cv::Interp::Bilinear)?;++ let mut out = vec![0u8; width as usize * height as usize * 3];+ for (pixel, ((&y, &u), &v)) in out+ .chunks_exact_mut(3)+ .zip(y.data.iter().zip(u.data.iter()).zip(v.data.iter()))+ {+ let (b, g, r) = yuv_to_bgr(y, u, v);+ pixel[0] = b;+ pixel[1] = g;+ pixel[2] = r;+ }ImageU8::new(width, height, 3, out)}@@ -334,6 +381,8 @@ mod tests {uv_row_stride: ceil_half(width),uv_pixel_stride: 1,},+ width,+ height,).expect("planar conversion");@@ -355,6 +404,8 @@ mod tests {uv_row_stride: cw * 2,uv_pixel_stride: 2,},+ width,+ height,).expect("interleaved conversion");assert_eq!(planar, interleaved);@@ -375,6 +426,8 @@ mod tests {uv_row_stride: cw * 2,uv_pixel_stride: 2,},+ width,+ height,).expect("nv12 conversion");assert_eq!(planar, nv12);@@ -401,6 +454,8 @@ mod tests {uv_row_stride: ceil_half(width),uv_pixel_stride: 1,},+ width,+ height,).expect("tight conversion");@@ -432,6 +487,8 @@ mod tests {uv_row_stride: uv_stride,uv_pixel_stride: 1,},+ width,+ height,).expect("padded conversion");assert_eq!(tight, padded);@@ -446,9 +503,9 @@ mod tests {uv_row_stride: 4,uv_pixel_stride: 1,};- assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout).is_err());- assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout).is_err());- assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout).is_err());+ assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout, 8, 8).is_err());+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout, 8, 8).is_err());+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout, 8, 8).is_err());assert!(yuv420_to_bgr(&[0u8; 64],@@ -457,7 +514,9 @@ mod tests {PlaneLayout {y_row_stride: 4,..layout- }+ },+ 8,+ 8).is_err());@@ -472,7 +531,9 @@ mod tests {y_row_stride: 1,uv_row_stride: 1,uv_pixel_stride: 1,- }+ },+ 1,+ 1).is_ok());