Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Convert only the YUV samples the model actually reads

laurenspriemcbe10cdfeab3
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rs
index 54b705f027..0a3af0430c 100644
--- a/rust/crates/ml/src/scan/scanner.rs
+++ b/rust/crates/ml/src/scan/scanner.rs
@@ -90,7 +90,8 @@ impl ScannerSession {
) -> Result<Option<Quad>, ScanError> {
let bgr =
rgba_to_bgr(rgba, to_i32(width)?, to_i32(height)?).map_err(ScanError::InvalidInput)?;
- self.live_detect(&bgr, rotation_degrees)
+ let frame = ImageSize::new(bgr.width as f64, bgr.height as f64);
+ self.live_detect(&bgr, frame, rotation_degrees)
}
pub fn live_detect_bgra(
@@ -103,7 +104,8 @@ impl ScannerSession {
) -> Result<Option<Quad>, ScanError> {
let bgr = bgra_to_bgr(bgra, to_i32(row_stride)?, to_i32(width)?, to_i32(height)?)
.map_err(ScanError::InvalidInput)?;
- self.live_detect(&bgr, rotation_degrees)
+ let frame = ImageSize::new(bgr.width as f64, bgr.height as f64);
+ self.live_detect(&bgr, frame, rotation_degrees)
}
pub fn live_detect_yuv420(
@@ -114,15 +116,21 @@ impl ScannerSession {
layout: PlaneLayout,
rotation_degrees: i32,
) -> Result<Option<Quad>, ScanError> {
- let bgr = yuv420_to_bgr(y, u, v, layout).map_err(ScanError::InvalidInput)?;
- self.live_detect(&bgr, rotation_degrees)
+ let bgr = yuv420_to_bgr(y, u, v, layout, MASK_SIDE, MASK_SIDE)
+ .map_err(ScanError::InvalidInput)?;
+ let frame = ImageSize::new(layout.width as f64, layout.height as f64);
+ self.live_detect(&bgr, frame, rotation_degrees)
}
- fn live_detect(&self, bgr: &ImageU8, rotation_degrees: i32) -> Result<Option<Quad>, ScanError> {
+ fn live_detect(
+ &self,
+ bgr: &ImageU8,
+ frame_size: ImageSize,
+ rotation_degrees: i32,
+ ) -> Result<Option<Quad>, ScanError> {
let mask = self.segment(bgr)?;
- let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);
let quad_in_mask =
- detect_document_quad(&mask, original_size, true).map_err(ScanError::Pipeline)?;
+ detect_document_quad(&mask, frame_size, true).map_err(ScanError::Pipeline)?;
let mask_size = ImageSize::new(mask.width as f64, mask.height as f64);
Ok(quad_in_mask.map(|q| q.rotate90(rotation_degrees / 90, mask_size)))
}
diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rs
index d75d8740df..3f1ddfaa5a 100644
--- a/rust/crates/ml/src/scan/yuv.rs
+++ b/rust/crates/ml/src/scan/yuv.rs
@@ -3,6 +3,7 @@
//! apply when converting frames to RGBA, so YUV and RGBA frames yield the
//! same quad.
+use crate::cv;
use crate::cv::image::ImageU8;
const YG: i32 = 18997; // round(1.164 * 64 * 256 * 256 / 257)
@@ -50,12 +51,12 @@ fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> usize {
(rows - 1) as usize * row_stride as usize + last_row_bytes as usize
}
-pub(crate) fn yuv420_to_bgr(
+fn check_layout(
y_plane: &[u8],
u_plane: &[u8],
v_plane: &[u8],
layout: PlaneLayout,
-) -> Result<ImageU8, String> {
+) -> Result<(i32, i32), String> {
let PlaneLayout {
width,
height,
@@ -104,25 +105,71 @@ pub(crate) fn yuv420_to_bgr(
v_plane.len()
));
}
+ Ok((cw, ch))
+}
- let mut out = vec![0u8; width as usize * height as usize * 3];
- for row in 0..height {
- let y_base = row as usize * y_row_stride as usize;
- let uv_base = (row / 2) as usize * uv_row_stride as usize;
- let out_base = row as usize * width as usize * 3;
- for col in 0..width {
- let uv_index = uv_base + (col / 2) as usize * uv_pixel_stride as usize;
- let (b, g, r) = yuv_to_bgr(
- y_plane[y_base + col as usize],
- u_plane[uv_index],
- v_plane[uv_index],
- );
- let o = out_base + col as usize * 3;
- out[o] = b;
- out[o + 1] = g;
- out[o + 2] = r;
+fn plane_to_image(
+ plane: &[u8],
+ width: i32,
+ height: i32,
+ row_stride: i32,
+ pixel_stride: i32,
+) -> Result<ImageU8, String> {
+ let (w, h) = (width as usize, height as usize);
+ let (row_stride, pixel_stride) = (row_stride as usize, pixel_stride as usize);
+ if pixel_stride == 1 && row_stride == w {
+ return ImageU8::new(width, height, 1, plane[..w * h].to_vec());
+ }
+ let mut out = vec![0u8; w * h];
+ for (row, dst) in out.chunks_exact_mut(w).enumerate() {
+ let base = row * row_stride;
+ for (col, sample) in dst.iter_mut().enumerate() {
+ *sample = plane[base + col * pixel_stride];
}
}
+ ImageU8::new(width, height, 1, out)
+}
+
+pub(crate) fn yuv420_to_bgr(
+ y_plane: &[u8],
+ u_plane: &[u8],
+ v_plane: &[u8],
+ layout: PlaneLayout,
+ width: i32,
+ height: i32,
+) -> Result<ImageU8, String> {
+ let (cw, ch) = check_layout(y_plane, u_plane, v_plane, layout)?;
+
+ let y = plane_to_image(y_plane, layout.width, layout.height, layout.y_row_stride, 1)?;
+ let u = plane_to_image(
+ u_plane,
+ cw,
+ ch,
+ layout.uv_row_stride,
+ layout.uv_pixel_stride,
+ )?;
+ let v = plane_to_image(
+ v_plane,
+ cw,
+ ch,
+ layout.uv_row_stride,
+ layout.uv_pixel_stride,
+ )?;
+
+ let y = cv::resize_u8(&y, width, height, cv::Interp::Bilinear)?;
+ let u = cv::resize_u8(&u, width, height, cv::Interp::Bilinear)?;
+ let v = cv::resize_u8(&v, width, height, cv::Interp::Bilinear)?;
+
+ let mut out = vec![0u8; width as usize * height as usize * 3];
+ for (pixel, ((&y, &u), &v)) in out
+ .chunks_exact_mut(3)
+ .zip(y.data.iter().zip(u.data.iter()).zip(v.data.iter()))
+ {
+ let (b, g, r) = yuv_to_bgr(y, u, v);
+ pixel[0] = b;
+ pixel[1] = g;
+ pixel[2] = r;
+ }
ImageU8::new(width, height, 3, out)
}
@@ -334,6 +381,8 @@ mod tests {
uv_row_stride: ceil_half(width),
uv_pixel_stride: 1,
},
+ width,
+ height,
)
.expect("planar conversion");
@@ -355,6 +404,8 @@ mod tests {
uv_row_stride: cw * 2,
uv_pixel_stride: 2,
},
+ width,
+ height,
)
.expect("interleaved conversion");
assert_eq!(planar, interleaved);
@@ -375,6 +426,8 @@ mod tests {
uv_row_stride: cw * 2,
uv_pixel_stride: 2,
},
+ width,
+ height,
)
.expect("nv12 conversion");
assert_eq!(planar, nv12);
@@ -401,6 +454,8 @@ mod tests {
uv_row_stride: ceil_half(width),
uv_pixel_stride: 1,
},
+ width,
+ height,
)
.expect("tight conversion");
@@ -432,6 +487,8 @@ mod tests {
uv_row_stride: uv_stride,
uv_pixel_stride: 1,
},
+ width,
+ height,
)
.expect("padded conversion");
assert_eq!(tight, padded);
@@ -446,9 +503,9 @@ mod tests {
uv_row_stride: 4,
uv_pixel_stride: 1,
};
- assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout).is_err());
- assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout).is_err());
- assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout).is_err());
+ assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout, 8, 8).is_err());
+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout, 8, 8).is_err());
+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout, 8, 8).is_err());
assert!(
yuv420_to_bgr(
&[0u8; 64],
@@ -457,7 +514,9 @@ mod tests {
PlaneLayout {
y_row_stride: 4,
..layout
- }
+ },
+ 8,
+ 8
)
.is_err()
);
@@ -472,7 +531,9 @@ mod tests {
y_row_stride: 1,
uv_row_stride: 1,
uv_pixel_stride: 1,
- }
+ },
+ 1,
+ 1
)
.is_ok()
);