Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Stop converting whole camera frames to feed a 256x256 tensor

laurenspriem3075ed0241fd
diff --git a/rust/bindings/frb/locker/src/api/document_scanner_api.rs b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
index d7f1a8b778..56d68c6eb1 100644
--- a/rust/bindings/frb/locker/src/api/document_scanner_api.rs
+++ b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
@@ -112,10 +112,13 @@ impl ScannerSession {
rotation_degrees: i32,
) -> Result<Option<RustQuad>, RustScanError> {
catch_panic(AssertUnwindSafe(|| {
- let rgba = bgra_to_rgba(&bgra, row_stride, width, height)?;
- let quad = self
- .inner
- .live_detect_rgba(&rgba, width, height, rotation_degrees)?;
+ let row_stride = u32::try_from(row_stride).map_err(|_| RustScanError {
+ kind: RustScanErrorKind::InvalidInput,
+ message: format!("negative row stride {row_stride}"),
+ })?;
+ let quad =
+ self.inner
+ .live_detect_bgra(&bgra, row_stride, width, height, rotation_degrees)?;
Ok(quad.map(to_api_quad))
}))
}
@@ -187,55 +190,6 @@ fn panic_message(panic: &Box<dyn Any + Send>) -> String {
}
}
-fn bgra_to_rgba(
- bgra: &[u8],
- row_stride: i32,
- width: u32,
- height: u32,
-) -> Result<Vec<u8>, RustScanError> {
- let invalid = |message: String| RustScanError {
- kind: RustScanErrorKind::InvalidInput,
- message,
- };
- if width == 0 || height == 0 {
- return Err(invalid(format!("empty frame: {width}x{height}")));
- }
- let width = width as usize;
- let height = height as usize;
- let row_stride = usize::try_from(row_stride)
- .map_err(|_| invalid(format!("negative row stride {row_stride}")))?;
- let row_bytes = width
- .checked_mul(4)
- .ok_or_else(|| invalid(format!("width {width} overflows")))?;
- if row_stride < row_bytes {
- return Err(invalid(format!(
- "row stride {row_stride} is less than {row_bytes} bytes per row"
- )));
- }
- let required = (height - 1)
- .checked_mul(row_stride)
- .and_then(|bulk| bulk.checked_add(row_bytes))
- .ok_or_else(|| invalid(format!("frame {width}x{height} overflows")))?;
- if bgra.len() < required {
- return Err(invalid(format!(
- "buffer holds {} bytes, {required} required",
- bgra.len()
- )));
- }
-
- let mut rgba = vec![0u8; height * row_bytes];
- for (row_index, rgba_row) in rgba.chunks_exact_mut(row_bytes).enumerate() {
- let bgra_row = &bgra[row_index * row_stride..row_index * row_stride + row_bytes];
- for (rgba_px, bgra_px) in rgba_row.chunks_exact_mut(4).zip(bgra_row.chunks_exact(4)) {
- rgba_px[0] = bgra_px[2];
- rgba_px[1] = bgra_px[1];
- rgba_px[2] = bgra_px[0];
- rgba_px[3] = bgra_px[3];
- }
- }
- Ok(rgba)
-}
-
impl From<scan::ScanError> for RustScanError {
fn from(value: scan::ScanError) -> Self {
let (kind, message) = match value {
diff --git a/rust/crates/ml/src/cv/channels.rs b/rust/crates/ml/src/cv/channels.rs
index db4531f780..971790efe5 100644
--- a/rust/crates/ml/src/cv/channels.rs
+++ b/rust/crates/ml/src/cv/channels.rs
@@ -17,19 +17,6 @@ fn require_channels(src: &ImageU8, channels: i32, op: &str) -> OpResult<()> {
Ok(())
}
-pub(crate) fn bgr_to_rgb(src: &ImageU8) -> OpResult<ImageU8> {
- require_channels(src, 3, "bgr_to_rgb")?;
- let mut data = vec![0u8; src.data.len()];
- super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {
- for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {
- out[0] = px[2];
- out[1] = px[1];
- out[2] = px[0];
- }
- });
- ImageU8::new(src.width, src.height, 3, data)
-}
-
pub(crate) fn gray_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {
require_channels(src, 1, "gray_to_bgr")?;
let mut data = vec![0u8; src.data.len() * 3];
diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rs
index 0ff49e9107..df417bd909 100644
--- a/rust/crates/ml/src/cv/mod.rs
+++ b/rust/crates/ml/src/cv/mod.rs
@@ -28,7 +28,7 @@ pub(crate) use arith::{
};
pub(crate) use bilateral::bilateral_filter_u8;
pub(crate) use canny::canny;
-pub(crate) use channels::{bgr_to_gray, bgr_to_rgb, gray_to_bgr, merge_u8, split_u8};
+pub(crate) use channels::{bgr_to_gray, gray_to_bgr, merge_u8, split_u8};
pub(crate) use contours::find_contours;
pub(crate) use convert::{f32_to_u8, u8_to_f32};
pub(crate) use draw::fill_poly;
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rs
index 0854a54204..d7e6e7072c 100644
--- a/rust/crates/ml/src/scan/scanner.rs
+++ b/rust/crates/ml/src/scan/scanner.rs
@@ -6,7 +6,7 @@ use super::detection::{detect_document_quad, extract_document, resize_for_max_pi
use super::geometry::{ImageSize, Quad};
use super::mask::Mask;
use super::segmentation::{MASK_SIDE, Segmenter};
-use super::yuv::{PlaneLayout, rgba_to_bgr, yuv420_to_bgr};
+use super::yuv::{PlaneLayout, bgra_to_bgr, rgba_to_bgr, yuv420_to_bgr};
use crate::cv;
use crate::cv::image::ImageU8;
@@ -102,6 +102,19 @@ impl ScannerSession {
self.live_detect(&bgr, rotation_degrees)
}
+ pub fn live_detect_bgra(
+ &self,
+ bgra: &[u8],
+ row_stride: u32,
+ width: u32,
+ height: u32,
+ rotation_degrees: i32,
+ ) -> Result<Option<Quad>, ScanError> {
+ let bgr = bgra_to_bgr(bgra, to_i32(row_stride)?, to_i32(width)?, to_i32(height)?)
+ .map_err(ScanError::InvalidInput)?;
+ self.live_detect(&bgr, rotation_degrees)
+ }
+
pub fn live_detect_yuv420(
&self,
y: &[u8],
diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rs
index 444df23d26..f980a0bf2e 100644
--- a/rust/crates/ml/src/scan/segmentation.rs
+++ b/rust/crates/ml/src/scan/segmentation.rs
@@ -40,14 +40,23 @@ impl Segmenter {
}
pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {
- let rgb = cv::bgr_to_rgb(bgr)?;
- let resized = cv::resize_bilinear(ImageRef::U8(&rgb), MASK_SIDE, MASK_SIDE)?.into_u8()?;
+ if bgr.channels != 3 {
+ return Err(format!(
+ "probability_map_u8: expected a 3-channel image, got {}",
+ bgr.channels
+ ));
+ }
+ // Resize first: the channel swap is per-pixel, so doing it after the
+ // downscale costs 256x256 instead of a full-resolution pass, and it
+ // folds into the tensor fill below.
+ let resized = cv::resize_bilinear(ImageRef::U8(bgr), MASK_SIDE, MASK_SIDE)?.into_u8()?;
- let input: Vec<f32> = resized
- .data
- .iter()
- .map(|&v| (v as f32 - 127.5) / 127.5)
- .collect();
+ let mut input = vec![0.0f32; resized.data.len()];
+ for (out, px) in input.chunks_exact_mut(3).zip(resized.data.chunks_exact(3)) {
+ out[0] = (px[2] as f32 - 127.5) / 127.5;
+ out[1] = (px[1] as f32 - 127.5) / 127.5;
+ out[2] = (px[0] as f32 - 127.5) / 127.5;
+ }
let side = MASK_SIDE as i64;
let mut session = match self.session.lock() {
diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rs
index b0a68a9a3a..c72caf670a 100644
--- a/rust/crates/ml/src/scan/yuv.rs
+++ b/rust/crates/ml/src/scan/yuv.rs
@@ -127,6 +127,44 @@ pub(crate) fn yuv420_to_bgr(
ImageU8::new(width, height, 3, out)
}
+/// Camera frames arrive strided; convert straight to BGR so the live path
+/// materialises one full-resolution buffer instead of three.
+pub(crate) fn bgra_to_bgr(
+ bgra: &[u8],
+ row_stride: i32,
+ width: i32,
+ height: i32,
+) -> Result<ImageU8, String> {
+ if width <= 0 || height <= 0 {
+ return Err(format!("invalid frame size {width}x{height}"));
+ }
+ let row_bytes = width as usize * 4;
+ if row_stride < 0 || (row_stride as usize) < row_bytes {
+ return Err(format!(
+ "row stride {row_stride} is smaller than {row_bytes} bytes per row"
+ ));
+ }
+ let row_stride = row_stride as usize;
+ let needed = required_len(height, row_stride as i32, row_bytes as i32);
+ if bgra.len() < needed {
+ return Err(format!(
+ "BGRA buffer has {} bytes, needs {needed}",
+ bgra.len()
+ ));
+ }
+
+ let mut out = vec![0u8; width as usize * height as usize * 3];
+ for (row, dst_row) in out.chunks_exact_mut(width as usize * 3).enumerate() {
+ let src_row = &bgra[row * row_stride..row * row_stride + row_bytes];
+ for (dst, src) in dst_row.chunks_exact_mut(3).zip(src_row.chunks_exact(4)) {
+ dst[0] = src[0];
+ dst[1] = src[1];
+ dst[2] = src[2];
+ }
+ }
+ ImageU8::new(width, height, 3, out)
+}
+
pub(crate) fn rgba_to_bgr(rgba: &[u8], width: i32, height: i32) -> Result<ImageU8, String> {
if width <= 0 || height <= 0 {
return Err(format!("invalid frame size {width}x{height}"));
@@ -442,6 +480,31 @@ mod tests {
);
}
+ #[test]
+ fn bgra_to_bgr_honours_row_padding_and_rejects_bad_geometry() {
+ let stride = 2 * 4 + 4;
+ let mut padded = vec![0xAAu8; stride * 2];
+ let rows = [
+ [[1u8, 2, 3, 255], [10, 20, 30, 0]],
+ [[40u8, 50, 60, 255], [70, 80, 90, 128]],
+ ];
+ for (row, pixels) in rows.iter().enumerate() {
+ for (col, px) in pixels.iter().enumerate() {
+ padded[row * stride + col * 4..row * stride + col * 4 + 4].copy_from_slice(px);
+ }
+ }
+
+ let bgr = bgra_to_bgr(&padded, stride as i32, 2, 2).expect("conversion");
+ assert_eq!(bgr.data, vec![1, 2, 3, 10, 20, 30, 40, 50, 60, 70, 80, 90]);
+
+ let needed = stride + 2 * 4;
+ assert!(bgra_to_bgr(&padded[..needed], stride as i32, 2, 2).is_ok());
+ assert!(bgra_to_bgr(&padded[..needed - 1], stride as i32, 2, 2).is_err());
+ assert!(bgra_to_bgr(&padded, 7, 2, 2).is_err());
+ assert!(bgra_to_bgr(&padded, -8, 2, 2).is_err());
+ assert!(bgra_to_bgr(&padded, stride as i32, 0, 2).is_err());
+ }
+
#[test]
fn rgba_to_bgr_drops_alpha_and_reverses_channels() {
let rgba = vec![1u8, 2, 3, 255, 10, 20, 30, 0];