π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Stop converting whole camera frames to feed a 256x256 tensor
laurenspriem3075ed0241fd
diff --git a/rust/bindings/frb/locker/src/api/document_scanner_api.rs b/rust/bindings/frb/locker/src/api/document_scanner_api.rsindex d7f1a8b778..56d68c6eb1 100644--- a/rust/bindings/frb/locker/src/api/document_scanner_api.rs+++ b/rust/bindings/frb/locker/src/api/document_scanner_api.rs@@ -112,10 +112,13 @@ impl ScannerSession {rotation_degrees: i32,) -> Result<Option<RustQuad>, RustScanError> {catch_panic(AssertUnwindSafe(|| {- let rgba = bgra_to_rgba(&bgra, row_stride, width, height)?;- let quad = self- .inner- .live_detect_rgba(&rgba, width, height, rotation_degrees)?;+ let row_stride = u32::try_from(row_stride).map_err(|_| RustScanError {+ kind: RustScanErrorKind::InvalidInput,+ message: format!("negative row stride {row_stride}"),+ })?;+ let quad =+ self.inner+ .live_detect_bgra(&bgra, row_stride, width, height, rotation_degrees)?;Ok(quad.map(to_api_quad))}))}@@ -187,55 +190,6 @@ fn panic_message(panic: &Box<dyn Any + Send>) -> String {}}-fn bgra_to_rgba(- bgra: &[u8],- row_stride: i32,- width: u32,- height: u32,-) -> Result<Vec<u8>, RustScanError> {- let invalid = |message: String| RustScanError {- kind: RustScanErrorKind::InvalidInput,- message,- };- if width == 0 || height == 0 {- return Err(invalid(format!("empty frame: {width}x{height}")));- }- let width = width as usize;- let height = height as usize;- let row_stride = usize::try_from(row_stride)- .map_err(|_| invalid(format!("negative row stride {row_stride}")))?;- let row_bytes = width- .checked_mul(4)- .ok_or_else(|| invalid(format!("width {width} overflows")))?;- if row_stride < row_bytes {- return Err(invalid(format!(- "row stride {row_stride} is less than {row_bytes} bytes per row"- )));- }- let required = (height - 1)- .checked_mul(row_stride)- .and_then(|bulk| bulk.checked_add(row_bytes))- .ok_or_else(|| invalid(format!("frame {width}x{height} overflows")))?;- if bgra.len() < required {- return Err(invalid(format!(- "buffer holds {} bytes, {required} required",- bgra.len()- )));- }-- let mut rgba = vec![0u8; height * row_bytes];- for (row_index, rgba_row) in rgba.chunks_exact_mut(row_bytes).enumerate() {- let bgra_row = &bgra[row_index * row_stride..row_index * row_stride + row_bytes];- for (rgba_px, bgra_px) in rgba_row.chunks_exact_mut(4).zip(bgra_row.chunks_exact(4)) {- rgba_px[0] = bgra_px[2];- rgba_px[1] = bgra_px[1];- rgba_px[2] = bgra_px[0];- rgba_px[3] = bgra_px[3];- }- }- Ok(rgba)-}-impl From<scan::ScanError> for RustScanError {fn from(value: scan::ScanError) -> Self {let (kind, message) = match value {diff --git a/rust/crates/ml/src/cv/channels.rs b/rust/crates/ml/src/cv/channels.rsindex db4531f780..971790efe5 100644--- a/rust/crates/ml/src/cv/channels.rs+++ b/rust/crates/ml/src/cv/channels.rs@@ -17,19 +17,6 @@ fn require_channels(src: &ImageU8, channels: i32, op: &str) -> OpResult<()> {Ok(())}-pub(crate) fn bgr_to_rgb(src: &ImageU8) -> OpResult<ImageU8> {- require_channels(src, 3, "bgr_to_rgb")?;- let mut data = vec![0u8; src.data.len()];- super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {- for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {- out[0] = px[2];- out[1] = px[1];- out[2] = px[0];- }- });- ImageU8::new(src.width, src.height, 3, data)-}-pub(crate) fn gray_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {require_channels(src, 1, "gray_to_bgr")?;let mut data = vec![0u8; src.data.len() * 3];diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rsindex 0ff49e9107..df417bd909 100644--- a/rust/crates/ml/src/cv/mod.rs+++ b/rust/crates/ml/src/cv/mod.rs@@ -28,7 +28,7 @@ pub(crate) use arith::{};pub(crate) use bilateral::bilateral_filter_u8;pub(crate) use canny::canny;-pub(crate) use channels::{bgr_to_gray, bgr_to_rgb, gray_to_bgr, merge_u8, split_u8};+pub(crate) use channels::{bgr_to_gray, gray_to_bgr, merge_u8, split_u8};pub(crate) use contours::find_contours;pub(crate) use convert::{f32_to_u8, u8_to_f32};pub(crate) use draw::fill_poly;diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rsindex 0854a54204..d7e6e7072c 100644--- a/rust/crates/ml/src/scan/scanner.rs+++ b/rust/crates/ml/src/scan/scanner.rs@@ -6,7 +6,7 @@ use super::detection::{detect_document_quad, extract_document, resize_for_max_piuse super::geometry::{ImageSize, Quad};use super::mask::Mask;use super::segmentation::{MASK_SIDE, Segmenter};-use super::yuv::{PlaneLayout, rgba_to_bgr, yuv420_to_bgr};+use super::yuv::{PlaneLayout, bgra_to_bgr, rgba_to_bgr, yuv420_to_bgr};use crate::cv;use crate::cv::image::ImageU8;@@ -102,6 +102,19 @@ impl ScannerSession {self.live_detect(&bgr, rotation_degrees)}+ pub fn live_detect_bgra(+ &self,+ bgra: &[u8],+ row_stride: u32,+ width: u32,+ height: u32,+ rotation_degrees: i32,+ ) -> Result<Option<Quad>, ScanError> {+ let bgr = bgra_to_bgr(bgra, to_i32(row_stride)?, to_i32(width)?, to_i32(height)?)+ .map_err(ScanError::InvalidInput)?;+ self.live_detect(&bgr, rotation_degrees)+ }+pub fn live_detect_yuv420(&self,y: &[u8],diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rsindex 444df23d26..f980a0bf2e 100644--- a/rust/crates/ml/src/scan/segmentation.rs+++ b/rust/crates/ml/src/scan/segmentation.rs@@ -40,14 +40,23 @@ impl Segmenter {}pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {- let rgb = cv::bgr_to_rgb(bgr)?;- let resized = cv::resize_bilinear(ImageRef::U8(&rgb), MASK_SIDE, MASK_SIDE)?.into_u8()?;+ if bgr.channels != 3 {+ return Err(format!(+ "probability_map_u8: expected a 3-channel image, got {}",+ bgr.channels+ ));+ }+ // Resize first: the channel swap is per-pixel, so doing it after the+ // downscale costs 256x256 instead of a full-resolution pass, and it+ // folds into the tensor fill below.+ let resized = cv::resize_bilinear(ImageRef::U8(bgr), MASK_SIDE, MASK_SIDE)?.into_u8()?;- let input: Vec<f32> = resized- .data- .iter()- .map(|&v| (v as f32 - 127.5) / 127.5)- .collect();+ let mut input = vec![0.0f32; resized.data.len()];+ for (out, px) in input.chunks_exact_mut(3).zip(resized.data.chunks_exact(3)) {+ out[0] = (px[2] as f32 - 127.5) / 127.5;+ out[1] = (px[1] as f32 - 127.5) / 127.5;+ out[2] = (px[0] as f32 - 127.5) / 127.5;+ }let side = MASK_SIDE as i64;let mut session = match self.session.lock() {diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rsindex b0a68a9a3a..c72caf670a 100644--- a/rust/crates/ml/src/scan/yuv.rs+++ b/rust/crates/ml/src/scan/yuv.rs@@ -127,6 +127,44 @@ pub(crate) fn yuv420_to_bgr(ImageU8::new(width, height, 3, out)}+/// Camera frames arrive strided; convert straight to BGR so the live path+/// materialises one full-resolution buffer instead of three.+pub(crate) fn bgra_to_bgr(+ bgra: &[u8],+ row_stride: i32,+ width: i32,+ height: i32,+) -> Result<ImageU8, String> {+ if width <= 0 || height <= 0 {+ return Err(format!("invalid frame size {width}x{height}"));+ }+ let row_bytes = width as usize * 4;+ if row_stride < 0 || (row_stride as usize) < row_bytes {+ return Err(format!(+ "row stride {row_stride} is smaller than {row_bytes} bytes per row"+ ));+ }+ let row_stride = row_stride as usize;+ let needed = required_len(height, row_stride as i32, row_bytes as i32);+ if bgra.len() < needed {+ return Err(format!(+ "BGRA buffer has {} bytes, needs {needed}",+ bgra.len()+ ));+ }++ let mut out = vec![0u8; width as usize * height as usize * 3];+ for (row, dst_row) in out.chunks_exact_mut(width as usize * 3).enumerate() {+ let src_row = &bgra[row * row_stride..row * row_stride + row_bytes];+ for (dst, src) in dst_row.chunks_exact_mut(3).zip(src_row.chunks_exact(4)) {+ dst[0] = src[0];+ dst[1] = src[1];+ dst[2] = src[2];+ }+ }+ ImageU8::new(width, height, 3, out)+}+pub(crate) fn rgba_to_bgr(rgba: &[u8], width: i32, height: i32) -> Result<ImageU8, String> {if width <= 0 || height <= 0 {return Err(format!("invalid frame size {width}x{height}"));@@ -442,6 +480,31 @@ mod tests {);}+ #[test]+ fn bgra_to_bgr_honours_row_padding_and_rejects_bad_geometry() {+ let stride = 2 * 4 + 4;+ let mut padded = vec![0xAAu8; stride * 2];+ let rows = [+ [[1u8, 2, 3, 255], [10, 20, 30, 0]],+ [[40u8, 50, 60, 255], [70, 80, 90, 128]],+ ];+ for (row, pixels) in rows.iter().enumerate() {+ for (col, px) in pixels.iter().enumerate() {+ padded[row * stride + col * 4..row * stride + col * 4 + 4].copy_from_slice(px);+ }+ }++ let bgr = bgra_to_bgr(&padded, stride as i32, 2, 2).expect("conversion");+ assert_eq!(bgr.data, vec![1, 2, 3, 10, 20, 30, 40, 50, 60, 70, 80, 90]);++ let needed = stride + 2 * 4;+ assert!(bgra_to_bgr(&padded[..needed], stride as i32, 2, 2).is_ok());+ assert!(bgra_to_bgr(&padded[..needed - 1], stride as i32, 2, 2).is_err());+ assert!(bgra_to_bgr(&padded, 7, 2, 2).is_err());+ assert!(bgra_to_bgr(&padded, -8, 2, 2).is_err());+ assert!(bgra_to_bgr(&padded, stride as i32, 0, 2).is_err());+ }+#[test]fn rgba_to_bgr_drops_alpha_and_reverses_channels() {let rgba = vec![1u8, 2, 3, 255, 10, 20, 30, 0];