π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Add document scanning pipeline to the ml crate
laurenspriemb93f31c26aa0
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex d41211b5a4..1a9b38fed9 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2539,6 +2539,7 @@ dependencies = ["once_cell","ort","plsfix",+ "rayon","regex","serde","serde_json",diff --git a/rust/crates/ml/Cargo.toml b/rust/crates/ml/Cargo.tomlindex 80b9a56e00..0d52c60d11 100644--- a/rust/crates/ml/Cargo.toml+++ b/rust/crates/ml/Cargo.toml@@ -19,6 +19,7 @@ log.workspace = truenalgebra = "0.33.2"once_cell = "1.20.2"plsfix = "0.1.8"+rayon = "1"regex = "1.11.1"thiserror.workspace = true# The FRB photos bridge calls the SimSIMD symbol exported by usearch 2.21.x.diff --git a/rust/crates/ml/src/cv/arith.rs b/rust/crates/ml/src/cv/arith.rsnew file mode 100644index 0000000000..96cde77215--- /dev/null+++ b/rust/crates/ml/src/cv/arith.rs@@ -0,0 +1,147 @@+//! Scalar multiply and the weighted sum deliberately promote to f64; the+//! rest stays in f32.++use rayon::prelude::*;++use crate::cv::OpResult;+use crate::cv::image::ImageF32;++const PARALLEL_MIN_ELEMS: usize = 200_000;++fn fill<T: Send + Sync + Copy>(+ out: &mut [f32],+ src: &[T],+ chunk: usize,+ f: impl Fn(&mut [f32], &[T]) + Send + Sync,+) {+ if out.len() >= PARALLEL_MIN_ELEMS {+ out.par_chunks_mut(chunk)+ .zip(src.par_chunks(chunk))+ .for_each(|(o, s)| f(o, s));+ } else {+ f(out, src);+ }+}++fn chunk_len(channels: i32) -> usize {+ 65536 * channels as usize+}++fn same_geometry(a: &ImageF32, b: &ImageF32, op: &str) -> OpResult<()> {+ if !a.same_geometry(b) {+ return Err(format!("{op}: operands have different geometry"));+ }+ Ok(())+}++fn map1(a: &ImageF32, f: impl Fn(f32) -> f32 + Send + Sync) -> OpResult<ImageF32> {+ let mut data = vec![0.0f32; a.data.len()];+ fill(&mut data, &a.data, chunk_len(a.channels), |o, s| {+ for (o, &v) in o.iter_mut().zip(s.iter()) {+ *o = f(v);+ }+ });+ ImageF32::new(a.width, a.height, a.channels, data)+}++fn map2(+ a: &ImageF32,+ b: &ImageF32,+ op: &str,+ f: impl Fn(f32, f32) -> f32 + Send + Sync,+) -> OpResult<ImageF32> {+ same_geometry(a, b, op)?;+ let mut data = vec![0.0f32; a.data.len()];+ let chunk = chunk_len(a.channels);+ if data.len() >= PARALLEL_MIN_ELEMS {+ data.par_chunks_mut(chunk)+ .zip(a.data.par_chunks(chunk))+ .zip(b.data.par_chunks(chunk))+ .for_each(|((o, x), y)| {+ for ((o, &x), &y) in o.iter_mut().zip(x.iter()).zip(y.iter()) {+ *o = f(x, y);+ }+ });+ } else {+ for ((o, &x), &y) in data.iter_mut().zip(a.data.iter()).zip(b.data.iter()) {+ *o = f(x, y);+ }+ }+ ImageF32::new(a.width, a.height, a.channels, data)+}++fn map_scalar(+ a: &ImageF32,+ scalar: [f64; 4],+ f: impl Fn(f32, f64) -> f32 + Send + Sync,+) -> OpResult<ImageF32> {+ let cn = a.channels as usize;+ if cn > 4 {+ return Err("scalar operations support at most 4 channels".to_string());+ }+ let mut data = vec![0.0f32; a.data.len()];+ fill(&mut data, &a.data, chunk_len(a.channels), |o, s| {+ for (i, (o, &v)) in o.iter_mut().zip(s.iter()).enumerate() {+ *o = f(v, scalar[i % cn]);+ }+ });+ ImageF32::new(a.width, a.height, a.channels, data)+}++pub(crate) fn multiply_f32(a: &ImageF32, b: &ImageF32) -> OpResult<ImageF32> {+ map2(a, b, "multiply_f32", |x, y| x * y)+}++pub(crate) fn multiply_f32_scalar(a: &ImageF32, scalar: [f64; 4]) -> OpResult<ImageF32> {+ map_scalar(a, scalar, |v, s| (v as f64 * s) as f32)+}++pub(crate) fn add_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| v + s as f32)+}++pub(crate) fn subtract_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| v - s as f32)+}++pub(crate) fn subtract_f32(a: &ImageF32, b: &ImageF32) -> OpResult<ImageF32> {+ map2(a, b, "subtract_f32", |x, y| x - y)+}++pub(crate) fn add_weighted_f32(+ a: &ImageF32,+ alpha: f64,+ b: &ImageF32,+ beta: f64,+ gamma: f64,+) -> OpResult<ImageF32> {+ map2(a, b, "add_weighted_f32", |x, y| {+ (x as f64 * alpha + y as f64 * beta + gamma) as f32+ })+}++pub(crate) fn min_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| {+ let s = s as f32;+ if v < s { v } else { s }+ })+}++pub(crate) fn max_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| {+ let s = s as f32;+ if v > s { v } else { s }+ })+}++pub(crate) fn log_f32(a: &ImageF32) -> OpResult<ImageF32> {+ map1(a, f32::ln)+}++pub(crate) fn exp_f32(a: &ImageF32) -> OpResult<ImageF32> {+ map1(a, f32::exp)+}++pub(crate) fn magnitude_f32(x: &ImageF32, y: &ImageF32) -> OpResult<ImageF32> {+ map2(x, y, "magnitude_f32", |a, b| (a * a + b * b).sqrt())+}diff --git a/rust/crates/ml/src/cv/bilateral.rs b/rust/crates/ml/src/cv/bilateral.rsnew file mode 100644index 0000000000..68855c5c62--- /dev/null+++ b/rust/crates/ml/src/cv/bilateral.rs@@ -0,0 +1,81 @@+use rayon::prelude::*;++use super::reflect101;+use crate::cv::OpResult;+use crate::cv::image::ImageU8;++pub(crate) fn bilateral_filter_u8(+ src: &ImageU8,+ d: i32,+ sigma_color: f64,+ sigma_space: f64,+) -> OpResult<ImageU8> {+ if src.channels != 1 {+ return Err("bilateral_filter_u8: expected a single-channel image".to_string());+ }+ if sigma_color <= 1e-6 || sigma_space <= 1e-6 {+ return Ok(src.clone());+ }+ let radius = if d <= 0 {+ (sigma_space * 1.5).round() as i32+ } else {+ d / 2+ }+ .max(1) as usize;+ let (w, h) = (src.width as usize, src.height as usize);++ let stride = w + 2 * radius;+ let mut temp = vec![0u8; stride * (h + 2 * radius)];+ let cols: Vec<usize> = (0..stride)+ .map(|x| reflect101(x as i64 - radius as i64, src.width) as usize)+ .collect();+ for (y, trow) in temp.chunks_exact_mut(stride).enumerate() {+ let sy = reflect101(y as i64 - radius as i64, src.height) as usize;+ let srow = &src.data[sy * w..(sy + 1) * w];+ for (t, &sx) in trow.iter_mut().zip(cols.iter()) {+ *t = srow[sx];+ }+ }++ let color_coeff = -0.5 / (sigma_color * sigma_color);+ let space_coeff = -0.5 / (sigma_space * sigma_space);+ let color_weight: Vec<f32> = (0..256)+ .map(|i| ((i * i) as f64 * color_coeff).exp() as f32)+ .collect();++ let ir = radius as i64;+ let mut taps = Vec::new();+ for i in -ir..=ir {+ for j in -ir..=ir {+ let r2 = (i * i + j * j) as f64;+ if r2 > (ir * ir) as f64 {+ continue;+ }+ taps.push((+ ((i + ir) as usize * stride + (j + ir) as usize),+ (r2 * space_coeff).exp() as f32,+ ));+ }+ }++ let mut out = vec![0u8; w * h];+ out.par_chunks_exact_mut(w)+ .enumerate()+ .for_each(|(y, dst)| {+ let win = &temp[y * stride..];+ let centre = radius * stride + radius;+ for (x, d) in dst.iter_mut().enumerate() {+ let mid = win[centre + x] as i32;+ let mut sum = 0.0f32;+ let mut wsum = 0.0f32;+ for &(off, sw) in &taps {+ let val = win[off + x];+ let weight = sw * color_weight[(val as i32 - mid).unsigned_abs() as usize];+ wsum += weight;+ sum += val as f32 * weight;+ }+ *d = (sum / wsum).round_ties_even() as u8;+ }+ });+ ImageU8::new(src.width, src.height, 1, out)+}diff --git a/rust/crates/ml/src/cv/canny.rs b/rust/crates/ml/src/cv/canny.rsnew file mode 100644index 0000000000..c38c82201b--- /dev/null+++ b/rust/crates/ml/src/cv/canny.rs@@ -0,0 +1,152 @@+//! The 75/200 thresholds used downstream are calibrated against this exact+//! gradient scaling: integer `|dx| + |dy|` of a 3x3 Sobel with replicated+//! borders, thresholds floored, the low one compared strictly.++use crate::cv::OpResult;+use crate::cv::image::ImageU8;++const CANNY_SHIFT: i32 = 15;+/// `(int)(tan(22.5 deg) * (1 << CANNY_SHIFT) + 0.5)`.+const TG22: i32 = 13573;++const MAYBE: u8 = 0;+const NO: u8 = 1;+const EDGE: u8 = 2;++fn sobel_planes(src: &[u8], w: usize, h: usize, cn: usize, c: usize) -> (Vec<i16>, Vec<i16>) {+ let mut h_diff = vec![0i16; w * h];+ let mut h_smooth = vec![0i16; w * h];+ for y in 0..h {+ let row = &src[y * w * cn..(y + 1) * w * cn];+ let at = |x: usize| row[x * cn + c] as i16;+ let (d, s) = (+ &mut h_diff[y * w..(y + 1) * w],+ &mut h_smooth[y * w..(y + 1) * w],+ );+ for x in 0..w {+ let left = at(x.saturating_sub(1));+ let right = at((x + 1).min(w - 1));+ d[x] = right - left;+ s[x] = left + 2 * at(x) + right;+ }+ }++ let mut dx = vec![0i16; w * h];+ let mut dy = vec![0i16; w * h];+ for y in 0..h {+ let up = y.saturating_sub(1) * w;+ let down = (y + 1).min(h - 1) * w;+ let here = y * w;+ for x in 0..w {+ dx[here + x] = h_diff[up + x] + 2 * h_diff[here + x] + h_diff[down + x];+ dy[here + x] = h_smooth[down + x] - h_smooth[up + x];+ }+ }+ (dx, dy)+}++fn floor_threshold(t: f64) -> i32 {+ t.floor().clamp(i32::MIN as f64, i32::MAX as f64) as i32+}++pub(crate) fn canny(src: &ImageU8, threshold1: f64, threshold2: f64) -> OpResult<ImageU8> {+ let (w, h) = (src.width as usize, src.height as usize);+ let cn = src.channels as usize;++ let (low_thresh, high_thresh) = if threshold1 > threshold2 {+ (threshold2, threshold1)+ } else {+ (threshold1, threshold2)+ };+ let low = floor_threshold(low_thresh);+ let high = floor_threshold(high_thresh);++ let (mut dx, mut dy) = sobel_planes(&src.data, w, h, cn, 0);+ let mut norm: Vec<i32> = dx+ .iter()+ .zip(dy.iter())+ .map(|(&x, &y)| (x as i32).abs() + (y as i32).abs())+ .collect();+ for c in 1..cn {+ let (cdx, cdy) = sobel_planes(&src.data, w, h, cn, c);+ for i in 0..w * h {+ let m = (cdx[i] as i32).abs() + (cdy[i] as i32).abs();+ if m > norm[i] {+ norm[i] = m;+ dx[i] = cdx[i];+ dy[i] = cdy[i];+ }+ }+ }++ // The one-pixel `NO` frame keeps the unguarded hysteresis flood inside.+ let step = w + 2;+ let mut mag = vec![0i32; step * (h + 2)];+ for y in 0..h {+ mag[(y + 1) * step + 1..(y + 1) * step + 1 + w].copy_from_slice(&norm[y * w..(y + 1) * w]);+ }+ let mut map = vec![NO; step * (h + 2)];+ let mut stack: Vec<usize> = Vec::new();++ for y in 0..h {+ for x in 0..w {+ let p = (y + 1) * step + x + 1;+ let m = mag[p];+ if m > low {+ let (xs, ys) = (dx[y * w + x] as i32, dy[y * w + x] as i32);+ let ax = xs.abs();+ let ay = ys.abs() << CANNY_SHIFT;+ let tg22x = ax * TG22;+ let keep = if ay < tg22x {+ m > mag[p - 1] && m >= mag[p + 1]+ } else if ay > tg22x + (ax << (CANNY_SHIFT + 1)) {+ m > mag[p - step] && m >= mag[p + step]+ } else {+ let s: isize = if (xs ^ ys) < 0 { -1 } else { 1 };+ let up = (p as isize - step as isize - s) as usize;+ let down = (p as isize + step as isize + s) as usize;+ m > mag[up] && m > mag[down]+ };+ if keep {+ if m > high {+ map[p] = EDGE;+ stack.push(p);+ } else {+ map[p] = MAYBE;+ }+ continue;+ }+ }+ map[p] = NO;+ }+ }++ let offsets = [+ -(step as isize) - 1,+ -(step as isize),+ -(step as isize) + 1,+ -1,+ 1,+ step as isize - 1,+ step as isize,+ step as isize + 1,+ ];+ while let Some(p) = stack.pop() {+ for offset in offsets {+ let q = (p as isize + offset) as usize;+ if map[q] == MAYBE {+ map[q] = EDGE;+ stack.push(q);+ }+ }+ }++ let mut out = vec![0u8; w * h];+ for y in 0..h {+ let row = &map[(y + 1) * step + 1..(y + 1) * step + 1 + w];+ for (o, &v) in out[y * w..(y + 1) * w].iter_mut().zip(row.iter()) {+ *o = if v == EDGE { 255 } else { 0 };+ }+ }+ ImageU8::new(src.width, src.height, 1, out)+}diff --git a/rust/crates/ml/src/cv/channels.rs b/rust/crates/ml/src/cv/channels.rsnew file mode 100644index 0000000000..db4531f780--- /dev/null+++ b/rust/crates/ml/src/cv/channels.rs@@ -0,0 +1,94 @@+use crate::cv::OpResult;+use crate::cv::image::ImageU8;++// 15-bit fixed-point BT.601 luma weights.+const GRAY_SHIFT: i32 = 15;+const RY15: i32 = 9798;+const GY15: i32 = 19235;+const BY15: i32 = 3735;++fn require_channels(src: &ImageU8, channels: i32, op: &str) -> OpResult<()> {+ if src.channels != channels {+ return Err(format!(+ "{op}: expected {channels} channel(s), got {}",+ src.channels+ ));+ }+ Ok(())+}++pub(crate) fn bgr_to_rgb(src: &ImageU8) -> OpResult<ImageU8> {+ require_channels(src, 3, "bgr_to_rgb")?;+ let mut data = vec![0u8; src.data.len()];+ super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {+ for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {+ out[0] = px[2];+ out[1] = px[1];+ out[2] = px[0];+ }+ });+ ImageU8::new(src.width, src.height, 3, data)+}++pub(crate) fn gray_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {+ require_channels(src, 1, "gray_to_bgr")?;+ let mut data = vec![0u8; src.data.len() * 3];+ super::pointwise(&mut data, 3, &src.data, 1, |data, srcd| {+ for (out, &v) in data.chunks_exact_mut(3).zip(srcd.iter()) {+ out[0] = v;+ out[1] = v;+ out[2] = v;+ }+ });+ ImageU8::new(src.width, src.height, 3, data)+}++pub(crate) fn bgr_to_gray(src: &ImageU8) -> OpResult<ImageU8> {+ require_channels(src, 3, "bgr_to_gray")?;+ let round = 1i32 << (GRAY_SHIFT - 1);+ let mut data = vec![0u8; src.data.len() / 3];+ super::pointwise(&mut data, 1, &src.data, 3, |data, srcd| {+ for (out, px) in data.iter_mut().zip(srcd.chunks_exact(3)) {+ let sum = px[0] as i32 * BY15 + px[1] as i32 * GY15 + px[2] as i32 * RY15;+ *out = ((sum + round) >> GRAY_SHIFT) as u8;+ }+ });+ ImageU8::new(src.width, src.height, 1, data)+}++pub(crate) fn split_u8(src: &ImageU8) -> OpResult<Vec<ImageU8>> {+ let cn = src.channels as usize;+ let mut planes = Vec::with_capacity(cn);+ for c in 0..cn {+ let mut data = vec![0u8; src.data.len() / cn];+ super::pointwise(&mut data, 1, &src.data, cn, |data, srcd| {+ for (out, px) in data.iter_mut().zip(srcd.chunks_exact(cn)) {+ *out = px[c];+ }+ });+ planes.push(ImageU8::new(src.width, src.height, 1, data)?);+ }+ Ok(planes)+}++pub(crate) fn merge_u8(channels: &[ImageU8]) -> OpResult<ImageU8> {+ let first = channels+ .first()+ .ok_or_else(|| "merge_u8: no input planes".to_string())?;+ for plane in channels {+ if plane.channels != 1 {+ return Err("merge_u8: every input plane must be single-channel".to_string());+ }+ if plane.width != first.width || plane.height != first.height {+ return Err("merge_u8: input planes have different sizes".to_string());+ }+ }+ let cn = channels.len();+ let mut data = vec![0u8; first.data.len() * cn];+ for (c, plane) in channels.iter().enumerate() {+ for (i, &v) in plane.data.iter().enumerate() {+ data[i * cn + c] = v;+ }+ }+ ImageU8::new(first.width, first.height, cn as i32, data)+}diff --git a/rust/crates/ml/src/cv/contours.rs b/rust/crates/ml/src/cv/contours.rsnew file mode 100644index 0000000000..73252cbd50--- /dev/null+++ b/rust/crates/ml/src/cv/contours.rs@@ -0,0 +1,25 @@+use image::GrayImage;++use crate::cv::OpResult;+use crate::cv::image::{Contour, ImageU8};++pub(crate) fn find_contours(src: &ImageU8) -> OpResult<Vec<Contour>> {+ if src.channels != 1 {+ return Err(format!(+ "find_contours: expected a single-channel image, got {}",+ src.channels+ ));+ }+ let img = GrayImage::from_raw(src.width as u32, src.height as u32, src.data.clone())+ .ok_or_else(|| "find_contours: source buffer mismatch".to_string())?;+ Ok(imageproc::contours::find_contours::<i32>(&img)+ .into_iter()+ .map(|c| {+ let area = imageproc::geometry::contour_area(&c.points);+ Contour {+ points: c.points.into_iter().map(|p| (p.x, p.y)).collect(),+ area,+ }+ })+ .collect())+}diff --git a/rust/crates/ml/src/cv/convert.rs b/rust/crates/ml/src/cv/convert.rsnew file mode 100644index 0000000000..38d38a6521--- /dev/null+++ b/rust/crates/ml/src/cv/convert.rs@@ -0,0 +1,23 @@+use super::saturate_u8_f32;+use crate::cv::OpResult;+use crate::cv::image::{ImageF32, ImageU8};++pub(crate) fn u8_to_f32(src: &ImageU8) -> OpResult<ImageF32> {+ let mut data = vec![0.0f32; src.data.len()];+ super::pointwise(&mut data, 1, &src.data, 1, |o, s| {+ for (o, &v) in o.iter_mut().zip(s) {+ *o = v as f32;+ }+ });+ ImageF32::new(src.width, src.height, src.channels, data)+}++pub(crate) fn f32_to_u8(src: &ImageF32) -> OpResult<ImageU8> {+ let mut data = vec![0u8; src.data.len()];+ super::pointwise(&mut data, 1, &src.data, 1, |o, s| {+ for (o, &v) in o.iter_mut().zip(s) {+ *o = saturate_u8_f32(v);+ }+ });+ ImageU8::new(src.width, src.height, src.channels, data)+}diff --git a/rust/crates/ml/src/cv/draw.rs b/rust/crates/ml/src/cv/draw.rsnew file mode 100644index 0000000000..4db69545fa--- /dev/null+++ b/rust/crates/ml/src/cv/draw.rs@@ -0,0 +1,48 @@+use image::{GrayImage, Luma};+use imageproc::drawing::draw_polygon_mut;+use imageproc::point::Point;++use super::saturate_u8_f64;+use crate::cv::OpResult;+use crate::cv::image::ImageU8;++pub(crate) fn fill_poly(+ width: i32,+ height: i32,+ polygon: &[(i32, i32)],+ value: f64,+) -> OpResult<ImageU8> {+ if width <= 0 || height <= 0 {+ return Err(format!("fill_poly: invalid mask size {width}x{height}"));+ }+ let mut canvas = GrayImage::new(width as u32, height as u32);+ let color = Luma([saturate_u8_f64(value)]);++ // imageproc wants an open path with distinct first/last points.+ let mut pts: Vec<Point<i32>> = Vec::with_capacity(polygon.len());+ for &(x, y) in polygon {+ if pts.last() != Some(&Point::new(x, y)) {+ pts.push(Point::new(x, y));+ }+ }+ while pts.len() > 1 && pts.first() == pts.last() {+ pts.pop();+ }++ match pts.as_slice() {+ [] => {}+ [p] => {+ if (0..width).contains(&p.x) && (0..height).contains(&p.y) {+ canvas.put_pixel(p.x as u32, p.y as u32, color);+ }+ }+ [a, b] => imageproc::drawing::draw_line_segment_mut(+ &mut canvas,+ (a.x as f32, a.y as f32),+ (b.x as f32, b.y as f32),+ color,+ ),+ _ => draw_polygon_mut(&mut canvas, &pts, color),+ }+ ImageU8::new(width, height, 1, canvas.into_raw())+}diff --git a/rust/crates/ml/src/cv/filter.rs b/rust/crates/ml/src/cv/filter.rsnew file mode 100644index 0000000000..a305273c40--- /dev/null+++ b/rust/crates/ml/src/cv/filter.rs@@ -0,0 +1,116 @@+use crate::cv::OpResult;+use crate::cv::image::{ImageF32, ImageU8};++use super::reflect101;++/// Taps sum to 256 (8 fractional bits).+fn gaussian_taps(n: i32) -> OpResult<&'static [u32]> {+ Ok(match n {+ 1 => &[256],+ 3 => &[64, 128, 64],+ 5 => &[16, 64, 96, 64, 16],+ 7 => &[8, 28, 56, 72, 56, 28, 8],+ _ => return Err(format!("gaussian_blur_u8: unsupported ksize {n}")),+ })+}++pub(crate) fn gaussian_blur_u8(src: &ImageU8, ksize: i32) -> OpResult<ImageU8> {+ if src.channels != 1 {+ return Err("gaussian_blur_u8: expected a single-channel image".to_string());+ }+ let kw = if src.width == 1 { 1 } else { ksize };+ let kh = if src.height == 1 { 1 } else { ksize };+ if kw == 1 && kh == 1 {+ return Ok(src.clone());+ }+ let kx = gaussian_taps(kw)?;+ let ky = gaussian_taps(kh)?;+ let (w, h) = (src.width as usize, src.height as usize);+ let (ax, ay) = ((kw / 2) as i64, (kh / 2) as i64);++ // Horizontal pass into u16 (taps sum to 256), vertical into u32.+ let mut hsum = vec![0u16; w * h];+ for y in 0..h {+ let row = &src.data[y * w..(y + 1) * w];+ let out = &mut hsum[y * w..(y + 1) * w];+ for (x, o) in out.iter_mut().enumerate() {+ let mut acc = 0u16;+ for (t, &tap) in kx.iter().enumerate() {+ let sx = reflect101(x as i64 + t as i64 - ax, src.width) as usize;+ acc += tap as u16 * row[sx] as u16;+ }+ *o = acc;+ }+ }+ let mut out = vec![0u8; w * h];+ let mut acc = vec![0u32; w];+ for y in 0..h {+ let dst = &mut out[y * w..(y + 1) * w];+ acc.iter_mut().for_each(|a| *a = 0);+ for (t, &tap) in ky.iter().enumerate() {+ let sy = reflect101(y as i64 + t as i64 - ay, src.height) as usize;+ for (a, &v) in acc.iter_mut().zip(&hsum[sy * w..(sy + 1) * w]) {+ *a += tap * v as u32;+ }+ }+ for (d, &a) in dst.iter_mut().zip(acc.iter()) {+ *d = ((a + (1 << 15)) >> 16).min(255) as u8;+ }+ }+ ImageU8::new(src.width, src.height, 1, out)+}++pub(crate) fn box_filter_f32(src: &ImageF32, kw: i32, kh: i32) -> OpResult<ImageF32> {+ if kw <= 0 || kh <= 0 {+ return Err(format!("box_filter_f32: invalid kernel {kw}x{kh}"));+ }+ if src.channels != 1 {+ return Err("box_filter_f32: expected a single-channel image".to_string());+ }+ let (w, h) = (src.width as usize, src.height as usize);+ let (ax, ay) = ((kw / 2) as i64, (kh / 2) as i64);+ let scale = 1.0 / (kw as f64 * kh as f64);++ let mut hsum = vec![0.0f64; w * h];+ let ext_idx: Vec<usize> = (0..w + kw as usize - 1)+ .map(|j| reflect101(j as i64 - ax, src.width) as usize)+ .collect();+ let mut ext = vec![0.0f64; w + kw as usize - 1];+ for y in 0..h {+ let row = &src.data[y * w..(y + 1) * w];+ for (e, &sx) in ext.iter_mut().zip(ext_idx.iter()) {+ *e = row[sx] as f64;+ }+ let out = &mut hsum[y * w..(y + 1) * w];+ let mut s: f64 = ext[..kw as usize].iter().sum();+ out[0] = s;+ for x in 1..w {+ s += ext[x + kw as usize - 1] - ext[x - 1];+ out[x] = s;+ }+ }++ let mut out = vec![0.0f32; w * h];+ let mut col = vec![0.0f64; w];+ for t in 0..kh as usize - 1 {+ let sy = reflect101(t as i64 - ay, src.height) as usize;+ for (c, &v) in col.iter_mut().zip(&hsum[sy * w..(sy + 1) * w]) {+ *c += v;+ }+ }+ for y in 0..h {+ let enter = reflect101(y as i64 + kh as i64 - 1 - ay, src.height) as usize;+ for (c, &v) in col.iter_mut().zip(&hsum[enter * w..(enter + 1) * w]) {+ *c += v;+ }+ let dst = &mut out[y * w..(y + 1) * w];+ for (d, &c) in dst.iter_mut().zip(col.iter()) {+ *d = (c * scale) as f32;+ }+ let leave = reflect101(y as i64 - ay, src.height) as usize;+ for (c, &v) in col.iter_mut().zip(&hsum[leave * w..(leave + 1) * w]) {+ *c -= v;+ }+ }+ ImageF32::new(src.width, src.height, 1, out)+}diff --git a/rust/crates/ml/src/cv/image.rs b/rust/crates/ml/src/cv/image.rsnew file mode 100644index 0000000000..a341ae86ce--- /dev/null+++ b/rust/crates/ml/src/cv/image.rs@@ -0,0 +1,148 @@+//! Row-major, channel-interleaved, fully packed (no row padding).++use super::OpResult;++#[derive(Clone, Debug, PartialEq, Eq)]+pub(crate) struct ImageU8 {+ pub width: i32,+ pub height: i32,+ pub channels: i32,+ pub data: Vec<u8>,+}++#[derive(Clone, Debug, PartialEq)]+pub(crate) struct ImageF32 {+ pub width: i32,+ pub height: i32,+ pub channels: i32,+ pub data: Vec<f32>,+}++fn check_len(width: i32, height: i32, channels: i32, len: usize) -> OpResult<()> {+ if width <= 0 || height <= 0 || channels <= 0 {+ return Err(format!(+ "invalid image geometry {width}x{height}x{channels}"+ ));+ }+ let expected = width as usize * height as usize * channels as usize;+ if len != expected {+ return Err(format!(+ "image buffer length {len} does not match {width}x{height}x{channels} ({expected})"+ ));+ }+ Ok(())+}++impl ImageU8 {+ pub(crate) fn new(width: i32, height: i32, channels: i32, data: Vec<u8>) -> OpResult<Self> {+ check_len(width, height, channels, data.len())?;+ Ok(Self {+ width,+ height,+ channels,+ data,+ })+ }++ pub(crate) fn zeros(width: i32, height: i32, channels: i32) -> OpResult<Self> {+ if width <= 0 || height <= 0 || channels <= 0 {+ return Err(format!(+ "invalid image geometry {width}x{height}x{channels}"+ ));+ }+ let len = width as usize * height as usize * channels as usize;+ Ok(Self {+ width,+ height,+ channels,+ data: vec![0u8; len],+ })+ }++ pub(crate) fn size(&self) -> (i32, i32) {+ (self.width, self.height)+ }++ pub(crate) fn same_geometry(&self, other: &ImageU8) -> bool {+ self.width == other.width && self.height == other.height && self.channels == other.channels+ }+}++impl ImageF32 {+ pub(crate) fn new(width: i32, height: i32, channels: i32, data: Vec<f32>) -> OpResult<Self> {+ check_len(width, height, channels, data.len())?;+ Ok(Self {+ width,+ height,+ channels,+ data,+ })+ }++ pub(crate) fn zeros(width: i32, height: i32, channels: i32) -> OpResult<Self> {+ if width <= 0 || height <= 0 || channels <= 0 {+ return Err(format!(+ "invalid image geometry {width}x{height}x{channels}"+ ));+ }+ let len = width as usize * height as usize * channels as usize;+ Ok(Self {+ width,+ height,+ channels,+ data: vec![0.0f32; len],+ })+ }++ pub(crate) fn pixels(&self) -> usize {+ self.width as usize * self.height as usize+ }++ pub(crate) fn same_geometry(&self, other: &ImageF32) -> bool {+ self.width == other.width && self.height == other.height && self.channels == other.channels+ }+}++#[derive(Clone, Copy, Debug)]+pub(crate) enum ImageRef<'a> {+ U8(&'a ImageU8),+ F32(&'a ImageF32),+}++#[derive(Clone, Debug)]+pub(crate) enum Image {+ U8(ImageU8),+ F32(ImageF32),+}++impl ImageRef<'_> {+ pub(crate) fn size(&self) -> (i32, i32) {+ match self {+ ImageRef::U8(i) => i.size(),+ ImageRef::F32(i) => (i.width, i.height),+ }+ }+}++impl Image {+ pub(crate) fn into_u8(self) -> OpResult<ImageU8> {+ match self {+ Image::U8(i) => Ok(i),+ Image::F32(_) => Err("expected an 8-bit image, got 32-bit float".to_string()),+ }+ }++ pub(crate) fn into_f32(self) -> OpResult<ImageF32> {+ match self {+ Image::F32(i) => Ok(i),+ Image::U8(_) => Err("expected a 32-bit float image, got 8-bit".to_string()),+ }+ }+}++/// `area` is the shoelace area.+#[derive(Clone, Debug, PartialEq)]+pub(crate) struct Contour {+ pub points: Vec<(i32, i32)>,+ pub area: f64,+}diff --git a/rust/crates/ml/src/cv/lab.rs b/rust/crates/ml/src/cv/lab.rsnew file mode 100644index 0000000000..53c88ac7ac--- /dev/null+++ b/rust/crates/ml/src/cv/lab.rs@@ -0,0 +1,239 @@+//! 8-bit BGR <-> Lab, as fixed-point dot products over precomputed tables.+//!+//! L, a and b are all scaled onto 0..255 (a and b biased by 128); the+//! pipeline's chroma and luminance thresholds are calibrated in these units.++use std::sync::OnceLock;++use crate::cv::OpResult;+use crate::cv::image::ImageU8;++const LAB_SHIFT: i32 = 12;+const LAB_SHIFT2: i32 = 15;+const GAMMA_SHIFT: i32 = 3;+const INV_GAMMA_SHIFT: i32 = 12;+const INV_GAMMA_TAB_SIZE: usize = 1 << 12;+const LAB_CBRT_TAB_SIZE: usize = 256 * 3 / 2 * (1 << GAMMA_SHIFT);+const BASE_SHIFT: i32 = 14;+const LAB_BASE: i32 = 1 << BASE_SHIFT;+const MIN_AB_VALUE: i32 = -8145;+const AB_TO_XZ_LEN: usize = (LAB_BASE as usize) * 9 / 4;+const LAB2RGB_SHIFT: i32 = LAB_SHIFT + (BASE_SHIFT - INV_GAMMA_SHIFT);++// sRGB <-> XYZ (D65), rows normalized by the white point during table build.+const SRGB2XYZ_D65: [f64; 9] = [+ 0.412453, 0.35758, 0.180423, 0.212671, 0.71516, 0.072169, 0.019334, 0.119193, 0.950227,+];+const XYZ2SRGB_D65: [f64; 9] = [+ 3.240479, -1.53715, -0.498535, -0.969256, 1.875991, 0.041556, 0.055648, -0.204043, 1.057311,+];+const D65: [f64; 3] = [0.950456, 1.0, 1.088754];++const LTHRESH: f32 = 216.0 / 24389.0;+const LSCALE: f32 = 841.0 / 108.0;+const LBIAS: f32 = 16.0 / 116.0;++const GAMMA_THRESHOLD: f64 = 809.0 / 20000.0;+const GAMMA_INV_THRESHOLD: f64 = 7827.0 / 2500000.0;+const GAMMA_LOW_SCALE: f64 = 323.0 / 25.0;+const GAMMA_POWER: f64 = 12.0 / 5.0;+const GAMMA_XSHIFT: f64 = 11.0 / 200.0;++#[inline]+fn descale(x: i32, n: i32) -> i32 {+ (x + (1 << (n - 1))) >> n+}++#[inline]+fn round_i32(x: f64) -> i32 {+ x.round_ties_even() as i32+}++#[inline]+fn saturate_u8(x: i32) -> u8 {+ x.clamp(0, 255) as u8+}++fn apply_gamma(x: f64) -> f64 {+ if x <= GAMMA_THRESHOLD {+ x / GAMMA_LOW_SCALE+ } else {+ ((x + GAMMA_XSHIFT) / (1.0 + GAMMA_XSHIFT)).powf(GAMMA_POWER)+ }+}++fn apply_inv_gamma(x: f64) -> f64 {+ if x <= GAMMA_INV_THRESHOLD {+ x * GAMMA_LOW_SCALE+ } else {+ x.powf(1.0 / GAMMA_POWER) * (1.0 + GAMMA_XSHIFT) - GAMMA_XSHIFT+ }+}++struct LabTabs {+ srgb_gamma: [u16; 256],+ lab_cbrt: [u16; LAB_CBRT_TAB_SIZE],+ srgb_inv_gamma: Box<[u16]>,+ lab_to_yf: [u16; 512],+ ab_to_xz: Box<[i32]>,+ /// Ordered so a BGR pixel is consumed in storage order.+ fwd_coeffs: [i32; 9],+ /// Rows are R, G, B.+ inv_coeffs: [i32; 9],+}++fn build_tabs() -> LabTabs {+ let mut srgb_gamma = [0u16; 256];+ let int_scale = (255 * (1 << GAMMA_SHIFT)) as f64;+ for (i, out) in srgb_gamma.iter_mut().enumerate() {+ *out = round_i32(int_scale * apply_gamma(i as f64 / 255.0)) as u16;+ }++ let mut srgb_inv_gamma = vec![0u16; INV_GAMMA_TAB_SIZE];+ for (i, out) in srgb_inv_gamma.iter_mut().enumerate() {+ let x = i as f64 / INV_GAMMA_TAB_SIZE as f64;+ *out = round_i32(255.0 * apply_inv_gamma(x)) as u16;+ }++ let mut lab_cbrt = [0u16; LAB_CBRT_TAB_SIZE];+ let cb_tab_scale = 1.0f32 / (255.0 * (1 << GAMMA_SHIFT) as f32);+ let lshift2 = (1 << LAB_SHIFT2) as f32;+ for (i, out) in lab_cbrt.iter_mut().enumerate() {+ let x = cb_tab_scale * i as f32;+ let f = if x < LTHRESH {+ x.mul_add(LSCALE, LBIAS)+ } else {+ x.cbrt()+ };+ *out = round_i32((lshift2 * f) as f64) as u16;+ }++ let mut lab_to_yf = [0u16; 512];+ for i in 0..256i32 {+ const LUT_BASE: i32 = 1 << 14;+ let (y, ify) = if i <= 20 {+ (+ round_i32((i * LUT_BASE * 20 * 9) as f64 / (17 * 29 * 29 * 29) as f64),+ round_i32(LUT_BASE as f64 * (16.0 / 116.0 + (i * 5) as f64 / (3 * 17 * 29) as f64)),+ )+ } else {+ let fy =+ (i * 100 * LUT_BASE) as f64 / (255 * 116) as f64 + (16 * LUT_BASE) as f64 / 116.0;+ (+ round_i32(fy * fy * fy / (LUT_BASE * LUT_BASE) as f64),+ round_i32(fy),+ )+ };+ lab_to_yf[i as usize * 2] = y as u16;+ lab_to_yf[i as usize * 2 + 1] = ify as u16;+ }++ // Integer division truncates toward zero; `i` starts negative.+ let mut ab_to_xz = vec![0i32; AB_TO_XZ_LEN];+ let bias = LAB_BASE * 16 / 116 * 108 / 841;+ for (k, out) in ab_to_xz.iter_mut().enumerate() {+ let i = k as i32 + MIN_AB_VALUE;+ *out = if i <= 3390 {+ i * 108 / 841 - bias+ } else {+ i * i / LAB_BASE * i / LAB_BASE+ };+ }++ let lshift = (1 << LAB_SHIFT) as f64;+ let mut fwd_coeffs = [0i32; 9];+ let mut inv_coeffs = [0i32; 9];+ for i in 0..3 {+ let wp = D65[i];+ fwd_coeffs[i * 3 + 2] = round_i32(lshift * SRGB2XYZ_D65[i * 3] / wp);+ fwd_coeffs[i * 3 + 1] = round_i32(lshift * SRGB2XYZ_D65[i * 3 + 1] / wp);+ fwd_coeffs[i * 3] = round_i32(lshift * SRGB2XYZ_D65[i * 3 + 2] / wp);++ inv_coeffs[i] = round_i32(lshift * XYZ2SRGB_D65[i] * wp);+ inv_coeffs[i + 3] = round_i32(lshift * XYZ2SRGB_D65[i + 3] * wp);+ inv_coeffs[i + 6] = round_i32(lshift * XYZ2SRGB_D65[i + 6] * wp);+ }++ LabTabs {+ srgb_gamma,+ lab_cbrt,+ srgb_inv_gamma: srgb_inv_gamma.into_boxed_slice(),+ lab_to_yf,+ ab_to_xz: ab_to_xz.into_boxed_slice(),+ fwd_coeffs,+ inv_coeffs,+ }+}++fn tabs() -> &'static LabTabs {+ static TABS: OnceLock<LabTabs> = OnceLock::new();+ TABS.get_or_init(build_tabs)+}++fn require_3ch(src: &ImageU8, op: &str) -> OpResult<()> {+ if src.channels != 3 {+ return Err(format!("{op}: expected 3 channel(s), got {}", src.channels));+ }+ Ok(())+}++pub(crate) fn bgr_to_lab(src: &ImageU8) -> OpResult<ImageU8> {+ require_3ch(src, "bgr_to_lab")?;+ let t = tabs();+ let c = t.fwd_coeffs;+ let l_scale = (116 * 255 + 50) / 100;+ let l_shift = -((16 * 255 * (1 << LAB_SHIFT2) + 50) / 100);+ let ab_shift = 128 * (1 << LAB_SHIFT2);++ let mut data = vec![0u8; src.data.len()];+ super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {+ for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {+ let v0 = t.srgb_gamma[px[0] as usize] as i32;+ let v1 = t.srgb_gamma[px[1] as usize] as i32;+ let v2 = t.srgb_gamma[px[2] as usize] as i32;+ let fx =+ t.lab_cbrt[descale(v0 * c[0] + v1 * c[1] + v2 * c[2], LAB_SHIFT) as usize] as i32;+ let fy =+ t.lab_cbrt[descale(v0 * c[3] + v1 * c[4] + v2 * c[5], LAB_SHIFT) as usize] as i32;+ let fz =+ t.lab_cbrt[descale(v0 * c[6] + v1 * c[7] + v2 * c[8], LAB_SHIFT) as usize] as i32;++ out[0] = saturate_u8(descale(l_scale * fy + l_shift, LAB_SHIFT2));+ out[1] = saturate_u8(descale(500 * (fx - fy) + ab_shift, LAB_SHIFT2));+ out[2] = saturate_u8(descale(200 * (fy - fz) + ab_shift, LAB_SHIFT2));+ }+ });+ ImageU8::new(src.width, src.height, 3, data)+}++pub(crate) fn lab_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {+ require_3ch(src, "lab_to_bgr")?;+ let t = tabs();+ let c = t.inv_coeffs;+ let a_bias = 128 * LAB_BASE / 500;+ let b_bias = 128 * LAB_BASE / 200 - 1;+ let top = INV_GAMMA_TAB_SIZE as i32 - 1;++ let mut data = vec![0u8; src.data.len()];+ super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {+ for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {+ let y = t.lab_to_yf[px[0] as usize * 2] as i32;+ let ify = t.lab_to_yf[px[0] as usize * 2 + 1] as i32;++ let adiv = px[1] as i32 * LAB_BASE / 500 - a_bias;+ let bdiv = px[2] as i32 * LAB_BASE / 200 - b_bias;++ let x = t.ab_to_xz[(ify + adiv - MIN_AB_VALUE) as usize];+ let z = t.ab_to_xz[(ify - bdiv - MIN_AB_VALUE) as usize];++ let r = descale(c[0] * x + c[1] * y + c[2] * z, LAB2RGB_SHIFT).clamp(0, top);+ let g = descale(c[3] * x + c[4] * y + c[5] * z, LAB2RGB_SHIFT).clamp(0, top);+ let b = descale(c[6] * x + c[7] * y + c[8] * z, LAB2RGB_SHIFT).clamp(0, top);++ out[0] = saturate_u8(t.srgb_inv_gamma[b as usize] as i32);+ out[1] = saturate_u8(t.srgb_inv_gamma[g as usize] as i32);+ out[2] = saturate_u8(t.srgb_inv_gamma[r as usize] as i32);+ }+ });+ ImageU8::new(src.width, src.height, 3, data)+}diff --git a/rust/crates/ml/src/cv/masking.rs b/rust/crates/ml/src/cv/masking.rsnew file mode 100644index 0000000000..e1adb777f9--- /dev/null+++ b/rust/crates/ml/src/cv/masking.rs@@ -0,0 +1,92 @@+use crate::cv::OpResult;+use crate::cv::image::{ImageF32, ImageU8};++pub(crate) fn threshold_binary_u8(src: &ImageU8, thresh: f64, maxval: f64) -> OpResult<ImageU8> {+ let ithresh = thresh.floor() as i64;+ let imaxval = maxval.round_ties_even().clamp(0.0, 255.0) as u8;+ ImageU8::new(+ src.width,+ src.height,+ src.channels,+ src.data+ .iter()+ .map(|&v| if v as i64 > ithresh { imaxval } else { 0 })+ .collect(),+ )+}++pub(crate) fn threshold_binary_f32(src: &ImageF32, thresh: f64, maxval: f64) -> OpResult<ImageF32> {+ let t = thresh as f32;+ let m = maxval as f32;+ ImageF32::new(+ src.width,+ src.height,+ src.channels,+ src.data+ .iter()+ .map(|&v| if v > t { m } else { 0.0 })+ .collect(),+ )+}++pub(crate) fn in_range_u8(src: &ImageU8, lower: f64, upper: f64) -> OpResult<ImageU8> {+ if src.channels != 1 {+ return Err("in_range_u8: expected a single-channel image".to_string());+ }+ let lo = lower.round_ties_even().clamp(0.0, 255.0) as u8;+ let hi = upper.round_ties_even().clamp(0.0, 255.0) as u8;+ ImageU8::new(+ src.width,+ src.height,+ 1,+ src.data+ .iter()+ .map(|&v| if lo <= v && v <= hi { 255 } else { 0 })+ .collect(),+ )+}++pub(crate) fn bitwise_and_u8(a: &ImageU8, b: &ImageU8) -> OpResult<ImageU8> {+ if !a.same_geometry(b) {+ return Err("bitwise_and_u8: operands have different geometry".to_string());+ }+ ImageU8::new(+ a.width,+ a.height,+ a.channels,+ a.data+ .iter()+ .zip(b.data.iter())+ .map(|(&x, &y)| x & y)+ .collect(),+ )+}++pub(crate) fn count_non_zero(src: &ImageU8) -> OpResult<i32> {+ if src.channels != 1 {+ return Err(format!(+ "count_non_zero: expected a single-channel image, got {}",+ src.channels+ ));+ }+ Ok(src.data.iter().filter(|&&v| v != 0).count() as i32)+}++pub(crate) fn copy_to_masked(src: &ImageU8, dst: &ImageU8, mask: &ImageU8) -> OpResult<ImageU8> {+ if !src.same_geometry(dst) {+ return Err("copy_to_masked: source and destination geometry differ".to_string());+ }+ if mask.channels != 1 || mask.width != src.width || mask.height != src.height {+ return Err("copy_to_masked: mask must be single-channel and the same size".to_string());+ }+ let cn = src.channels as usize;+ let mut out = dst.data.clone();+ super::pointwise3(&mut out, cn, &src.data, cn, &mask.data, 1, |o, s, m| {+ for (i, &m) in m.iter().enumerate() {+ if m != 0 {+ o[i * cn..(i + 1) * cn].copy_from_slice(&s[i * cn..(i + 1) * cn]);+ }+ }+ });+ ImageU8::new(src.width, src.height, src.channels, out)+}diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rsnew file mode 100644index 0000000000..0c4459a49f--- /dev/null+++ b/rust/crates/ml/src/cv/mod.rs@@ -0,0 +1,129 @@+mod arith;+mod bilateral;+mod canny;+mod channels;+mod contours;+mod convert;+mod draw;+mod filter;+pub(crate) mod image;+mod lab;+mod masking;+mod morph;+mod resize;+mod stats;+mod structuring;+mod transform;+mod warp;++use rayon::prelude::*;++use self::image::{Image, ImageRef};++pub(crate) type OpResult<T> = Result<T, String>;++pub(crate) use arith::{+ add_f32_scalar, add_weighted_f32, exp_f32, log_f32, magnitude_f32, max_f32_scalar,+ min_f32_scalar, multiply_f32, multiply_f32_scalar, subtract_f32, subtract_f32_scalar,+};+pub(crate) use bilateral::bilateral_filter_u8;+pub(crate) use canny::canny;+pub(crate) use channels::{bgr_to_gray, bgr_to_rgb, gray_to_bgr, merge_u8, split_u8};+pub(crate) use contours::find_contours;+pub(crate) use convert::{f32_to_u8, u8_to_f32};+pub(crate) use draw::fill_poly;+pub(crate) use filter::{box_filter_f32, gaussian_blur_u8};+pub(crate) use lab::{bgr_to_lab, lab_to_bgr};+pub(crate) use masking::{+ bitwise_and_u8, copy_to_masked, count_non_zero, in_range_u8, threshold_binary_f32,+ threshold_binary_u8,+};+pub(crate) use morph::{morphology_close, morphology_erode, morphology_open};+pub(crate) use stats::{+ hist_256_f32, hist_256_u8, mean_f32, mean_u8c3_masked, min_max_loc_f32, percentile_f32, sum_f32,+};+pub(crate) use structuring::ellipse_kernel;+pub(crate) use transform::rotate_u8;+pub(crate) use warp::warp_perspective;++use resize::Interp;++pub(crate) fn resize_bilinear(src: ImageRef<'_>, width: i32, height: i32) -> OpResult<Image> {+ resize::resize(src, width, height, Interp::Bilinear)+}++pub(crate) fn resize_area(src: ImageRef<'_>, width: i32, height: i32) -> OpResult<Image> {+ resize::resize(src, width, height, Interp::Area)+}++pub(crate) fn resize_bicubic(src: ImageRef<'_>, width: i32, height: i32) -> OpResult<Image> {+ resize::resize(src, width, height, Interp::Bicubic)+}++const PARALLEL_MIN_PIXELS: usize = 65_536;+const PIXELS_PER_CHUNK: usize = 65_536;++pub(crate) fn pointwise<T: Sync, U: Send>(+ out: &mut [U],+ out_stride: usize,+ src: &[T],+ src_stride: usize,+ f: impl Fn(&mut [U], &[T]) + Send + Sync,+) {+ if out.len() / out_stride.max(1) < PARALLEL_MIN_PIXELS {+ f(out, src);+ return;+ }+ out.par_chunks_mut(PIXELS_PER_CHUNK * out_stride)+ .zip(src.par_chunks(PIXELS_PER_CHUNK * src_stride))+ .for_each(|(o, s)| f(o, s));+}++pub(crate) fn pointwise3<T: Sync, V: Sync, U: Send>(+ out: &mut [U],+ out_stride: usize,+ a: &[T],+ a_stride: usize,+ b: &[V],+ b_stride: usize,+ f: impl Fn(&mut [U], &[T], &[V]) + Send + Sync,+) {+ if out.len() / out_stride.max(1) < PARALLEL_MIN_PIXELS {+ f(out, a, b);+ return;+ }+ out.par_chunks_mut(PIXELS_PER_CHUNK * out_stride)+ .zip(a.par_chunks(PIXELS_PER_CHUNK * a_stride))+ .zip(b.par_chunks(PIXELS_PER_CHUNK * b_stride))+ .for_each(|((o, a), b)| f(o, a, b));+}++pub(crate) fn saturate_u8_f32(v: f32) -> u8 {+ v.round_ties_even().clamp(0.0, 255.0) as u8+}++pub(crate) fn saturate_u8_f64(v: f64) -> u8 {+ v.round_ties_even().clamp(0.0, 255.0) as u8+}++/// Mirrors an out-of-range coordinate back into `[0, len)` without repeating+/// the edge pixel.+pub(crate) fn reflect101(mut p: i64, len: i32) -> i64 {+ let len = len as i64;+ if p >= 0 && p < len {+ return p;+ }+ if len == 1 {+ return 0;+ }+ loop {+ if p < 0 {+ p = -p;+ } else {+ p = len - 1 - (p - len) - 1;+ }+ if p >= 0 && p < len {+ return p;+ }+ }+}diff --git a/rust/crates/ml/src/cv/morph.rs b/rust/crates/ml/src/cv/morph.rsnew file mode 100644index 0000000000..9f384e4ca6--- /dev/null+++ b/rust/crates/ml/src/cv/morph.rs@@ -0,0 +1,78 @@+use rayon::prelude::*;++use crate::cv::OpResult;+use crate::cv::image::ImageU8;++const PARALLEL_MIN_PIXELS: usize = 200_000;++fn morph(src: &ImageU8, kernel: &ImageU8, erode: bool) -> OpResult<ImageU8> {+ if src.channels != 1 || kernel.channels != 1 {+ return Err("morphology: image and kernel must be single-channel".to_string());+ }+ if kernel.width * kernel.height == 1 {+ return Ok(src.clone());+ }+ let pad: u8 = if erode { 255 } else { 0 };+ let (w, h) = (src.width as usize, src.height as usize);+ let (kw, kh) = (kernel.width as usize, kernel.height as usize);++ let pw = w + kw - 1;+ let mut padded = vec![pad; pw * (h + kh - 1)];+ for y in 0..h {+ padded[(y + kh / 2) * pw + kw / 2..(y + kh / 2) * pw + kw / 2 + w]+ .copy_from_slice(&src.data[y * w..(y + 1) * w]);+ }++ let offsets: Vec<usize> = (0..kh * kw)+ .filter(|&i| kernel.data[i] != 0)+ .map(|i| (i / kw) * pw + i % kw)+ .collect();+ if offsets.is_empty() {+ return ImageU8::new(src.width, src.height, 1, vec![pad; w * h]);+ }++ let mut out = vec![0u8; w * h];+ // Tap-major keeps the min/max folds vectorizable.+ let row = |y: usize, dst: &mut [u8]| {+ let base = y * pw;+ dst.copy_from_slice(&padded[base + offsets[0]..base + offsets[0] + w]);+ for &off in &offsets[1..] {+ let src_slice = &padded[base + off..base + off + w];+ if erode {+ for (d, &s) in dst.iter_mut().zip(src_slice) {+ *d = (*d).min(s);+ }+ } else {+ for (d, &s) in dst.iter_mut().zip(src_slice) {+ *d = (*d).max(s);+ }+ }+ }+ };+ if w * h >= PARALLEL_MIN_PIXELS {+ out.par_chunks_mut(w)+ .enumerate()+ .for_each(|(y, dst)| row(y, dst));+ } else {+ for (y, dst) in out.chunks_mut(w).enumerate() {+ row(y, dst);+ }+ }+ ImageU8::new(src.width, src.height, 1, out)+}++pub(crate) fn morphology_erode(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {+ morph(src, kernel, true)+}++fn morphology_dilate(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {+ morph(src, kernel, false)+}++pub(crate) fn morphology_open(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {+ morphology_dilate(&morphology_erode(src, kernel)?, kernel)+}++pub(crate) fn morphology_close(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {+ morphology_erode(&morphology_dilate(src, kernel)?, kernel)+}diff --git a/rust/crates/ml/src/cv/resize.rs b/rust/crates/ml/src/cv/resize.rsnew file mode 100644index 0000000000..7b33fb55ae--- /dev/null+++ b/rust/crates/ml/src/cv/resize.rs@@ -0,0 +1,106 @@+//! The `Interpolation` algs sample without low-pass filtering; downstream+//! calibration depends on that.++use fast_image_resize::{+ FilterType, PixelType, ResizeAlg, ResizeOptions, Resizer,+ images::{Image as FirImage, ImageRef as FirImageRef},+};++use crate::cv::OpResult;+use crate::cv::image::{Image, ImageF32, ImageRef, ImageU8};++#[derive(Clone, Copy, PartialEq, Eq, Debug)]+pub(crate) enum Interp {+ Bilinear,+ Area,+ Bicubic,+}++fn pixel_type(channels: i32, is_f32: bool) -> OpResult<PixelType> {+ Ok(match (is_f32, channels) {+ (false, 1) => PixelType::U8,+ (false, 3) => PixelType::U8x3,+ (true, 1) => PixelType::F32,+ (true, 3) => PixelType::F32x3,+ (is_f32, n) => {+ return Err(format!(+ "resize: unsupported {n}-channel {} image",+ if is_f32 { "f32" } else { "u8" }+ ));+ }+ })+}++fn run(+ src_bytes: &[u8],+ src_size: (i32, i32),+ pixel_type: PixelType,+ width: i32,+ height: i32,+ alg: ResizeAlg,+) -> OpResult<Vec<u8>> {+ let src = FirImageRef::new(src_size.0 as u32, src_size.1 as u32, src_bytes, pixel_type)+ .map_err(|e| format!("resize: bad source: {e}"))?;+ let mut dst = FirImage::new(width as u32, height as u32, pixel_type);+ Resizer::new()+ .resize(&src, &mut dst, Some(&ResizeOptions::new().resize_alg(alg)))+ .map_err(|e| format!("resize failed: {e}"))?;+ Ok(dst.into_vec())+}++pub(crate) fn resize(+ src: ImageRef<'_>,+ width: i32,+ height: i32,+ interp: Interp,+) -> OpResult<Image> {+ let (sw, sh) = src.size();+ if width <= 0 || height <= 0 {+ return Err(format!("resize: invalid destination size {width}x{height}"));+ }+ if width == sw && height == sh {+ return Ok(match src {+ ImageRef::U8(i) => Image::U8(i.clone()),+ ImageRef::F32(i) => Image::F32(i.clone()),+ });+ }++ let alg = match interp {+ Interp::Bilinear => ResizeAlg::Interpolation(FilterType::Bilinear),+ Interp::Bicubic => ResizeAlg::Interpolation(FilterType::CatmullRom),+ Interp::Area if width >= sw && height >= sh => {+ ResizeAlg::Interpolation(FilterType::Bilinear)+ }+ Interp::Area => ResizeAlg::Convolution(FilterType::Box),+ };++ match src {+ ImageRef::U8(s) => {+ let data = run(+ &s.data,+ s.size(),+ pixel_type(s.channels, false)?,+ width,+ height,+ alg,+ )?;+ Ok(Image::U8(ImageU8::new(width, height, s.channels, data)?))+ }+ ImageRef::F32(s) => {+ let bytes: Vec<u8> = s.data.iter().flat_map(|v| v.to_ne_bytes()).collect();+ let out = run(+ &bytes,+ (s.width, s.height),+ pixel_type(s.channels, true)?,+ width,+ height,+ alg,+ )?;+ let data: Vec<f32> = out+ .chunks_exact(4)+ .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]]))+ .collect();+ Ok(Image::F32(ImageF32::new(width, height, s.channels, data)?))+ }+ }+}diff --git a/rust/crates/ml/src/cv/stats.rs b/rust/crates/ml/src/cv/stats.rsnew file mode 100644index 0000000000..d88b7348d7--- /dev/null+++ b/rust/crates/ml/src/cv/stats.rs@@ -0,0 +1,90 @@+use crate::cv::OpResult;+use crate::cv::image::{ImageF32, ImageU8};++fn sum_channel0(src: &ImageF32) -> f64 {+ let cn = src.channels as usize;+ if cn == 1 {+ src.data.iter().map(|&v| v as f64).sum()+ } else {+ src.data.chunks_exact(cn).map(|px| px[0] as f64).sum()+ }+}++pub(crate) fn sum_f32(src: &ImageF32) -> OpResult<f64> {+ Ok(sum_channel0(src))+}++pub(crate) fn mean_f32(src: &ImageF32) -> OpResult<f64> {+ Ok(sum_channel0(src) / src.pixels() as f64)+}++pub(crate) fn mean_u8c3_masked(src: &ImageU8, mask: &ImageU8) -> OpResult<[f64; 3]> {+ if src.channels != 3 {+ return Err(format!(+ "mean_u8c3_masked: expected a 3-channel image, got {}",+ src.channels+ ));+ }+ if mask.channels != 1 || mask.width != src.width || mask.height != src.height {+ return Err("mean_u8c3_masked: mask must be single-channel and the same size".to_string());+ }+ let mut sums = [0i64; 3];+ let mut count = 0i64;+ for (px, &m) in src.data.chunks_exact(3).zip(mask.data.iter()) {+ if m != 0 {+ sums[0] += px[0] as i64;+ sums[1] += px[1] as i64;+ sums[2] += px[2] as i64;+ count += 1;+ }+ }+ if count == 0 {+ return Ok([0.0; 3]);+ }+ Ok(sums.map(|s| s as f64 / count as f64))+}++pub(crate) fn min_max_loc_f32(src: &ImageF32) -> OpResult<(f64, f64)> {+ let mut min_val = f32::INFINITY;+ let mut max_val = f32::NEG_INFINITY;+ for &v in &src.data {+ if v < min_val {+ min_val = v;+ }+ if v > max_val {+ max_val = v;+ }+ }+ Ok((min_val as f64, max_val as f64))+}++pub(crate) fn hist_256_u8(src: &ImageU8) -> OpResult<Vec<f64>> {+ let cn = src.channels as usize;+ let mut bins = [0u32; 256];+ for px in src.data.chunks_exact(cn) {+ bins[px[0] as usize] += 1;+ }+ Ok(bins.iter().map(|&c| c as f64).collect())+}++pub(crate) fn hist_256_f32(src: &ImageF32) -> OpResult<Vec<f64>> {+ let cn = src.channels as usize;+ let mut bins = [0u32; 256];+ for px in src.data.chunks_exact(cn) {+ let v = px[0] as f64;+ if (0.0..256.0).contains(&v) {+ bins[v as usize] += 1;+ }+ }+ Ok(bins.iter().map(|&c| c as f64).collect())+}++pub(crate) fn percentile_f32(src: &ImageF32, p: f64) -> OpResult<f32> {+ let mut values = src.data.clone();+ if values.is_empty() {+ return Err("percentile_f32: empty image".to_string());+ }+ let index = ((values.len() as f64 * p) as usize).min(values.len() - 1);+ let (_, value, _) = values.select_nth_unstable_by(index, f32::total_cmp);+ Ok(*value)+}diff --git a/rust/crates/ml/src/cv/structuring.rs b/rust/crates/ml/src/cv/structuring.rsnew file mode 100644index 0000000000..12d8cd91cb--- /dev/null+++ b/rust/crates/ml/src/cv/structuring.rs@@ -0,0 +1,37 @@+use crate::cv::OpResult;+use crate::cv::image::ImageU8;++pub(crate) fn ellipse_kernel(ksize: i32) -> OpResult<ImageU8> {+ if ksize <= 0 {+ return Err(format!("ellipse_kernel: invalid ksize {ksize}"));+ }+ let mut element = ImageU8::zeros(ksize, ksize, 1)?;+ let rect = ksize == 1;+ let r = ksize / 2;+ let inv_r2 = if r != 0 {+ 1.0 / (r as f64 * r as f64)+ } else {+ 0.0+ };+ let c = ksize / 2;++ for i in 0..ksize {+ let (mut j1, mut j2) = (0i32, 0i32);+ if rect {+ j2 = ksize;+ } else {+ let dy = i - r;+ if dy.abs() <= r {+ let dx = (c as f64 * (((r * r - dy * dy) as f64) * inv_r2).sqrt()).round_ties_even()+ as i32;+ j1 = (c - dx).max(0);+ j2 = (c + dx + 1).min(ksize);+ }+ }+ let row = (i * ksize) as usize;+ for j in j1.max(0)..j2 {+ element.data[row + j as usize] = 1;+ }+ }+ Ok(element)+}diff --git a/rust/crates/ml/src/cv/transform.rs b/rust/crates/ml/src/cv/transform.rsnew file mode 100644index 0000000000..9293afb36f--- /dev/null+++ b/rust/crates/ml/src/cv/transform.rs@@ -0,0 +1,58 @@+use crate::cv::OpResult;+use crate::cv::image::ImageU8;++pub(crate) fn rotate_u8(src: &ImageU8, degrees: i32) -> OpResult<ImageU8> {+ let cn = src.channels as usize;+ let (w, h) = (src.width as usize, src.height as usize);+ let degrees = degrees.rem_euclid(360);+ let (out_w, out_h) = match degrees {+ 0 | 180 => (src.width, src.height),+ 90 | 270 => (src.height, src.width),+ other => {+ return Err(format!(+ "Only 0, 90, 180, 270 degrees are supported, got {other}"+ ));+ }+ };++ let mut data = vec![0u8; src.data.len()];+ // A per-pixel `copy_from_slice` is a `memcpy` call per pixel; the common+ // channel counts are spelled out instead.+ let move_px = |dst: &mut [u8], d: usize, s: usize| match cn {+ 1 => dst[d] = src.data[s],+ 3 => {+ dst[d] = src.data[s];+ dst[d + 1] = src.data[s + 1];+ dst[d + 2] = src.data[s + 2];+ }+ _ => dst[d..d + cn].copy_from_slice(&src.data[s..s + cn]),+ };++ match degrees {+ 0 => data.copy_from_slice(&src.data),+ 180 => {+ for y in 0..h {+ for x in 0..w {+ let s = ((h - 1 - y) * w + (w - 1 - x)) * cn;+ let d = (y * w + x) * cn;+ move_px(&mut data, d, s);+ }+ }+ }+ _ => {+ let ow = out_w as usize;+ let oh = out_h as usize;+ for y in 0..oh {+ for x in 0..ow {+ let (sx, sy) = if degrees == 90 {+ (y, h - 1 - x)+ } else {+ (w - 1 - y, x)+ };+ move_px(&mut data, (y * ow + x) * cn, (sy * w + sx) * cn);+ }+ }+ }+ }+ ImageU8::new(out_w, out_h, src.channels, data)+}diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rsnew file mode 100644index 0000000000..122d8a3f7f--- /dev/null+++ b/rust/crates/ml/src/cv/warp.rs@@ -0,0 +1,40 @@+use image::{Rgb, RgbImage};+use imageproc::geometric_transformations::{Interpolation, Projection, warp_into};++use crate::cv::OpResult;+use crate::cv::image::ImageU8;++pub(crate) fn warp_perspective(+ src: &ImageU8,+ src_corners: [(f64, f64); 4],+ dst_corners: [(f64, f64); 4],+ width: i32,+ height: i32,+) -> OpResult<ImageU8> {+ if width <= 0 || height <= 0 {+ return Err(format!(+ "warp_perspective: invalid destination size {width}x{height}"+ ));+ }+ if src.channels != 3 {+ return Err(format!(+ "warp_perspective: expected a 3-channel image, got {}",+ src.channels+ ));+ }+ let narrow = |c: [(f64, f64); 4]| c.map(|(x, y)| (x as f32, y as f32));+ let projection = Projection::from_control_points(narrow(src_corners), narrow(dst_corners))+ .ok_or_else(|| "warp_perspective: the corner pairs are degenerate".to_string())?;++ let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())+ .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;+ let mut out = RgbImage::new(width as u32, height as u32);+ warp_into(+ &source,+ &projection,+ Interpolation::Bilinear,+ Rgb([0, 0, 0]),+ &mut out,+ );+ ImageU8::new(width, height, 3, out.into_raw())+}diff --git a/rust/crates/ml/src/lib.rs b/rust/crates/ml/src/lib.rsindex eaf9fa9654..317864d02d 100644--- a/rust/crates/ml/src/lib.rs+++ b/rust/crates/ml/src/lib.rs@@ -1,5 +1,6 @@pub mod assets;mod clip;+mod cv;mod diagnostics;pub mod error;pub mod face;@@ -12,6 +13,7 @@ mod pet;mod postprocess;mod preprocess;mod runtime;+pub mod scan;pub mod types;#[cfg(feature = "usearch")]pub mod vector_db;diff --git a/rust/crates/ml/src/scan/codec.rs b/rust/crates/ml/src/scan/codec.rsnew file mode 100644index 0000000000..ffa1086ed2--- /dev/null+++ b/rust/crates/ml/src/scan/codec.rs@@ -0,0 +1,41 @@+use ente_image::decode::decode_image_from_bytes;+use ente_image::image_compression::{EncodedImageFormat, encode_rgb};++use super::scanner::ScanError;+use crate::cv::image::ImageU8;++/// EXIF orientation comes pre-applied by the decoder.+pub(crate) fn decode_bgr(bytes: &[u8]) -> Result<ImageU8, ScanError> {+ let decoded = decode_image_from_bytes(bytes)+ .map_err(|err| ScanError::Codec(format!("failed to decode image: {err}")))?;+ let width = i32::try_from(decoded.dimensions.width)+ .map_err(|_| ScanError::Codec("image width does not fit in i32".to_string()))?;+ let height = i32::try_from(decoded.dimensions.height)+ .map_err(|_| ScanError::Codec("image height does not fit in i32".to_string()))?;++ let mut bgr = decoded.rgb;+ for px in bgr.chunks_exact_mut(3) {+ px.swap(0, 2);+ }+ ImageU8::new(width, height, 3, bgr).map_err(ScanError::Codec)+}++pub(crate) fn encode_jpeg(image: &ImageU8, quality: u8) -> Result<Vec<u8>, ScanError> {+ if image.channels != 3 {+ return Err(ScanError::Codec(format!(+ "cannot encode a {}-channel image",+ image.channels+ )));+ }+ let mut rgb = image.data.clone();+ for px in rgb.chunks_exact_mut(3) {+ px.swap(0, 2);+ }+ encode_rgb(+ &rgb,+ image.width as u32,+ image.height as u32,+ EncodedImageFormat::Jpeg { quality },+ )+ .map_err(|err| ScanError::Codec(err.to_string()))+}diff --git a/rust/crates/ml/src/scan/color.rs b/rust/crates/ml/src/scan/color.rsnew file mode 100644index 0000000000..182e2e3c04--- /dev/null+++ b/rust/crates/ml/src/scan/color.rs@@ -0,0 +1,146 @@+use super::OpResult;+use super::detection::resize_for_max_pixels;+use super::geometry::{Quad, norm};+use super::mask::Mask;+use crate::cv;+use crate::cv::image::{ImageF32, ImageRef, ImageU8};++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum ColorMode {+ Color,+ Grayscale,+}++const CHROMA_THRESHOLD: f64 = 17.5;+const PROPORTION_THRESHOLD: f64 = 0.0003;+const LUMINANCE_MIN: f64 = 40.0;+const LUMINANCE_MAX: f64 = 180.0;++/// `quad` is in original-image coordinates.+pub(crate) fn auto_color_mode(img: &ImageU8, mask: &Mask, quad: &Quad) -> OpResult<ColorMode> {+ let resized_img = resize_for_max_pixels(img, 1024.0 * 768.0)?;+ let work_size = resized_img.size();++ let doc_mask = document_mask(mask, quad, img.size(), work_size)?;+ let white_balanced = apply_gray_world_to_document(&resized_img, &doc_mask)?;++ let lab = cv::bgr_to_lab(&white_balanced)?;+ let channels = cv::split_u8(&lab)?;+ let luminance = &channels[0];+ let a = &channels[1];+ let b = &channels[2];++ let chroma = chroma(a, b)?;++ let color_mask_f = cv::threshold_binary_f32(&chroma, CHROMA_THRESHOLD, 255.0)?;+ let color_mask = cv::f32_to_u8(&color_mask_f)?;++ let luminance_mask = cv::in_range_u8(luminance, LUMINANCE_MIN, LUMINANCE_MAX)?;++ let tmp = cv::bitwise_and_u8(&color_mask, &luminance_mask)?;+ let restricted_mask = cv::bitwise_and_u8(&tmp, &doc_mask)?;++ let colored_pixels = cv::count_non_zero(&restricted_mask)?;+ let total_pixels = cv::count_non_zero(&doc_mask)?;++ if total_pixels == 0 {+ return Ok(ColorMode::Grayscale);+ }+ let proportion = colored_pixels as f64 / total_pixels as f64;+ Ok(if proportion > PROPORTION_THRESHOLD {+ ColorMode::Color+ } else {+ ColorMode::Grayscale+ })+}++fn chroma(a: &ImageU8, b: &ImageU8) -> OpResult<ImageF32> {+ let a_float = cv::u8_to_f32(a)?;+ let b_float = cv::u8_to_f32(b)?;+ let a_shifted = cv::subtract_f32_scalar(&a_float, 128.0)?;+ let b_shifted = cv::subtract_f32_scalar(&b_float, 128.0)?;+ cv::magnitude_f32(&a_shifted, &b_shifted)+}++fn erode_border(mask: &ImageU8, quad: &Quad) -> OpResult<ImageU8> {+ let min_dim = quad+ .edges()+ .iter()+ .map(|&(from, to)| norm(from, to))+ .fold(f64::INFINITY, f64::min);+ let mut k = (min_dim * 0.02 + 0.5).floor() as i32;+ k = k.clamp(3, 15);+ if k % 2 == 0 {+ k += 1;+ }++ let kernel = cv::ellipse_kernel(k)?;+ cv::morphology_erode(mask, &kernel)+}++fn document_mask(+ mask: &Mask,+ quad: &Quad,+ orig_size: (i32, i32),+ work_size: (i32, i32),+) -> OpResult<ImageU8> {+ let mask_image = mask.to_image()?;+ let resized_mask =+ cv::resize_area(ImageRef::U8(&mask_image), work_size.0, work_size.1)?.into_u8()?;++ let resized_quad = quad.scaled_to(+ orig_size.0 as f64,+ orig_size.1 as f64,+ work_size.0 as f64,+ work_size.1 as f64,+ );++ let eroded_mask = erode_border(&resized_mask, &resized_quad)?;++ let pts: Vec<(i32, i32)> = resized_quad+ .corners()+ .iter()+ .map(|p| (p.x as i32, p.y as i32))+ .collect();+ let quad_mask = cv::fill_poly(eroded_mask.width, eroded_mask.height, &pts, 255.0)?;++ cv::bitwise_and_u8(&eroded_mask, &quad_mask)+}++pub(crate) fn apply_gray_world_to_document(img: &ImageU8, doc_mask: &ImageU8) -> OpResult<ImageU8> {+ assert_eq!(img.channels, 3, "gray-world expects an 8UC3 image");++ let non_zero = cv::count_non_zero(doc_mask)?;+ if non_zero == 0 {+ return Ok(img.clone());+ }++ let mean = cv::mean_u8c3_masked(img, doc_mask)?;+ let eps = 1e-6;+ let mean_b = if mean[0] < eps { eps } else { mean[0] };+ let mean_g = if mean[1] < eps { eps } else { mean[1] };+ let mean_r = if mean[2] < eps { eps } else { mean[2] };++ let mean_gray = (mean_b + mean_g + mean_r) / 3.0;+ let scales = [+ mean_gray / mean_b,+ mean_gray / mean_g,+ mean_gray / mean_r,+ 0.0,+ ];++ let img_f = cv::u8_to_f32(img)?;+ let scaled_f = cv::multiply_f32_scalar(&img_f, scales)?;+ let scaled8 = cv::f32_to_u8(&scaled_f)?;++ cv::copy_to_masked(&scaled8, img, doc_mask)+}++pub(crate) fn quad_to_image(quad: &Quad, mask: &Mask, image_size: (i32, i32)) -> Quad {+ quad.scaled_to(+ mask.width as f64,+ mask.height as f64,+ image_size.0 as f64,+ image_size.1 as f64,+ )+}diff --git a/rust/crates/ml/src/scan/contour_orientation.rs b/rust/crates/ml/src/scan/contour_orientation.rsnew file mode 100644index 0000000000..3a457d44b7--- /dev/null+++ b/rust/crates/ml/src/scan/contour_orientation.rs@@ -0,0 +1,329 @@+use super::geometry::{Point, average};++const SMOOTH_WINDOW: i32 = 5;+const MIN_SIDE_LENGTH_RATIO: f64 = 0.02;++#[derive(Clone, Copy, Debug)]+struct Line {+ p: Point,+ d: Point,+}++#[derive(Clone, Copy, Debug)]+struct ContourSegment {+ start: usize,+ end: usize,+ angle: f64,+ length: f64,+}++pub(crate) fn find_quad_from_contour_orientation(contour: &[Point]) -> Option<Vec<Point>> {+ let max_angle_var = 5.0f64.to_radians();+ let merge_angle = 7.0f64.to_radians();++ if contour.len() < 20 {+ return None;+ }++ let angles = compute_smoothed_angles(contour, SMOOTH_WINDOW);++ // Open polyline: n-1 terms.+ let perimeter: f64 = contour+ .windows(2)+ .map(|w| (w[1].x - w[0].x).hypot(w[1].y - w[0].y))+ .sum();++ let min_length = perimeter * MIN_SIDE_LENGTH_RATIO;+ let segments = extract_segments(contour, &angles, max_angle_var, min_length);+ let merged = merge_segments(&segments, merge_angle);+ let dominant = select_dominant_segments(&merged, 4, 25.0f64.to_radians());++ if dominant.len() != 4 {+ return None;+ }++ let lines: Vec<Line> = dominant+ .iter()+ .map(|s| {+ let points: Vec<Point> = if s.start < s.end {+ contour[s.start..s.end].to_vec()+ } else {+ let mut v = contour[s.start..].to_vec();+ v.extend_from_slice(&contour[..s.end]);+ v+ };+ fit_line(&points)+ })+ .collect();++ let mut corners = Vec::with_capacity(4);+ for i in 0..4 {+ corners.push(intersect_lines(lines[i], lines[(i + 1) % 4])?);+ }+ Some(corners)+}++fn normalize_angle(a: f64) -> f64 {+ let mut x = a;+ while x <= -std::f64::consts::PI {+ x += 2.0 * std::f64::consts::PI;+ }+ while x > std::f64::consts::PI {+ x -= 2.0 * std::f64::consts::PI;+ }+ x+}++fn angle_diff(a: f64, b: f64) -> f64 {+ normalize_angle(a - b).abs()+}++fn angle_mean(a: f64, b: f64) -> f64 {+ let x = a.cos() + b.cos();+ let y = a.sin() + b.sin();+ y.atan2(x)+}++/// Total least squares direction from the second-moment matrix.+fn fit_line(points: &[Point]) -> Line {+ let cx = average(points.iter().map(|p| p.x));+ let cy = average(points.iter().map(|p| p.y));++ let mut xx = 0.0;+ let mut xy = 0.0;+ let mut yy = 0.0;+ for p in points {+ let dx = p.x - cx;+ let dy = p.y - cy;+ xx += dx * dx;+ xy += dx * dy;+ yy += dy * dy;+ }++ let theta = 0.5 * (2.0 * xy).atan2(xx - yy);+ Line {+ p: Point::new(cx, cy),+ d: Point::new(theta.cos(), theta.sin()),+ }+}++fn intersect_lines(l1: Line, l2: Line) -> Option<Point> {+ let x1 = l1.p.x;+ let y1 = l1.p.y;+ let x2 = x1 + l1.d.x;+ let y2 = y1 + l1.d.y;++ let x3 = l2.p.x;+ let y3 = l2.p.y;+ let x4 = x3 + l2.d.x;+ let y4 = y3 + l2.d.y;++ let denom = (x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4);+ if denom.abs() < 1e-6 {+ return None;+ }++ let px = ((x1 * y2 - y1 * x2) * (x3 - x4) - (x1 - x2) * (x3 * y4 - y3 * x4)) / denom;+ let py = ((x1 * y2 - y1 * x2) * (y3 - y4) - (y1 - y2) * (x3 * y4 - y3 * x4)) / denom;+ Some(Point::new(px, py))+}++fn extract_segments(+ contour: &[Point],+ angles: &[f64],+ max_angle_var: f64,+ min_length: f64,+) -> Vec<ContourSegment> {+ let n = contour.len();+ let mut result = Vec::new();++ let start_index = find_best_start_index(angles);++ let segment_length = |s: usize, e: usize| -> f64 {+ let mut len = 0.0;+ let mut i = s;+ while i != e {+ let j = (i + 1) % n;+ len += (contour[j].x - contour[i].x).hypot(contour[j].y - contour[i].y);+ i = j;+ }+ len+ };++ let mut start = start_index;+ let mut ref_angle = angles[start_index];++ let mut steps = 1usize;+ while steps <= n {+ let idx = (start_index + steps) % n;+ if steps < n && angle_diff(angles[idx], ref_angle) < max_angle_var {+ ref_angle = angle_mean(ref_angle, angles[idx]);+ } else {+ let len = segment_length(start, idx);+ if len >= min_length {+ result.push(ContourSegment {+ start,+ end: idx,+ angle: ref_angle,+ length: len,+ });+ }+ start = idx;+ ref_angle = angles[idx];+ }+ steps += 1;+ }++ result+}++fn find_best_start_index(angles: &[f64]) -> usize {+ let n = angles.len();+ let mut best_index = 0usize;+ let mut best_delta = 0.0;+ for i in 0..n {+ let j = (i + 1) % n;+ let d = angle_diff(angles[i], angles[j]);+ if d > best_delta {+ best_delta = d;+ best_index = j;+ }+ }+ best_index+}++fn compute_smoothed_angles(contour: &[Point], window: i32) -> Vec<f64> {+ let n = contour.len();+ let ni = n as i32;++ let mut angles = vec![0.0f64; n];+ for i in 0..n {+ let p0 = contour[(i + n - 1) % n];+ let p1 = contour[(i + 1) % n];+ angles[i] = (p1.y - p0.y).atan2(p1.x - p0.x);+ }++ let cos_a: Vec<f64> = angles.iter().map(|a| a.cos()).collect();+ let sin_a: Vec<f64> = angles.iter().map(|a| a.sin()).collect();++ let mut smooth = vec![0.0f64; n];+ let mut sx = 0.0;+ let mut sy = 0.0;+ for k in -window..=window {+ // n >= 20 and window == 5, so `k + n` is always positive.+ let idx = ((k + ni) % ni) as usize;+ sx += cos_a[idx];+ sy += sin_a[idx];+ }+ smooth[0] = sy.atan2(sx);++ for (i, out) in smooth.iter_mut().enumerate().skip(1) {+ let ii = i as i32;+ let out_idx = ((ii - window - 1 + ni) % ni) as usize;+ let in_idx = ((ii + window) % ni) as usize;+ sx -= cos_a[out_idx];+ sy -= sin_a[out_idx];+ sx += cos_a[in_idx];+ sy += sin_a[in_idx];+ *out = sy.atan2(sx);+ }+ smooth+}++fn merge_segments(segments: &[ContourSegment], angle_threshold: f64) -> Vec<ContourSegment> {+ if segments.is_empty() {+ return Vec::new();+ }+ if segments.len() <= 4 {+ return segments.to_vec();+ }++ let mut merged: Vec<ContourSegment> = Vec::new();+ let mut cur = segments[0];+ for p in &segments[1..] {+ if angle_diff(p.angle, cur.angle) < angle_threshold {+ cur = ContourSegment {+ start: cur.start,+ end: p.end,+ angle: angle_mean(cur.angle, p.angle),+ length: cur.length + p.length,+ };+ } else {+ merged.push(cur);+ cur = *p;+ }+ }+ merged.push(cur);+ merged+}++/// Both sorts are stable, which decides ties.+fn select_dominant_segments(+ segments: &[ContourSegment],+ max_count: usize,+ min_angle_separation: f64,+) -> Vec<ContourSegment> {+ let mut sorted = segments.to_vec();+ sorted.sort_by(|a, b| b.length.total_cmp(&a.length));++ let mut selected: Vec<ContourSegment> = Vec::new();+ for p in &sorted {+ let too_close = selected+ .iter()+ .any(|s| angle_diff(p.angle, s.angle) < min_angle_separation);+ if !too_close {+ selected.push(*p);+ if selected.len() == max_count {+ break;+ }+ }+ }++ selected.sort_by_key(|s| s.start);+ selected+}++#[cfg(test)]+mod tests {+ use super::find_quad_from_contour_orientation;+ use crate::scan::geometry::{Point, norm};++ #[test]+ fn contour_orientation_recovers_a_rotated_rectangle() {+ let corners = [+ Point::new(40.0, 30.0),+ Point::new(220.0, 50.0),+ Point::new(200.0, 190.0),+ Point::new(20.0, 170.0),+ ];+ let mut contour = Vec::new();+ for i in 0..4 {+ let a = corners[i];+ let b = corners[(i + 1) % 4];+ let steps = 60;+ for s in 0..steps {+ let t = s as f64 / steps as f64;+ contour.push(Point::new(a.x + (b.x - a.x) * t, a.y + (b.y - a.y) * t));+ }+ }+ let quad =+ find_quad_from_contour_orientation(&contour).expect("quad from contour orientation");+ assert_eq!(quad.len(), 4);+ for corner in corners {+ let best = quad+ .iter()+ .map(|p| norm(*p, corner))+ .fold(f64::INFINITY, f64::min);+ assert!(+ best < 1.0,+ "corner {corner:?} not recovered (min dist {best})"+ );+ }+ }++ #[test]+ fn contour_orientation_rejects_short_contours() {+ let contour: Vec<Point> = (0..19).map(|i| Point::new(i as f64, 0.0)).collect();+ assert!(find_quad_from_contour_orientation(&contour).is_none());+ }+}diff --git a/rust/crates/ml/src/scan/detection.rs b/rust/crates/ml/src/scan/detection.rsnew file mode 100644index 0000000000..2b546dd7b7--- /dev/null+++ b/rust/crates/ml/src/scan/detection.rs@@ -0,0 +1,244 @@+use super::OpResult;+use super::color::ColorMode;+use super::contour_orientation::find_quad_from_contour_orientation;+use super::geometry::{ImageSize, Point, Quad, create_quad};+use super::mask::Mask;+use super::perspective::estimate_real_dimensions;+use super::postprocess::enhance_captured_image;+use super::quad_score::score_quad_against_probmap;+use crate::cv;+use crate::cv::image::{Contour, ImageRef, ImageU8};++const THRESHOLDS: [f64; 7] = [0.5, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];+const LIVE_THRESHOLDS: [f64; 1] = [0.9];++pub(crate) fn size_trunc(width: f64, height: f64) -> (i32, i32) {+ (width as i32, height as i32)+}++/// The returned quad is in mask coordinates.+pub(crate) fn detect_document_quad(+ mask: &Mask,+ original_size: ImageSize,+ is_live_analysis: bool,+) -> OpResult<Option<Quad>> {+ let mask_image = mask.to_image()?;+ let thresholds: &[f64] = if is_live_analysis {+ &LIVE_THRESHOLDS+ } else {+ &THRESHOLDS+ };++ let mut vertices =+ find_quad_from_orientation_with_adaptive_threshold(&mask_image, original_size, thresholds)?;++ if vertices.is_none() && !is_live_analysis {+ // Fallback: min-area rectangle of the biggest contour.+ if let Some(biggest) = biggest_contour(&mask_image)? {+ vertices = min_area_rect(&biggest.points, mask.width, mask.height);+ }+ }++ Ok(match vertices {+ Some(v) if v.len() == 4 => Some(create_quad(&v)),+ _ => None,+ })+}++fn find_quad_from_orientation_with_adaptive_threshold(+ mask_image: &ImageU8,+ original_size: ImageSize,+ thresholds: &[f64],+) -> OpResult<Option<Vec<Point>>> {+ let probmap_smooth = cv::gaussian_blur_u8(mask_image, 3)?;++ let kernel = cv::ellipse_kernel(5)?;+ let prob_float = cv::u8_to_f32(mask_image)?;+ let mut best_quad: Option<Vec<Point>> = None;+ let mut best_score = 0.0f64;++ for &thr in thresholds {+ let bin = cv::threshold_binary_u8(&probmap_smooth, thr * 255.0, 255.0)?;+ let closed = cv::morphology_close(&bin, &kernel)?;++ if let Some(quad) = find_quad_from_orientation(&closed, original_size)?+ && is_valid_quad(&quad)+ {+ let score = score_quad_against_probmap(&quad, &prob_float, 0.02)?;+ if score > best_score {+ best_score = score;+ best_quad = Some(quad);+ }+ }+ }++ Ok(best_quad)+}++/// Deliberately no upper bound: corners past the mask edge are legitimate+/// for edge-touching documents.+pub(crate) fn is_valid_quad(quad: &[Point]) -> bool {+ quad.iter().all(|p| p.x >= 0.0 && p.y >= 0.0)+}++fn find_quad_from_orientation(+ mask_image: &ImageU8,+ original_size: ImageSize,+) -> OpResult<Option<Vec<Point>>> {+ let Some(contour) = biggest_contour(mask_image)? else {+ return Ok(None);+ };++ let scale_x = original_size.width / mask_image.width as f64;+ let scale_y = original_size.height / mask_image.height as f64;++ let scaled: Vec<Point> = contour+ .points+ .iter()+ .map(|&(x, y)| Point::new(x as f64 * scale_x, y as f64 * scale_y))+ .collect();++ Ok(find_quad_from_contour_orientation(&scaled).map(|corners| {+ corners+ .iter()+ .map(|p| Point::new(p.x / scale_x, p.y / scale_y))+ .collect()+ }))+}++pub(crate) fn min_area_rect(points: &[(i32, i32)], width: i32, height: i32) -> Option<Vec<Point>> {+ if points.len() < 3 {+ return None;+ }+ let pts: Vec<imageproc::point::Point<i32>> = points+ .iter()+ .map(|&(x, y)| imageproc::point::Point::new(x, y))+ .collect();+ let rect = imageproc::geometry::min_area_rect(&pts);+ if imageproc::geometry::contour_area(&rect) == 0.0 {+ return None;+ }+ Some(+ rect.iter()+ .map(|p| {+ Point::new(+ (p.x as f64).clamp(0.0, width as f64 - 1.0),+ (p.y as f64).clamp(0.0, height as f64 - 1.0),+ )+ })+ .collect(),+ )+}++pub(crate) fn biggest_contour(image: &ImageU8) -> OpResult<Option<Contour>> {+ let refined = refine_mask(image)?;+ let blurred = cv::gaussian_blur_u8(&refined, 5)?;+ let edges = cv::canny(&blurred, 75.0, 200.0)?;+ let contours = cv::find_contours(&edges)?;++ let mut biggest: Option<Contour> = None;+ let mut max_area = 0.0f64;+ for contour in contours {+ let area = contour.area.abs();+ if area > max_area {+ max_area = area;+ biggest = Some(contour);+ }+ }+ Ok(biggest)+}++fn refine_mask(original: &ImageU8) -> OpResult<ImageU8> {+ let binary_mask = cv::threshold_binary_u8(original, 128.0, 255.0)?;+ let kernel = cv::ellipse_kernel(5)?;+ let closed = cv::morphology_close(&binary_mask, &kernel)?;+ cv::morphology_open(&closed, &kernel)+}++pub(crate) fn resize_for_max_pixels(img: &ImageU8, max_pixels: f64) -> OpResult<ImageU8> {+ let orig_pixels = img.width as i64 * img.height as i64;+ if orig_pixels as f64 <= max_pixels {+ return Ok(img.clone());+ }+ let scale = (max_pixels / orig_pixels as f64).sqrt();+ let (width, height) = size_trunc(img.width as f64 * scale, img.height as f64 * scale);+ cv::resize_area(ImageRef::U8(img), width, height)?.into_u8()+}++pub(crate) fn extract_document(+ input: &ImageU8,+ quad: &Quad,+ rotation_degrees: i32,+ color_mode: ColorMode,+ max_pixels: f64,+) -> OpResult<ImageU8> {+ let estimated = estimate_real_dimensions(quad, input.width, input.height);+ let (target_width, target_height) = estimated.to_pixel_dimensions(quad);++ let corners = quad.corners();+ let src_corners = [+ (corners[0].x, corners[0].y),+ (corners[1].x, corners[1].y),+ (corners[2].x, corners[2].y),+ (corners[3].x, corners[3].y),+ ];+ let dst_corners = [+ (0.0, 0.0),+ (target_width, 0.0),+ (target_width, target_height),+ (0.0, target_height),+ ];++ let (out_width, out_height) = size_trunc(target_width, target_height);+ let warped = cv::warp_perspective(input, src_corners, dst_corners, out_width, out_height)?;++ let resized = resize_for_max_pixels(&warped, max_pixels)?;+ let enhanced = enhance_captured_image(&resized, color_mode)?;+ cv::rotate_u8(&enhanced, rotation_degrees)+}++#[cfg(test)]+mod tests {+ use super::min_area_rect;++ fn assert_close(a: f64, b: f64, eps: f64) {+ assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");+ }++ #[test]+ fn min_area_rect_of_axis_aligned_box() {+ let polygon = vec![(10, 20), (60, 20), (60, 90), (10, 90), (35, 55)];+ let rect = min_area_rect(&polygon, 256, 256).expect("rect");+ assert_eq!(rect.len(), 4);+ let xs: Vec<f64> = rect.iter().map(|p| p.x).collect();+ let ys: Vec<f64> = rect.iter().map(|p| p.y).collect();+ assert_close(xs.iter().cloned().fold(f64::INFINITY, f64::min), 10.0, 1e-9);+ assert_close(+ xs.iter().cloned().fold(f64::NEG_INFINITY, f64::max),+ 60.0,+ 1e-9,+ );+ assert_close(ys.iter().cloned().fold(f64::INFINITY, f64::min), 20.0, 1e-9);+ assert_close(+ ys.iter().cloned().fold(f64::NEG_INFINITY, f64::max),+ 90.0,+ 1e-9,+ );+ }++ #[test]+ fn min_area_rect_clips_to_image_bounds() {+ let polygon = vec![(-30, -30), (500, -30), (500, 500), (-30, 500)];+ let rect = min_area_rect(&polygon, 256, 256).expect("rect");+ for p in &rect {+ assert!((0.0..=255.0).contains(&p.x), "x out of bounds: {p:?}");+ assert!((0.0..=255.0).contains(&p.y), "y out of bounds: {p:?}");+ }+ }++ #[test]+ fn min_area_rect_rejects_degenerate_input() {+ assert!(min_area_rect(&[(0, 0), (1, 1)], 256, 256).is_none());+ assert!(min_area_rect(&[(0, 0), (5, 5), (10, 10)], 256, 256).is_none());+ }+}diff --git a/rust/crates/ml/src/scan/geometry.rs b/rust/crates/ml/src/scan/geometry.rsnew file mode 100644index 0000000000..50f40b4630--- /dev/null+++ b/rust/crates/ml/src/scan/geometry.rs@@ -0,0 +1,194 @@+pub(crate) fn average(values: impl Iterator<Item = f64>) -> f64 {+ let mut sum = 0.0;+ let mut count = 0usize;+ for v in values {+ sum += v;+ count += 1;+ }+ if count == 0 {+ f64::NAN+ } else {+ sum / count as f64+ }+}++#[derive(Clone, Copy, Debug, PartialEq)]+pub struct Point {+ pub x: f64,+ pub y: f64,+}++impl Point {+ pub fn new(x: f64, y: f64) -> Self {+ Self { x, y }+ }++ pub(crate) fn scaled(&self, scale_x: f64, scale_y: f64) -> Point {+ Point::new(self.x * scale_x, self.y * scale_y)+ }+}++pub(crate) fn norm(p1: Point, p2: Point) -> f64 {+ (p2.x - p1.x).hypot(p2.y - p1.y)+}++#[derive(Clone, Copy, Debug)]+pub(crate) struct ImageSize {+ pub width: f64,+ pub height: f64,+}++impl ImageSize {+ pub(crate) fn new(width: f64, height: f64) -> Self {+ Self { width, height }+ }+}++#[derive(Clone, Copy, Debug, PartialEq)]+pub struct Quad {+ pub top_left: Point,+ pub top_right: Point,+ pub bottom_right: Point,+ pub bottom_left: Point,+}++impl Quad {+ pub fn corners(&self) -> [Point; 4] {+ [+ self.top_left,+ self.top_right,+ self.bottom_right,+ self.bottom_left,+ ]+ }++ pub(crate) fn edges(&self) -> [(Point, Point); 4] {+ [+ (self.top_left, self.top_right),+ (self.top_right, self.bottom_right),+ (self.bottom_right, self.bottom_left),+ (self.bottom_left, self.top_left),+ ]+ }++ /// A negative `iterations` falls into the identity branch (truncating+ /// `%`).+ pub(crate) fn rotate90(&self, iterations: i32, image_size: ImageSize) -> Quad {+ let rotate = |p: Point| -> Point {+ match iterations % 4 {+ 1 => Point::new(image_size.height - p.y, p.x),+ 2 => Point::new(image_size.width - p.x, image_size.height - p.y),+ 3 => Point::new(p.y, image_size.width - p.x),+ _ => p,+ }+ };+ create_quad(&[+ rotate(self.top_left),+ rotate(self.top_right),+ rotate(self.bottom_right),+ rotate(self.bottom_left),+ ])+ }++ pub(crate) fn scaled_to(+ &self,+ from_width: f64,+ from_height: f64,+ to_width: f64,+ to_height: f64,+ ) -> Quad {+ let scale_x = to_width / from_width;+ let scale_y = to_height / from_height;+ Quad {+ top_left: self.top_left.scaled(scale_x, scale_y),+ top_right: self.top_right.scaled(scale_x, scale_y),+ bottom_right: self.bottom_right.scaled(scale_x, scale_y),+ bottom_left: self.bottom_left.scaled(scale_x, scale_y),+ }+ }+}++pub(crate) fn create_quad(vertices: &[Point]) -> Quad {+ assert_eq!(vertices.len(), 4, "create_quad requires exactly 4 vertices");+ let cx = average(vertices.iter().map(|p| p.x));+ let cy = average(vertices.iter().map(|p| p.y));++ let mut sorted = vertices.to_vec();+ sorted.sort_by(|a, b| {+ let aa = (a.y - cy).atan2(a.x - cx);+ let bb = (b.y - cy).atan2(b.x - cx);+ aa.total_cmp(&bb)+ });++ Quad {+ top_left: sorted[0],+ top_right: sorted[1],+ bottom_right: sorted[2],+ bottom_left: sorted[3],+ }+}++#[cfg(test)]+mod tests {+ use super::{ImageSize, Point, Quad, create_quad, norm};++ fn assert_close(a: f64, b: f64, eps: f64) {+ assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");+ }++ #[test]+ fn create_quad_orders_corners_by_angle_from_centroid() {+ // With y down, atan2 ascending starts at the top-left.+ let vertices = vec![+ Point::new(100.0, 100.0),+ Point::new(0.0, 100.0),+ Point::new(0.0, 0.0),+ Point::new(100.0, 0.0),+ ];+ let quad = create_quad(&vertices);+ assert_eq!(quad.top_left, Point::new(0.0, 0.0));+ assert_eq!(quad.top_right, Point::new(100.0, 0.0));+ assert_eq!(quad.bottom_right, Point::new(100.0, 100.0));+ assert_eq!(quad.bottom_left, Point::new(0.0, 100.0));+ }++ #[test]+ fn create_quad_is_idempotent() {+ let quad = create_quad(&[+ Point::new(10.0, 12.0),+ Point::new(90.0, 8.0),+ Point::new(95.0, 70.0),+ Point::new(5.0, 74.0),+ ]);+ let again = create_quad(&quad.corners());+ assert_eq!(quad.top_left, again.top_left);+ assert_eq!(quad.top_right, again.top_right);+ assert_eq!(quad.bottom_right, again.bottom_right);+ assert_eq!(quad.bottom_left, again.bottom_left);+ }++ #[test]+ fn scaled_to_maps_mask_space_to_image_space() {+ let quad = Quad {+ top_left: Point::new(0.0, 0.0),+ top_right: Point::new(128.0, 0.0),+ bottom_right: Point::new(128.0, 256.0),+ bottom_left: Point::new(0.0, 256.0),+ };+ let scaled = quad.scaled_to(256.0, 256.0, 1024.0, 512.0);+ assert_close(scaled.top_right.x, 512.0, 1e-9);+ assert_close(scaled.bottom_right.y, 512.0, 1e-9);+ }++ #[test]+ fn norm_matches_hypot() {+ assert_close(norm(Point::new(0.0, 0.0), Point::new(3.0, 4.0)), 5.0, 1e-12);+ }++ #[test]+ fn image_size_keeps_doubles() {+ let size = ImageSize::new(1024.0, 768.0);+ assert_close(size.width, 1024.0, 0.0);+ assert_close(size.height, 768.0, 0.0);+ }+}diff --git a/rust/crates/ml/src/scan/mask.rs b/rust/crates/ml/src/scan/mask.rsnew file mode 100644index 0000000000..286219878e--- /dev/null+++ b/rust/crates/ml/src/scan/mask.rs@@ -0,0 +1,26 @@+use super::OpResult;+use crate::cv::image::ImageU8;++pub(crate) struct Mask {+ pub width: i32,+ pub height: i32,+ binary: Vec<u8>,+}++impl Mask {+ pub(crate) fn from_probmap(probmap: &[u8], width: i32, height: i32) -> Self {+ let binary = probmap+ .iter()+ .map(|&v| if v >= 128 { 255u8 } else { 0u8 })+ .collect();+ Self {+ width,+ height,+ binary,+ }+ }++ pub(crate) fn to_image(&self) -> OpResult<ImageU8> {+ ImageU8::new(self.width, self.height, 1, self.binary.clone())+ }+}diff --git a/rust/crates/ml/src/scan/mod.rs b/rust/crates/ml/src/scan/mod.rsnew file mode 100644index 0000000000..87e4497c73--- /dev/null+++ b/rust/crates/ml/src/scan/mod.rs@@ -0,0 +1,72 @@+mod codec;+mod color;+mod contour_orientation;+mod detection;+mod geometry;+mod mask;+mod perspective;+mod postprocess;+mod quad_score;+mod scanner;+mod segmentation;+mod yuv;++pub use color::ColorMode;+pub use geometry::{Point, Quad};+pub use scanner::{ReprocessOptions, ScanError, ScanOptions, ScanResult, ScannerSession};+pub use segmentation::MASK_SIDE;+pub use yuv::PlaneLayout;++use std::path::PathBuf;++use ente_assets::download::CancellationToken;+use ente_assets::{Asset, AssetFile, AssetStore};++/// Downloads are verified against this before the path reaches+/// [`ScannerSession::new`].+pub const SEGMENTATION_MODEL_SHA256: &str =+ "36b8eeadd42592af496bf2e125a6aad9bebcbca1bda2ac19fa22e108574217a3";++// Temporary dev hosting; will move to models.ente.com.+const SEGMENTATION_MODEL_URL: &str = "https://entedevassets.priem.dev/document_segmentation.onnx";+const SEGMENTATION_MODEL_FILE: &str = "document_segmentation.onnx";++fn segmentation_model_asset() -> Asset {+ Asset::file(+ vec!["models".to_string(), "document_segmentation".to_string()],+ AssetFile {+ name: SEGMENTATION_MODEL_FILE.to_string(),+ url: SEGMENTATION_MODEL_URL.to_string(),+ sha256: SEGMENTATION_MODEL_SHA256.to_string(),+ },+ )+ .expect("valid segmentation model asset")+}++pub async fn ensure_segmentation_model(+ store: &AssetStore,+) -> Result<PathBuf, ente_assets::download::Error> {+ let asset = segmentation_model_asset();+ if store.is_downloaded(&asset) {+ log::info!("segmentation model: using cached copy");+ } else {+ log::info!("segmentation model: downloading from {SEGMENTATION_MODEL_URL}");+ let start = std::time::Instant::now();+ store+ .download(+ std::slice::from_ref(&asset),+ |_| {},+ CancellationToken::default(),+ )+ .await?;+ log::info!(+ "segmentation model: downloaded in {}ms",+ start.elapsed().as_millis()+ );+ }+ Ok(store+ .file_path(&asset, SEGMENTATION_MODEL_FILE)+ .expect("segmentation model file"))+}++pub(crate) use crate::cv::OpResult;diff --git a/rust/crates/ml/src/scan/perspective.rs b/rust/crates/ml/src/scan/perspective.rsnew file mode 100644index 0000000000..0033a1bee9--- /dev/null+++ b/rust/crates/ml/src/scan/perspective.rs@@ -0,0 +1,181 @@+//! Aspect-ratio estimation from the quad's vanishing points.++use super::geometry::{Point, Quad, norm};++#[derive(Clone, Copy, Debug)]+struct Vector3D {+ x: f64,+ y: f64,+ z: f64,+}++impl Vector3D {+ fn new(x: f64, y: f64, z: f64) -> Self {+ Self { x, y, z }+ }++ fn sub(self, other: Vector3D) -> Vector3D {+ Vector3D::new(self.x - other.x, self.y - other.y, self.z - other.z)+ }++ fn scale(self, t: f64) -> Vector3D {+ Vector3D::new(self.x * t, self.y * t, self.z * t)+ }++ fn dot(self, other: Vector3D) -> f64 {+ self.x * other.x + self.y * other.y + self.z * other.z+ }++ fn cross(self, other: Vector3D) -> Vector3D {+ Vector3D::new(+ self.y * other.z - self.z * other.y,+ self.z * other.x - self.x * other.z,+ self.x * other.y - self.y * other.x,+ )+ }++ fn norm(self) -> f64 {+ (self.x * self.x + self.y * self.y + self.z * self.z).sqrt()+ }+}++#[derive(Clone, Copy, Debug, PartialEq)]+pub(crate) struct EstimatedDimensions {+ pub(crate) width: f64,+ pub(crate) height: f64,+}++impl EstimatedDimensions {+ pub(crate) fn to_pixel_dimensions(self, quad: &Quad) -> (f64, f64) {+ let w =+ (norm(quad.top_left, quad.top_right) + norm(quad.bottom_left, quad.bottom_right)) / 2.0;+ let h =+ (norm(quad.top_left, quad.bottom_left) + norm(quad.top_right, quad.bottom_right)) / 2.0;+ let projected_area = w * h;++ let ratio = self.height / self.width;+ let target_width = (projected_area / ratio).sqrt();+ let target_height = target_width * ratio;+ (target_width, target_height)+ }+}++pub(crate) fn estimate_real_dimensions(+ quad: &Quad,+ image_width: i32,+ image_height: i32,+) -> EstimatedDimensions {+ let average_sides = || EstimatedDimensions {+ width: (norm(quad.top_left, quad.top_right) + norm(quad.bottom_left, quad.bottom_right))+ / 2.0,+ height: (norm(quad.top_left, quad.bottom_left) + norm(quad.top_right, quad.bottom_right))+ / 2.0,+ };++ let to_h = |p: Point| Vector3D::new(p.x, p.y, 1.0);+ let line_through = |p1: Point, p2: Point| to_h(p1).cross(to_h(p2));++ let v1h = line_through(quad.top_left, quad.top_right)+ .cross(line_through(quad.bottom_left, quad.bottom_right));+ let v2h = line_through(quad.top_left, quad.bottom_left)+ .cross(line_through(quad.top_right, quad.bottom_right));++ if v1h.z.abs() < 1e-6 || v2h.z.abs() < 1e-6 {+ return average_sides();+ }++ let cx = image_width as f64 / 2.0;+ let cy = image_height as f64 / 2.0;++ let v1 = Point::new(v1h.x / v1h.z - cx, v1h.y / v1h.z - cy);+ let v2 = Point::new(v2h.x / v2h.z - cx, v2h.y / v2h.z - cy);++ let f2 = -(v1.x * v2.x + v1.y * v2.y);+ if f2 <= 0.0 {+ return average_sides();+ }+ let f = f2.sqrt();++ if f > (image_width.max(image_height)) as f64 * 1.2 {+ return average_sides();+ }++ let d1 = Vector3D::new(v1.x, v1.y, f);+ let d2 = Vector3D::new(v2.x, v2.y, f);+ let n = d1.cross(d2);++ let ray = |p: Point| Vector3D::new((p.x - cx) / f, (p.y - cy) / f, 1.0);++ let corner3d = |p: Point| {+ let r = ray(p);+ r.scale(1.0 / n.dot(r))+ };++ let x_tl = corner3d(quad.top_left);+ let x_tr = corner3d(quad.top_right);+ let x_br = corner3d(quad.bottom_right);+ let x_bl = corner3d(quad.bottom_left);++ let real_w = (x_tr.sub(x_tl).norm() + x_br.sub(x_bl).norm()) / 2.0;+ let real_h = (x_bl.sub(x_tl).norm() + x_br.sub(x_tr).norm()) / 2.0;++ EstimatedDimensions {+ width: real_w,+ height: real_h,+ }+}++#[cfg(test)]+mod tests {+ use super::{EstimatedDimensions, estimate_real_dimensions};+ use crate::scan::geometry::{Point, Quad};++ fn assert_close(a: f64, b: f64, eps: f64) {+ assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");+ }++ fn rect_quad(w: f64, h: f64) -> Quad {+ Quad {+ top_left: Point::new(0.0, 0.0),+ top_right: Point::new(w, 0.0),+ bottom_right: Point::new(w, h),+ bottom_left: Point::new(0.0, h),+ }+ }++ #[test]+ fn estimate_real_dimensions_falls_back_for_parallel_sides() {+ let quad = rect_quad(400.0, 300.0);+ let dims = estimate_real_dimensions(&quad, 1000, 800);+ assert_eq!(+ dims,+ EstimatedDimensions {+ width: 400.0,+ height: 300.0+ }+ );+ }++ #[test]+ fn estimate_real_dimensions_falls_back_for_weak_perspective() {+ // f exceeds max(w,h)*1.2, so the fallback applies.+ let quad = Quad {+ top_left: Point::new(100.0, 100.0),+ top_right: Point::new(900.0, 100.0),+ bottom_right: Point::new(899.0, 700.0),+ bottom_left: Point::new(101.0, 700.0),+ };+ let dims = estimate_real_dimensions(&quad, 1000, 800);+ assert_close(dims.width, 799.0, 1e-9);+ assert_close(dims.height, 600.0008333, 1e-6);+ }++ #[test]+ fn to_pixel_dimensions_preserves_area_and_ratio() {+ let quad = rect_quad(400.0, 300.0);+ let dims = estimate_real_dimensions(&quad, 1000, 800);+ let (w, h) = dims.to_pixel_dimensions(&quad);+ assert_close(w * h, 400.0 * 300.0, 1e-6);+ assert_close(h / w, 300.0 / 400.0, 1e-12);+ }+}diff --git a/rust/crates/ml/src/scan/postprocess.rs b/rust/crates/ml/src/scan/postprocess.rsnew file mode 100644index 0000000000..a3453cfe23--- /dev/null+++ b/rust/crates/ml/src/scan/postprocess.rs@@ -0,0 +1,334 @@+use super::OpResult;+use super::color::ColorMode;+use super::detection::size_trunc;+use crate::cv;+use crate::cv::image::{ImageF32, ImageRef, ImageU8};++pub(crate) fn enhance_captured_image(img: &ImageU8, color_mode: ColorMode) -> OpResult<ImageU8> {+ match color_mode {+ ColorMode::Color => multi_scale_retinex_on_l(img),+ ColorMode::Grayscale => enhance_grayscale_image(img),+ }+}++const SHADOW_MAP_LONG_EDGE: f64 = 256.0;+const SHADOW_KERNEL_FRAC: f64 = 1.0 / 16.0;+const SHADOW_MAX_GAIN: f32 = 2.8;+const SHADOW_PAPER_PERCENTILE: f64 = 0.90;++/// Flat-field illumination correction on a luminance channel: estimate the+/// background map (closing + blur at map scale), then divide toward the+/// paper level. Below `paper / SHADOW_MAX_GAIN` the gain ramps back to 1:+/// regions that much darker than paper are content (photos, figures), not+/// cast shadows, and must not be brightened.+fn remove_shadows(channel: &ImageU8) -> OpResult<ImageU8> {+ let (width, height) = channel.size();+ let long_edge = width.max(height) as f64;+ let (map_w, map_h) = if long_edge > SHADOW_MAP_LONG_EDGE {+ let scale = SHADOW_MAP_LONG_EDGE / long_edge;+ size_trunc(width as f64 * scale, height as f64 * scale)+ } else {+ (width, height)+ };+ let small = cv::resize_area(ImageRef::U8(channel), map_w, map_h)?.into_u8()?;++ let k = ((map_w.max(map_h) as f64 * SHADOW_KERNEL_FRAC) as i32).max(3) | 1;+ let kernel = cv::ellipse_kernel(k)?;+ let closed = cv::morphology_close(&small, &kernel)?;++ let closed_f = cv::u8_to_f32(&closed)?;+ let map_small = cv::box_filter_f32(&cv::box_filter_f32(&closed_f, k, k)?, k, k)?;++ let paper = cv::percentile_f32(&map_small, SHADOW_PAPER_PERCENTILE)?;+ if paper < 1.0 {+ return Ok(channel.clone());+ }+ let map = cv::resize_bilinear(ImageRef::F32(&map_small), width, height)?.into_f32()?;++ let knee = paper / SHADOW_MAX_GAIN;+ let mut out = ImageU8::zeros(width, height, 1)?;+ cv::pointwise3(+ &mut out.data,+ 1,+ &channel.data,+ 1,+ &map.data,+ 1,+ |out, src, map| {+ for ((o, &v), &m) in out.iter_mut().zip(src).zip(map) {+ let m = m.max(1.0);+ let gain = if m >= knee {+ paper / m+ } else {+ 1.0 + (SHADOW_MAX_GAIN - 1.0) * (m / knee)+ };+ *o = cv::saturate_u8_f32(v as f32 * gain);+ }+ },+ );+ Ok(out)+}++fn multi_scale_retinex_on_l(bgr: &ImageU8) -> OpResult<ImageU8> {+ let lab = cv::bgr_to_lab(bgr)?;+ let mut lab_channels = cv::split_u8(&lab)?;+ let l = remove_shadows(&lab_channels[0])?;++ let l_float_raw = cv::u8_to_f32(&l)?;+ let l_float = cv::add_f32_scalar(&l_float_raw, 1.0)?;++ let scale_factor = 2.0;+ // `max_dim_small` deliberately uses the untruncated sizes.+ let small_w = l_float.width as f64 / scale_factor;+ let small_h = l_float.height as f64 / scale_factor;+ let (small_cols, small_rows) = size_trunc(small_w, small_h);++ let l_small = cv::resize_area(ImageRef::F32(&l_float), small_cols, small_rows)?.into_f32()?;++ let log_l_small = cv::log_f32(&l_small)?;++ let max_dim_small = small_w.max(small_h);+ let kernel_sizes = [+ max_dim_small / 80.0,+ max_dim_small / 10.0,+ max_dim_small / 2.0,+ ];++ let weight = 1.0 / kernel_sizes.len() as f64;+ let mut retinex_small = ImageF32::zeros(l_small.width, l_small.height, 1)?;++ for ks in kernel_sizes {+ let k = (ks as i32).max(3) | 1;+ let blur_log = cv::box_filter_f32(&log_l_small, k, k)?;+ let diff = cv::subtract_f32(&log_l_small, &blur_log)?;+ retinex_small = cv::add_weighted_f32(&retinex_small, 1.0, &diff, weight, 0.0)?;+ }++ let (min_val, max_val) = cv::min_max_loc_f32(&retinex_small)?;+ let mut retinex_norm_small = cv::subtract_f32_scalar(&retinex_small, min_val)?;++ let range = max_val - min_val;+ if range > 1e-6 {+ retinex_norm_small =+ cv::multiply_f32_scalar(&retinex_norm_small, [1.0 / range, 0.0, 0.0, 0.0])?;+ }++ let retinex_norm = cv::resize_bicubic(+ ImageRef::F32(&retinex_norm_small),+ l_float.width,+ l_float.height,+ )?+ .into_f32()?;++ let l_original_float = cv::u8_to_f32(&l)?;++ let mean_l = cv::mean_f32(&l_original_float)?;+ let amplitude = 60.0;++ let corrected_l = cv::multiply_f32_scalar(&retinex_norm, [amplitude, 0.0, 0.0, 0.0])?;+ let corrected_l = cv::add_f32_scalar(&corrected_l, mean_l - amplitude / 2.0)?;++ let alpha = 0.6;+ let corrected_l =+ cv::add_weighted_f32(&l_original_float, 1.0 - alpha, &corrected_l, alpha, 0.0)?;++ let p_low_orig = percentile_l(&l_original_float, 0.001)?;+ let p_low = percentile_l(&corrected_l, 0.001)?;+ let p_high = percentile_l(&corrected_l, 0.995)?;++ let target_low = p_low.min(p_low_orig);+ let target_high = 245.0;+ let scale = (target_high - target_low) / (p_high - p_low + 1e-6);++ let shifted = cv::subtract_f32_scalar(&corrected_l, p_low)?;+ let stretched = cv::multiply_f32_scalar(&shifted, [scale, 0.0, 0.0, 0.0])?;+ let offset = cv::add_f32_scalar(&stretched, target_low)?;++ let clamped_high = cv::min_f32_scalar(&offset, 255.0)?;+ let clamped = cv::max_f32_scalar(&clamped_high, 0.0)?;++ lab_channels[0] = cv::f32_to_u8(&clamped)?;++ let merged = cv::merge_u8(&lab_channels)?;+ cv::lab_to_bgr(&merged)+}++/// Values outside [0,256) are not counted, but the denominator stays the+/// full pixel count.+pub(crate) fn percentile_l(l: &ImageF32, p: f64) -> OpResult<f64> {+ let hist = cv::hist_256_f32(l)?;+ let total = l.pixels() as f64;+ let mut sum = 0.0;+ for (i, count) in hist.iter().enumerate() {+ sum += count;+ if sum / total >= p {+ return Ok(i as f64);+ }+ }+ Ok(255.0)+}++fn enhance_grayscale_image(img: &ImageU8) -> OpResult<ImageU8> {+ let gray = match img.channels {+ 3 => cv::bgr_to_gray(img)?,+ 1 => img.clone(),+ other => {+ return Err(format!(+ "grayscale enhancement expects a 1- or 3-channel image, got {other}"+ ));+ }+ };+ let gray = remove_shadows(&gray)?;++ let max_dim = gray.width.max(gray.height) as f64;++ let img_float_raw = cv::u8_to_f32(&gray)?;+ let img_float = cv::add_f32_scalar(&img_float_raw, 1.0)?;+ let log_img = cv::log_f32(&img_float)?;++ // Unlike the color path, no odd-kernel forcing.+ let kernel_sizes = [max_dim / 6.0, max_dim / 50.0];+ let weight = 1.0 / kernel_sizes.len() as f64;+ let mut retinex = ImageF32::zeros(gray.width, gray.height, 1)?;++ for kernel_size in kernel_sizes {+ // Unlike the color path, blurs the linear image and logs afterwards.+ let (kw, kh) = size_trunc(kernel_size, kernel_size);+ let blur_raw = cv::box_filter_f32(&img_float, kw, kh)?;+ let blur = cv::add_f32_scalar(&blur_raw, 1.0)?;+ let log_blur = cv::log_f32(&blur)?;+ let diff = cv::subtract_f32(&log_img, &log_blur)?;+ retinex = cv::add_weighted_f32(&retinex, 1.0, &diff, weight, 0.0)?;+ }++ let retinex_exp = cv::exp_f32(&retinex)?;++ let p_low = cv::percentile_f32(&retinex_exp, 0.004)? as f64;+ let p_high = cv::percentile_f32(&retinex_exp, 0.99)? as f64;++ let normalized = cv::subtract_f32_scalar(&retinex_exp, p_low)?;+ let scale = if p_high > p_low {+ 255.0 / (p_high - p_low)+ } else {+ 1.0+ };+ let scaled = cv::multiply_f32_scalar(&normalized, [scale, 0.0, 0.0, 0.0])?;+ let clamped_high = cv::min_f32_scalar(&scaled, 255.0)?;+ let clamped = cv::max_f32_scalar(&clamped_high, 0.0)?;++ let result8u = cv::f32_to_u8(&clamped)?;++ // The mode over [180,255] estimates the background level.+ let hist = cv::hist_256_u8(&result8u)?;+ let mut mode_val = 220usize;+ let mut mode_count = 0.0f64;+ for (i, &c) in hist.iter().enumerate().take(256).skip(180) {+ if c > mode_count {+ mode_count = c;+ mode_val = i;+ }+ }++ let stretched8u = if mode_val >= 254 {+ // Retinex over-amplified; normalize the original grayscale instead.+ let gray_f = cv::u8_to_f32(&gray)?;+ let g_low = cv::percentile_f32(&gray_f, 0.01)? as f64;+ let g_high = cv::percentile_f32(&gray_f, 0.99)? as f64;++ let shifted = cv::subtract_f32_scalar(&gray_f, g_low)?;+ let scaled =+ cv::multiply_f32_scalar(&shifted, [255.0 / (g_high - g_low + 1e-6), 0.0, 0.0, 0.0])?;+ let min_clamped = cv::min_f32_scalar(&scaled, 255.0)?;+ let max_clamped = cv::max_f32_scalar(&min_clamped, 0.0)?;+ cv::f32_to_u8(&max_clamped)?+ } else {+ let stretched_f = cv::u8_to_f32(&result8u)?;+ let multiplied =+ cv::multiply_f32_scalar(&stretched_f, [255.0 / mode_val as f64, 0.0, 0.0, 0.0])?;+ let min_clamped = cv::min_f32_scalar(&multiplied, 255.0)?;+ cv::f32_to_u8(&min_clamped)?+ };++ let denoised = cv::bilateral_filter_u8(&stretched8u, 9, 20.0, 10.0)?;+ cv::gray_to_bgr(&denoised)+}++#[cfg(test)]+mod tests {+ use super::remove_shadows;+ use crate::cv::image::ImageU8;++ fn background_mean(img: &ImageU8, x0: i32, x1: i32) -> f64 {+ let mut sum = 0.0;+ let mut count = 0.0;+ for y in 20..img.height - 20 {+ for x in x0..x1 {+ if x % 17 != 3 || y % 11 != 5 {+ sum += img.data[(y * img.width + x) as usize] as f64;+ count += 1.0;+ }+ }+ }+ sum / count+ }++ #[test]+ fn remove_shadows_lifts_shadowed_paper_and_keeps_text_dark() {+ let (w, h) = (400, 300);+ let mut data = vec![230u8; (w * h) as usize];+ for y in 0..h {+ for x in 0..w {+ let i = (y * w + x) as usize;+ if x % 17 == 3 && y % 11 == 5 {+ data[i] = 40;+ }+ if x < w / 2 {+ data[i] = (data[i] as f64 * 0.55) as u8;+ }+ }+ }+ let img = ImageU8::new(w, h, 1, data).unwrap();+ let out = remove_shadows(&img).unwrap();++ let shadow_bg = background_mean(&out, 20, w / 4);+ let lit_bg = background_mean(&out, 3 * w / 4, w - 20);+ assert!(+ (shadow_bg - lit_bg).abs() < 12.0,+ "shadowed paper {shadow_bg:.1} should match lit paper {lit_bg:.1}"+ );++ let mut min_shadow_half = 255u8;+ for y in 20..h - 20 {+ for x in 20..w / 4 {+ min_shadow_half = min_shadow_half.min(out.data[(y * w + x) as usize]);+ }+ }+ assert!(+ min_shadow_half < 90,+ "text in the lifted shadow should stay dark, got {min_shadow_half}"+ );+ }++ #[test]+ fn remove_shadows_is_near_identity_on_evenly_lit_paper() {+ let (w, h) = (400, 300);+ let mut data = vec![225u8; (w * h) as usize];+ for y in 0..h {+ for x in 0..w {+ if x % 17 == 3 && y % 11 == 5 {+ data[(y * w + x) as usize] = 40;+ }+ }+ }+ let img = ImageU8::new(w, h, 1, data.clone()).unwrap();+ let out = remove_shadows(&img).unwrap();+ let mean_delta: f64 = out+ .data+ .iter()+ .zip(&data)+ .map(|(&a, &b)| (a as f64 - b as f64).abs())+ .sum::<f64>()+ / data.len() as f64;+ assert!(mean_delta < 4.0, "expected near-identity, got {mean_delta}");+ }+}diff --git a/rust/crates/ml/src/scan/quad_score.rs b/rust/crates/ml/src/scan/quad_score.rsnew file mode 100644index 0000000000..dd2c62141d--- /dev/null+++ b/rust/crates/ml/src/scan/quad_score.rs@@ -0,0 +1,32 @@+use super::OpResult;+use super::geometry::Point;+use crate::cv;+use crate::cv::image::{ImageF32, ImageU8};++pub(crate) fn make_polygon_mask(width: i32, height: i32, polygon: &[Point]) -> OpResult<ImageU8> {+ let pts: Vec<(i32, i32)> = polygon.iter().map(|p| (p.x as i32, p.y as i32)).collect();+ cv::fill_poly(width, height, &pts, 1.0)+}++/// `probmap` values are 0..255 (the binarized mask as f32), not normalized+/// probabilities.+pub(crate) fn score_quad_against_probmap(+ quad: &[Point],+ probmap: &ImageF32,+ min_quad_area_ratio: f64,+) -> OpResult<f64> {+ let mask = make_polygon_mask(probmap.width, probmap.height, quad)?;+ let mask_float = cv::u8_to_f32(&mask)?;+ let masked = cv::multiply_f32(probmap, &mask_float)?;++ let sum_masked = cv::sum_f32(&masked)?;+ let sum_mask = cv::sum_f32(&mask_float)?;+ let mean_prob = sum_masked / sum_mask;+ let area_ratio = sum_mask / (probmap.height * probmap.width) as f64;++ Ok(if area_ratio < min_quad_area_ratio {+ 0.0+ } else {+ mean_prob * (0.7 + 0.3 * area_ratio)+ })+}diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rsnew file mode 100644index 0000000000..28750f1360--- /dev/null+++ b/rust/crates/ml/src/scan/scanner.rs@@ -0,0 +1,274 @@+use thiserror::Error;++use super::codec;+use super::color::{ColorMode, auto_color_mode, quad_to_image};+use super::detection::{detect_document_quad, extract_document, resize_for_max_pixels};+use super::geometry::{ImageSize, Quad};+use super::mask::Mask;+use super::segmentation::{MASK_SIDE, Segmenter};+use super::yuv::{PlaneLayout, rgba_to_bgr, yuv420_to_bgr};+use crate::cv;+use crate::cv::image::ImageU8;++const DEFAULT_MAX_PIXELS: u32 = 2_000_000;+const DEFAULT_JPEG_QUALITY: u8 = 75;++#[derive(Debug, Clone, PartialEq, Eq, Error)]+pub enum ScanError {+ #[error("invalid input: {0}")]+ InvalidInput(String),+ #[error("model load: {0}")]+ ModelLoad(String),+ #[error("codec: {0}")]+ Codec(String),+ #[error("pipeline: {0}")]+ Pipeline(String),+}++#[derive(Debug, Clone, Default, PartialEq)]+pub struct ScanOptions {+ pub color_mode_override: Option<ColorMode>,+ pub max_pixels: Option<u32>,+ /// Must be a multiple of 90.+ pub rotation_degrees: i32,+}++#[derive(Debug, Clone, PartialEq)]+pub struct ReprocessOptions {+ /// Same coordinate space as [`ScanResult::quad`]: the decoded source+ /// image.+ pub quad: Quad,+ /// Must be a multiple of 90.+ pub rotation_degrees: i32,+ pub color_mode: ColorMode,+ pub max_pixels: Option<u32>,+ pub jpeg_quality: Option<u8>,+}++#[derive(Debug, Clone, PartialEq)]+pub struct ScanResult {+ /// In decoded-source coordinates (EXIF applied, before+ /// `rotation_degrees`); `None` when nothing was detected and the whole+ /// frame was returned.+ pub quad: Option<Quad>,+ pub color_mode: ColorMode,+ pub output_width: u32,+ pub output_height: u32,+ /// The size [`ScanResult::quad`] is relative to.+ pub source_width: u32,+ pub source_height: u32,+ /// JPEG-encoded.+ pub processed_image: Vec<u8>,+}++pub struct ScannerSession {+ segmenter: Segmenter,+}++impl ScannerSession {+ /// The caller is expected to have verified the model file against+ /// [`SEGMENTATION_MODEL_SHA256`](super::SEGMENTATION_MODEL_SHA256).+ pub fn new(model_path: &str) -> Result<Self, ScanError> {+ let start = std::time::Instant::now();+ let result = Self::load(model_path);+ match &result {+ Ok(_) => log::info!(+ "session ready: {} loaded in {}ms",+ basename(model_path),+ start.elapsed().as_millis()+ ),+ Err(error) => log::error!("session init failed: {error}"),+ }+ result+ }++ fn load(model_path: &str) -> Result<Self, ScanError> {+ if !std::path::Path::new(model_path).is_file() {+ return Err(ScanError::ModelLoad(format!(+ "no model file at {model_path}"+ )));+ }+ Ok(Self {+ segmenter: Segmenter::new(model_path)?,+ })+ }++ fn segment(&self, bgr: &ImageU8) -> Result<Mask, ScanError> {+ let probmap = self+ .segmenter+ .probability_map_u8(bgr)+ .map_err(ScanError::Pipeline)?;+ Ok(Mask::from_probmap(&probmap, MASK_SIDE, MASK_SIDE))+ }++ /// The returned quad is in mask space (256x256), already rotated by+ /// `rotation_degrees`: the caller rotates the overlay quad, not the+ /// frame.+ pub fn live_detect_rgba(+ &self,+ rgba: &[u8],+ width: u32,+ height: u32,+ rotation_degrees: i32,+ ) -> Result<Option<Quad>, ScanError> {+ let bgr =+ rgba_to_bgr(rgba, to_i32(width)?, to_i32(height)?).map_err(ScanError::InvalidInput)?;+ self.live_detect(&bgr, rotation_degrees)+ }++ /// Quad semantics as in [`Self::live_detect_rgba`].+ pub fn live_detect_yuv420(+ &self,+ y: &[u8],+ u: &[u8],+ v: &[u8],+ layout: PlaneLayout,+ rotation_degrees: i32,+ ) -> Result<Option<Quad>, ScanError> {+ let bgr = yuv420_to_bgr(y, u, v, layout).map_err(ScanError::InvalidInput)?;+ self.live_detect(&bgr, rotation_degrees)+ }++ fn live_detect(&self, bgr: &ImageU8, rotation_degrees: i32) -> Result<Option<Quad>, ScanError> {+ let mask = self.segment(bgr)?;+ let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);+ let quad_in_mask =+ detect_document_quad(&mask, original_size, true).map_err(ScanError::Pipeline)?;+ let mask_size = ImageSize::new(mask.width as f64, mask.height as f64);+ Ok(quad_in_mask.map(|q| q.rotate90(rotation_degrees / 90, mask_size)))+ }++ pub fn process_capture(+ &self,+ image_bytes: &[u8],+ options: &ScanOptions,+ ) -> Result<ScanResult, ScanError> {+ let start = std::time::Instant::now();+ let result = self.process_capture_inner(image_bytes, options);+ match &result {+ Ok(scan) => log::info!(+ "capture: {}x{} -> {}x{} {:?}, quad {}, {}ms",+ scan.source_width,+ scan.source_height,+ scan.output_width,+ scan.output_height,+ scan.color_mode,+ if scan.quad.is_some() { "found" } else { "none" },+ start.elapsed().as_millis()+ ),+ Err(error) => log::error!("capture failed: {error}"),+ }+ result+ }++ fn process_capture_inner(+ &self,+ image_bytes: &[u8],+ options: &ScanOptions,+ ) -> Result<ScanResult, ScanError> {+ let bgr = codec::decode_bgr(image_bytes)?;+ let max_pixels = options.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS) as f64;++ let mask = self.segment(&bgr)?;+ let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);+ let quad_in_mask =+ detect_document_quad(&mask, original_size, false).map_err(ScanError::Pipeline)?;++ let Some(quad_in_mask) = quad_in_mask else {+ let resized = resize_for_max_pixels(&bgr, max_pixels).map_err(ScanError::Pipeline)?;+ let page =+ cv::rotate_u8(&resized, options.rotation_degrees).map_err(ScanError::Pipeline)?;+ return finish(None, ColorMode::Color, &bgr, &page, DEFAULT_JPEG_QUALITY);+ };++ let quad = quad_to_image(&quad_in_mask, &mask, bgr.size());+ let auto = auto_color_mode(&bgr, &mask, &quad).map_err(ScanError::Pipeline)?;+ let color_mode = options.color_mode_override.unwrap_or(auto);+ let page = extract_document(+ &bgr,+ &quad,+ options.rotation_degrees,+ color_mode,+ max_pixels,+ )+ .map_err(ScanError::Pipeline)?;++ finish(Some(quad), color_mode, &bgr, &page, DEFAULT_JPEG_QUALITY)+ }++ /// No inference runs.+ pub fn reprocess(+ &self,+ source_bytes: &[u8],+ options: &ReprocessOptions,+ ) -> Result<ScanResult, ScanError> {+ let start = std::time::Instant::now();+ let result = self.reprocess_inner(source_bytes, options);+ match &result {+ Ok(scan) => log::info!(+ "reprocess: -> {}x{} in {}ms",+ scan.output_width,+ scan.output_height,+ start.elapsed().as_millis()+ ),+ Err(error) => log::error!("reprocess failed: {error}"),+ }+ result+ }++ fn reprocess_inner(+ &self,+ source_bytes: &[u8],+ options: &ReprocessOptions,+ ) -> Result<ScanResult, ScanError> {+ let bgr = codec::decode_bgr(source_bytes)?;+ let page = extract_document(+ &bgr,+ &options.quad,+ options.rotation_degrees,+ options.color_mode,+ options.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS) as f64,+ )+ .map_err(ScanError::Pipeline)?;++ finish(+ Some(options.quad),+ options.color_mode,+ &bgr,+ &page,+ options.jpeg_quality.unwrap_or(DEFAULT_JPEG_QUALITY),+ )+ }+}++fn basename(path: &str) -> &str {+ std::path::Path::new(path)+ .file_name()+ .and_then(|name| name.to_str())+ .unwrap_or(path)+}++fn to_i32(value: u32) -> Result<i32, ScanError> {+ i32::try_from(value)+ .map_err(|_| ScanError::InvalidInput(format!("{value} does not fit in i32")))+}++fn finish(+ quad: Option<Quad>,+ color_mode: ColorMode,+ source: &ImageU8,+ page: &ImageU8,+ jpeg_quality: u8,+) -> Result<ScanResult, ScanError> {+ let processed_image = codec::encode_jpeg(page, jpeg_quality)?;++ Ok(ScanResult {+ quad,+ color_mode,+ output_width: page.width as u32,+ output_height: page.height as u32,+ source_width: source.width as u32,+ source_height: source.height as u32,+ processed_image,+ })+}diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rsnew file mode 100644index 0000000000..afdffa63e4--- /dev/null+++ b/rust/crates/ml/src/scan/segmentation.rs@@ -0,0 +1,85 @@+use std::sync::Mutex;++use ort::ep::CPU;+use ort::session::Session;+use ort::session::builder::GraphOptimizationLevel;+use ort::value::TensorRef;++use super::OpResult;+use super::scanner::ScanError;+use crate::cv;+use crate::cv::image::{ImageRef, ImageU8};++pub const MASK_SIDE: i32 = 256;++pub(crate) struct Segmenter {+ // `Session::run` needs `&mut self`.+ session: Mutex<Session>,+}++impl Segmenter {+ /// CPU-only on purpose: the model is tiny and the thresholded downstream+ /// pipeline needs deterministic output across devices.+ pub(crate) fn new(model_path: &str) -> Result<Self, ScanError> {+ let build = || -> Result<Session, ort::Error> {+ Session::builder()?+ .with_optimization_level(GraphOptimizationLevel::All)?+ .with_intra_threads(1)?+ .with_inter_threads(1)?+ .with_execution_providers([CPU::default().with_arena_allocator(true).build()])?+ .commit_from_file(model_path)+ };+ let session = build().map_err(|err| ScanError::ModelLoad(err.to_string()))?;+ if session.inputs().len() != 1 {+ return Err(ScanError::ModelLoad(format!(+ "expected exactly 1 model input, got {}",+ session.inputs().len()+ )));+ }+ Ok(Self {+ session: Mutex::new(session),+ })+ }++ pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {+ let rgb = cv::bgr_to_rgb(bgr)?;+ let resized = cv::resize_bilinear(ImageRef::U8(&rgb), MASK_SIDE, MASK_SIDE)?.into_u8()?;++ let input: Vec<f32> = resized+ .data+ .iter()+ .map(|&v| (v as f32 - 127.5) / 127.5)+ .collect();++ let side = MASK_SIDE as i64;+ let mut session = match self.session.lock() {+ Ok(guard) => guard,+ Err(poisoned) => poisoned.into_inner(),+ };+ let mut run = || -> Result<Vec<f32>, String> {+ let tensor = TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))+ .map_err(|err| err.to_string())?;+ let outputs = session+ .run(ort::inputs![tensor])+ .map_err(|err| err.to_string())?;+ if outputs.len() == 0 {+ return Err("model produced no outputs".to_string());+ }+ let (shape, data) = outputs[0]+ .try_extract_tensor::<f32>()+ .map_err(|err| err.to_string())?;+ let expected = (MASK_SIDE * MASK_SIDE) as usize;+ if data.len() != expected {+ return Err(format!("unexpected model output shape {shape:?}"));+ }+ Ok(data.to_vec())+ };+ let data = run()?;++ // The model outputs probabilities directly; no sigmoid.+ Ok(data+ .iter()+ .map(|&v| (v.clamp(0.0, 1.0) * 255.0).round_ties_even() as u8)+ .collect())+ }+}diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rsnew file mode 100644index 0000000000..d8351454d3--- /dev/null+++ b/rust/crates/ml/src/scan/yuv.rs@@ -0,0 +1,462 @@+//! YUV_420_888 camera planes -> packed BGR.+//!+//! Reproduces libyuv's limited-range BT.601 fixed-point arithmetic β not the+//! textbook full-range matrix β because that is what Android camera stacks+//! apply when converting frames to RGBA, so YUV and RGBA frames yield the+//! same quad.++use crate::cv::image::ImageU8;++// Q6 fixed-point BT.601 limited-range coefficients.+const YG: i32 = 18997; // round(1.164 * 64 * 256 * 256 / 257)+const YGB: i32 = -1160; // 1.164 * 64 * -16 + 64 * 0.5 (rounding bias folded in)+const UB: i32 = -128; // max(-128, round(-2.018 * 64)): clamped to fit a signed byte+const UG: i32 = 25; // round(0.391 * 64)+const VG: i32 = 52; // round(0.813 * 64)+const VR: i32 = -102; // round(-1.596 * 64)++// Bias terms; fold in the -128 U/V offset.+const BB: i32 = UB * 128 + YGB;+const BG: i32 = UG * 128 + VG * 128 + YGB;+const BR: i32 = VR * 128 + YGB;++fn ceil_half(value: i32) -> i32 {+ (value + 1) / 2+}++fn clamp_u8(value: i32) -> u8 {+ value.clamp(0, 255) as u8+}++#[inline]+pub(crate) fn yuv_to_bgr(y: u8, u: u8, v: u8) -> (u8, u8, u8) {+ let (y, u, v) = (y as i32, u as i32, v as i32);+ // 0x0101 replicates 8 bits to 16, keeping the luma multiply in 32 bits.+ let y1 = ((y as u32 * 0x0101 * YG as u32) >> 16) as i32;+ let b = clamp_u8((-(u * UB) + y1 + BB) >> 6);+ let g = clamp_u8((-(u * UG + v * VG) + y1 + BG) >> 6);+ let r = clamp_u8((-(v * VR) + y1 + BR) >> 6);+ (b, g, r)+}++#[derive(Clone, Copy, Debug)]+pub struct PlaneLayout {+ pub width: i32,+ pub height: i32,+ pub y_row_stride: i32,+ pub uv_row_stride: i32,+ pub uv_pixel_stride: i32,+}++fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> usize {+ if rows <= 0 {+ return 0;+ }+ (rows - 1) as usize * row_stride as usize + last_row_bytes as usize+}++pub(crate) fn yuv420_to_bgr(+ y_plane: &[u8],+ u_plane: &[u8],+ v_plane: &[u8],+ layout: PlaneLayout,+) -> Result<ImageU8, String> {+ let PlaneLayout {+ width,+ height,+ y_row_stride,+ uv_row_stride,+ uv_pixel_stride,+ } = layout;++ if width <= 0 || height <= 0 {+ return Err(format!("invalid frame size {width}x{height}"));+ }+ if uv_pixel_stride < 1 {+ return Err(format!("invalid uv pixel stride {uv_pixel_stride}"));+ }+ if y_row_stride < width {+ return Err(format!(+ "y row stride {y_row_stride} is smaller than the frame width {width}"+ ));+ }+ let cw = ceil_half(width);+ let ch = ceil_half(height);+ if uv_row_stride < (cw - 1) * uv_pixel_stride + 1 {+ return Err(format!(+ "uv row stride {uv_row_stride} is too small for {cw} chroma samples \+ at pixel stride {uv_pixel_stride}"+ ));+ }++ let need_y = required_len(height, y_row_stride, width);+ let need_uv = required_len(ch, uv_row_stride, (cw - 1) * uv_pixel_stride + 1);+ if y_plane.len() < need_y {+ return Err(format!(+ "y plane has {} bytes, needs {need_y}",+ y_plane.len()+ ));+ }+ if u_plane.len() < need_uv {+ return Err(format!(+ "u plane has {} bytes, needs {need_uv}",+ u_plane.len()+ ));+ }+ if v_plane.len() < need_uv {+ return Err(format!(+ "v plane has {} bytes, needs {need_uv}",+ v_plane.len()+ ));+ }++ let mut out = vec![0u8; width as usize * height as usize * 3];+ for row in 0..height {+ let y_base = row as usize * y_row_stride as usize;+ let uv_base = (row / 2) as usize * uv_row_stride as usize;+ let out_base = row as usize * width as usize * 3;+ for col in 0..width {+ let uv_index = uv_base + (col / 2) as usize * uv_pixel_stride as usize;+ let (b, g, r) = yuv_to_bgr(+ y_plane[y_base + col as usize],+ u_plane[uv_index],+ v_plane[uv_index],+ );+ let o = out_base + col as usize * 3;+ out[o] = b;+ out[o + 1] = g;+ out[o + 2] = r;+ }+ }++ ImageU8::new(width, height, 3, out)+}++pub(crate) fn rgba_to_bgr(rgba: &[u8], width: i32, height: i32) -> Result<ImageU8, String> {+ if width <= 0 || height <= 0 {+ return Err(format!("invalid frame size {width}x{height}"));+ }+ let expected = width as usize * height as usize * 4;+ if rgba.len() != expected {+ return Err(format!(+ "RGBA buffer has {} bytes, expected {expected} for {width}x{height}",+ rgba.len()+ ));+ }+ let mut out = vec![0u8; width as usize * height as usize * 3];+ for (dst, src) in out.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) {+ dst[0] = src[2];+ dst[1] = src[1];+ dst[2] = src[0];+ }+ ImageU8::new(width, height, 3, out)+}++#[cfg(test)]+mod tests {+ use super::*;++ /// `ub` is a parameter: libyuv's effective blue coefficient (-2.0, from+ /// the `UB` clamp) differs from the documented -2.018.+ fn reference_yuv_to_bgr(y: u8, u: u8, v: u8, ub: f64) -> (u8, u8, u8) {+ let yf = (y as f64 - 16.0) * 1.164;+ let uf = u as f64 - 128.0;+ let vf = v as f64 - 128.0;+ let r = yf + 1.596 * vf;+ let g = yf - 0.391 * uf - 0.813 * vf;+ let b = yf - ub * uf;+ let to_u8 = |v: f64| v.round().clamp(0.0, 255.0) as u8;+ (to_u8(b), to_u8(g), to_u8(r))+ }++ fn worst_deviation(triples: impl Iterator<Item = (u8, u8, u8)>, ub: f64) -> [i32; 3] {+ let mut worst = [0i32; 3];+ for (y, u, v) in triples {+ let got = yuv_to_bgr(y, u, v);+ let want = reference_yuv_to_bgr(y, u, v, ub);+ let pairs = [(got.0, want.0), (got.1, want.1), (got.2, want.2)];+ for (i, (a, b)) in pairs.iter().enumerate() {+ worst[i] = worst[i].max((*a as i32 - *b as i32).abs());+ }+ }+ worst+ }++ fn all_triples() -> impl Iterator<Item = (u8, u8, u8)> {+ (0..=255u8)+ .flat_map(|y| (0..=255u8).map(move |u| (y, u)))+ .flat_map(|(y, u)| (0..=255u8).map(move |v| (y, u, v)))+ }++ fn strided_triples() -> impl Iterator<Item = (u8, u8, u8)> {+ let axis = |step: usize| (0..=255u8).step_by(step).chain(std::iter::once(255));+ axis(6).flat_map(move |y| axis(7).flat_map(move |u| axis(11).map(move |v| (y, u, v))))+ }++ struct Rng(u64);++ impl Rng {+ fn next_u8(&mut self) -> u8 {+ let mut x = self.0;+ x ^= x >> 12;+ x ^= x << 25;+ x ^= x >> 27;+ self.0 = x;+ (x.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 33) as u8+ }+ }++ #[test]+ fn anchor_values_match_the_reference_matrix() {+ assert_eq!(yuv_to_bgr(16, 128, 128), (0, 0, 0));+ assert_eq!(yuv_to_bgr(235, 128, 128), (255, 255, 255));+ assert_eq!(yuv_to_bgr(0, 128, 128), (0, 0, 0));+ assert_eq!(yuv_to_bgr(255, 128, 128), (255, 255, 255));+ // Y=126 is the mid point of the limited range 16..235.+ let (b, g, r) = yuv_to_bgr(126, 128, 128);+ assert_eq!((b, g, r), (128, 128, 128));+ }++ #[test]+ fn fixed_point_matches_the_effective_matrix_on_a_subsample() {+ let worst = worst_deviation(strided_triples(), -2.0);+ assert!(+ worst[0] == 0 && worst[1] <= 1 && worst[2] <= 1,+ "B/G/R deviation from the effective matrix changed: {worst:?}"+ );+ }++ #[test]+ #[ignore = "exhaustive 2^24 sweep; run explicitly"]+ fn fixed_point_matches_the_effective_matrix_exhaustively() {+ let worst = worst_deviation(all_triples(), -2.0);+ assert_eq!(+ worst,+ [0, 1, 1],+ "B/G/R deviation from the effective matrix changed"+ );+ }++ #[test]+ #[ignore = "exhaustive 2^24 sweep; run explicitly"]+ fn documented_matrix_differs_only_in_blue_by_the_ub_clamp() {+ let worst = worst_deviation(all_triples(), -2.018);+ assert_eq!(+ worst,+ [3, 1, 1],+ "B/G/R deviation from the documented matrix changed"+ );+ }++ #[test]+ fn grey_ramp_is_monotonic_and_neutral() {+ let mut previous = 0u8;+ for y in 16..=235u8 {+ let (b, g, r) = yuv_to_bgr(y, 128, 128);+ assert_eq!(b, g, "neutral chroma must stay grey at Y={y}");+ assert_eq!(g, r, "neutral chroma must stay grey at Y={y}");+ assert!(b >= previous, "luma ramp must be monotonic at Y={y}");+ previous = b;+ }+ }++ fn planes_from_pixels(+ width: i32,+ height: i32,+ pixel: impl Fn(i32, i32) -> (u8, u8, u8),+ ) -> (Vec<u8>, Vec<u8>, Vec<u8>) {+ let cw = ceil_half(width);+ let ch = ceil_half(height);+ let mut y = vec![0u8; (width * height) as usize];+ let mut u = vec![0u8; (cw * ch) as usize];+ let mut v = vec![0u8; (cw * ch) as usize];+ for row in 0..height {+ for col in 0..width {+ y[(row * width + col) as usize] = pixel(col, row).0;+ }+ }+ for row in 0..ch {+ for col in 0..cw {+ let (_, cu, cv) = pixel(col * 2, row * 2);+ u[(row * cw + col) as usize] = cu;+ v[(row * cw + col) as usize] = cv;+ }+ }+ (y, u, v)+ }++ #[test]+ fn planar_and_interleaved_layouts_agree() {+ let (width, height) = (37, 21);+ let mut rng = Rng(0xC0FF_EE00_1234_5677);+ let samples: Vec<(u8, u8, u8)> = (0..(width * height))+ .map(|_| (rng.next_u8(), rng.next_u8(), rng.next_u8()))+ .collect();+ let pixel = |x: i32, y: i32| samples[(y * width + x) as usize];++ let (y_plane, u_plane, v_plane) = planes_from_pixels(width, height, pixel);+ let planar = yuv420_to_bgr(+ &y_plane,+ &u_plane,+ &v_plane,+ PlaneLayout {+ width,+ height,+ y_row_stride: width,+ uv_row_stride: ceil_half(width),+ uv_pixel_stride: 1,+ },+ )+ .expect("planar conversion");++ // Android exposes NV21 as two plane views one byte apart into one+ // interleaved buffer, both with pixel stride 2.+ let cw = ceil_half(width);+ let ch = ceil_half(height);+ let mut vu = vec![0u8; (cw * ch * 2) as usize];+ for i in 0..(cw * ch) as usize {+ vu[i * 2] = v_plane[i];+ vu[i * 2 + 1] = u_plane[i];+ }+ let interleaved = yuv420_to_bgr(+ &y_plane,+ &vu[1..],+ &vu,+ PlaneLayout {+ width,+ height,+ y_row_stride: width,+ uv_row_stride: cw * 2,+ uv_pixel_stride: 2,+ },+ )+ .expect("interleaved conversion");+ assert_eq!(planar, interleaved);++ let mut uv = vec![0u8; (cw * ch * 2) as usize];+ for i in 0..(cw * ch) as usize {+ uv[i * 2] = u_plane[i];+ uv[i * 2 + 1] = v_plane[i];+ }+ let nv12 = yuv420_to_bgr(+ &y_plane,+ &uv,+ &uv[1..],+ PlaneLayout {+ width,+ height,+ y_row_stride: width,+ uv_row_stride: cw * 2,+ uv_pixel_stride: 2,+ },+ )+ .expect("nv12 conversion");+ assert_eq!(planar, nv12);+ }++ #[test]+ fn row_padding_is_honoured() {+ let (width, height) = (10, 6);+ let mut rng = Rng(0x1234_5678_9ABC_DEF1);+ let samples: Vec<(u8, u8, u8)> = (0..(width * height))+ .map(|_| (rng.next_u8(), rng.next_u8(), rng.next_u8()))+ .collect();+ let pixel = |x: i32, y: i32| samples[(y * width + x) as usize];+ let (y_tight, u_tight, v_tight) = planes_from_pixels(width, height, pixel);++ let tight = yuv420_to_bgr(+ &y_tight,+ &u_tight,+ &v_tight,+ PlaneLayout {+ width,+ height,+ y_row_stride: width,+ uv_row_stride: ceil_half(width),+ uv_pixel_stride: 1,+ },+ )+ .expect("tight conversion");++ let (y_stride, uv_stride) = (width + 7, ceil_half(width) + 3);+ let cw = ceil_half(width);+ let ch = ceil_half(height);+ let mut y_pad = vec![0xA5u8; (y_stride * height) as usize];+ let mut u_pad = vec![0x5Au8; (uv_stride * ch) as usize];+ let mut v_pad = vec![0x3Cu8; (uv_stride * ch) as usize];+ for row in 0..height {+ for col in 0..width {+ y_pad[(row * y_stride + col) as usize] = y_tight[(row * width + col) as usize];+ }+ }+ for row in 0..ch {+ for col in 0..cw {+ u_pad[(row * uv_stride + col) as usize] = u_tight[(row * cw + col) as usize];+ v_pad[(row * uv_stride + col) as usize] = v_tight[(row * cw + col) as usize];+ }+ }+ let padded = yuv420_to_bgr(+ &y_pad,+ &u_pad,+ &v_pad,+ PlaneLayout {+ width,+ height,+ y_row_stride: y_stride,+ uv_row_stride: uv_stride,+ uv_pixel_stride: 1,+ },+ )+ .expect("padded conversion");+ assert_eq!(tight, padded);+ }++ #[test]+ fn odd_sizes_and_short_planes_are_rejected_not_panicked() {+ let layout = PlaneLayout {+ width: 8,+ height: 8,+ y_row_stride: 8,+ uv_row_stride: 4,+ uv_pixel_stride: 1,+ };+ assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout).is_err());+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout).is_err());+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout).is_err());+ assert!(+ yuv420_to_bgr(+ &[0u8; 64],+ &[0u8; 16],+ &[0u8; 16],+ PlaneLayout {+ y_row_stride: 4,+ ..layout+ }+ )+ .is_err()+ );+ assert!(+ yuv420_to_bgr(+ &[128u8],+ &[128u8],+ &[128u8],+ PlaneLayout {+ width: 1,+ height: 1,+ y_row_stride: 1,+ uv_row_stride: 1,+ uv_pixel_stride: 1,+ }+ )+ .is_ok()+ );+ }++ #[test]+ fn rgba_to_bgr_drops_alpha_and_reverses_channels() {+ let rgba = vec![1u8, 2, 3, 255, 10, 20, 30, 0];+ let bgr = rgba_to_bgr(&rgba, 2, 1).expect("conversion");+ assert_eq!(bgr.data, vec![3, 2, 1, 30, 20, 10]);+ assert!(rgba_to_bgr(&rgba, 3, 1).is_err());+ }+}diff --git a/rust/crates/ml/tests/scan.rs b/rust/crates/ml/tests/scan.rsnew file mode 100644index 0000000000..2ec0d97109--- /dev/null+++ b/rust/crates/ml/tests/scan.rs@@ -0,0 +1,179 @@+//! Point `ENTE_DOC_SEGMENTATION_MODEL` at the model file; without it every+//! test skips.++use std::io::Cursor;++use ente_ml::scan::{ColorMode, PlaneLayout, Quad, ReprocessOptions, ScanOptions, ScannerSession};+use image::{ImageFormat, Rgb, RgbImage};++const MODEL_ENV: &str = "ENTE_DOC_SEGMENTATION_MODEL";++fn session() -> Option<ScannerSession> {+ let Ok(path) = std::env::var(MODEL_ENV) else {+ eprintln!("skipping: {MODEL_ENV} is not set");+ return None;+ };+ Some(ScannerSession::new(&path).expect("model should load"))+}++fn synthetic_document() -> RgbImage {+ let (width, height) = (640u32, 480u32);+ let mut img = RgbImage::from_pixel(width, height, Rgb([40, 45, 50]));+ let (x0, y0, x1, y1) = (120u32, 80u32, 520u32, 400u32);+ for y in y0..y1 {+ for x in x0..x1 {+ img.put_pixel(x, y, Rgb([235, 233, 228]));+ }+ }+ for line in 0..8u32 {+ let ly = y0 + 30 + line * 36;+ for y in ly..ly + 6 {+ for x in x0 + 30..x1 - 30 {+ img.put_pixel(x, y, Rgb([60, 60, 65]));+ }+ }+ }+ img+}++fn encode(img: &RgbImage, format: ImageFormat) -> Vec<u8> {+ let mut bytes = Cursor::new(Vec::new());+ img.write_to(&mut bytes, format).expect("encode");+ bytes.into_inner()+}++fn assert_document_result(result: &ente_ml::scan::ScanResult) {+ let quad = result.quad.expect("the synthetic document should be found");+ for p in quad.corners() {+ assert!(p.x >= 0.0 && p.x <= result.source_width as f64, "{quad:?}");+ assert!(p.y >= 0.0 && p.y <= result.source_height as f64, "{quad:?}");+ }+ assert_eq!(result.source_width, 640);+ assert_eq!(result.source_height, 480);+ assert!(result.output_width > 100 && result.output_height > 100);+ assert_jpeg(&result.processed_image);+}++fn assert_jpeg(bytes: &[u8]) {+ assert_eq!(+ image::guess_format(bytes).expect("recognizable output"),+ ImageFormat::Jpeg+ );+}++#[test]+fn processes_a_png_capture() {+ let Some(session) = session() else { return };+ let bytes = encode(&synthetic_document(), ImageFormat::Png);++ let result = session+ .process_capture(&bytes, &ScanOptions::default())+ .expect("process_capture");+ assert_document_result(&result);++ let decoded = image::load_from_memory(&result.processed_image).expect("valid JPEG output");+ assert_eq!(decoded.width(), result.output_width);+ assert_eq!(decoded.height(), result.output_height);+}++#[test]+fn processes_a_jpeg_capture_and_reprocesses_it() {+ let Some(session) = session() else { return };+ let bytes = encode(&synthetic_document(), ImageFormat::Jpeg);++ let result = session+ .process_capture(&bytes, &ScanOptions::default())+ .expect("process_capture");+ assert_document_result(&result);+ image::load_from_memory(&result.processed_image).expect("valid JPEG output");++ let reprocessed = session+ .reprocess(+ &bytes,+ &ReprocessOptions {+ quad: result.quad.expect("quad"),+ rotation_degrees: 90,+ color_mode: ColorMode::Grayscale,+ max_pixels: Some(500_000),+ jpeg_quality: Some(60),+ },+ )+ .expect("reprocess");+ assert_eq!(reprocessed.color_mode, ColorMode::Grayscale);+ assert_jpeg(&reprocessed.processed_image);+ assert!(reprocessed.output_width < reprocessed.output_height);+ image::load_from_memory(&reprocessed.processed_image).expect("valid JPEG output");+}++#[test]+fn live_detect_rgba_and_yuv_agree_on_a_grayscale_frame() {+ let Some(session) = session() else { return };+ let doc = synthetic_document();+ let (width, height) = (doc.width() as i32, doc.height() as i32);++ let mut rgba = Vec::with_capacity((width * height * 4) as usize);+ let mut y_plane = Vec::with_capacity((width * height) as usize);+ for px in doc.pixels() {+ // value = 1.164 * (y - 16), chroma neutral, so the limited-range YUV+ // round trip is exact.+ let y = 16 + (px.0[0] as i32 * 219 / 255) as u8;+ let value = (((y as i32 - 16) * 1164 + 500) / 1000).clamp(0, 255) as u8;+ rgba.extend_from_slice(&[value, value, value, 255]);+ y_plane.push(y);+ }+ let chroma_len = ((width + 1) / 2 * (height + 1) / 2) as usize;+ let u_plane = vec![128u8; chroma_len];+ let v_plane = vec![128u8; chroma_len];++ let from_rgba = session+ .live_detect_rgba(&rgba, width as u32, height as u32, 0)+ .expect("live_detect_rgba")+ .expect("quad from RGBA");+ let from_yuv = session+ .live_detect_yuv420(+ &y_plane,+ &u_plane,+ &v_plane,+ PlaneLayout {+ width,+ height,+ y_row_stride: width,+ uv_row_stride: (width + 1) / 2,+ uv_pixel_stride: 1,+ },+ 0,+ )+ .expect("live_detect_yuv420")+ .expect("quad from YUV");++ let max_delta = quad_max_delta(&from_rgba, &from_yuv);+ assert!(+ max_delta < 2.0,+ "RGBA and YUV quads diverge by {max_delta} in mask space:\n{from_rgba:?}\n{from_yuv:?}"+ );++ let rotated = session+ .live_detect_rgba(&rgba, width as u32, height as u32, 180)+ .expect("live_detect_rgba rotated")+ .expect("quad");+ for p in rotated.corners() {+ assert!((0.0..=256.0).contains(&p.x) && (0.0..=256.0).contains(&p.y));+ }+}++#[test]+fn rejects_undecodable_bytes_with_a_codec_error() {+ let Some(session) = session() else { return };+ let error = session+ .process_capture(b"not an image", &ScanOptions::default())+ .expect_err("garbage must not decode");+ assert!(matches!(error, ente_ml::scan::ScanError::Codec(_)));+}++fn quad_max_delta(a: &Quad, b: &Quad) -> f64 {+ a.corners()+ .iter()+ .zip(b.corners())+ .map(|(p, q)| (p.x - q.x).hypot(p.y - q.y))+ .fold(0.0, f64::max)+}