Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Add document scanning pipeline to the ml crate

laurenspriemb93f31c26aa0
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index d41211b5a4..1a9b38fed9 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2539,6 +2539,7 @@ dependencies = [
"once_cell",
"ort",
"plsfix",
+ "rayon",
"regex",
"serde",
"serde_json",
diff --git a/rust/crates/ml/Cargo.toml b/rust/crates/ml/Cargo.toml
index 80b9a56e00..0d52c60d11 100644
--- a/rust/crates/ml/Cargo.toml
+++ b/rust/crates/ml/Cargo.toml
@@ -19,6 +19,7 @@ log.workspace = true
nalgebra = "0.33.2"
once_cell = "1.20.2"
plsfix = "0.1.8"
+rayon = "1"
regex = "1.11.1"
thiserror.workspace = true
# The FRB photos bridge calls the SimSIMD symbol exported by usearch 2.21.x.
diff --git a/rust/crates/ml/src/cv/arith.rs b/rust/crates/ml/src/cv/arith.rs
new file mode 100644
index 0000000000..96cde77215
--- /dev/null
+++ b/rust/crates/ml/src/cv/arith.rs
@@ -0,0 +1,147 @@
+//! Scalar multiply and the weighted sum deliberately promote to f64; the
+//! rest stays in f32.
+
+use rayon::prelude::*;
+
+use crate::cv::OpResult;
+use crate::cv::image::ImageF32;
+
+const PARALLEL_MIN_ELEMS: usize = 200_000;
+
+fn fill<T: Send + Sync + Copy>(
+ out: &mut [f32],
+ src: &[T],
+ chunk: usize,
+ f: impl Fn(&mut [f32], &[T]) + Send + Sync,
+) {
+ if out.len() >= PARALLEL_MIN_ELEMS {
+ out.par_chunks_mut(chunk)
+ .zip(src.par_chunks(chunk))
+ .for_each(|(o, s)| f(o, s));
+ } else {
+ f(out, src);
+ }
+}
+
+fn chunk_len(channels: i32) -> usize {
+ 65536 * channels as usize
+}
+
+fn same_geometry(a: &ImageF32, b: &ImageF32, op: &str) -> OpResult<()> {
+ if !a.same_geometry(b) {
+ return Err(format!("{op}: operands have different geometry"));
+ }
+ Ok(())
+}
+
+fn map1(a: &ImageF32, f: impl Fn(f32) -> f32 + Send + Sync) -> OpResult<ImageF32> {
+ let mut data = vec![0.0f32; a.data.len()];
+ fill(&mut data, &a.data, chunk_len(a.channels), |o, s| {
+ for (o, &v) in o.iter_mut().zip(s.iter()) {
+ *o = f(v);
+ }
+ });
+ ImageF32::new(a.width, a.height, a.channels, data)
+}
+
+fn map2(
+ a: &ImageF32,
+ b: &ImageF32,
+ op: &str,
+ f: impl Fn(f32, f32) -> f32 + Send + Sync,
+) -> OpResult<ImageF32> {
+ same_geometry(a, b, op)?;
+ let mut data = vec![0.0f32; a.data.len()];
+ let chunk = chunk_len(a.channels);
+ if data.len() >= PARALLEL_MIN_ELEMS {
+ data.par_chunks_mut(chunk)
+ .zip(a.data.par_chunks(chunk))
+ .zip(b.data.par_chunks(chunk))
+ .for_each(|((o, x), y)| {
+ for ((o, &x), &y) in o.iter_mut().zip(x.iter()).zip(y.iter()) {
+ *o = f(x, y);
+ }
+ });
+ } else {
+ for ((o, &x), &y) in data.iter_mut().zip(a.data.iter()).zip(b.data.iter()) {
+ *o = f(x, y);
+ }
+ }
+ ImageF32::new(a.width, a.height, a.channels, data)
+}
+
+fn map_scalar(
+ a: &ImageF32,
+ scalar: [f64; 4],
+ f: impl Fn(f32, f64) -> f32 + Send + Sync,
+) -> OpResult<ImageF32> {
+ let cn = a.channels as usize;
+ if cn > 4 {
+ return Err("scalar operations support at most 4 channels".to_string());
+ }
+ let mut data = vec![0.0f32; a.data.len()];
+ fill(&mut data, &a.data, chunk_len(a.channels), |o, s| {
+ for (i, (o, &v)) in o.iter_mut().zip(s.iter()).enumerate() {
+ *o = f(v, scalar[i % cn]);
+ }
+ });
+ ImageF32::new(a.width, a.height, a.channels, data)
+}
+
+pub(crate) fn multiply_f32(a: &ImageF32, b: &ImageF32) -> OpResult<ImageF32> {
+ map2(a, b, "multiply_f32", |x, y| x * y)
+}
+
+pub(crate) fn multiply_f32_scalar(a: &ImageF32, scalar: [f64; 4]) -> OpResult<ImageF32> {
+ map_scalar(a, scalar, |v, s| (v as f64 * s) as f32)
+}
+
+pub(crate) fn add_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {
+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| v + s as f32)
+}
+
+pub(crate) fn subtract_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {
+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| v - s as f32)
+}
+
+pub(crate) fn subtract_f32(a: &ImageF32, b: &ImageF32) -> OpResult<ImageF32> {
+ map2(a, b, "subtract_f32", |x, y| x - y)
+}
+
+pub(crate) fn add_weighted_f32(
+ a: &ImageF32,
+ alpha: f64,
+ b: &ImageF32,
+ beta: f64,
+ gamma: f64,
+) -> OpResult<ImageF32> {
+ map2(a, b, "add_weighted_f32", |x, y| {
+ (x as f64 * alpha + y as f64 * beta + gamma) as f32
+ })
+}
+
+pub(crate) fn min_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {
+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| {
+ let s = s as f32;
+ if v < s { v } else { s }
+ })
+}
+
+pub(crate) fn max_f32_scalar(a: &ImageF32, scalar: f64) -> OpResult<ImageF32> {
+ map_scalar(a, [scalar, 0.0, 0.0, 0.0], |v, s| {
+ let s = s as f32;
+ if v > s { v } else { s }
+ })
+}
+
+pub(crate) fn log_f32(a: &ImageF32) -> OpResult<ImageF32> {
+ map1(a, f32::ln)
+}
+
+pub(crate) fn exp_f32(a: &ImageF32) -> OpResult<ImageF32> {
+ map1(a, f32::exp)
+}
+
+pub(crate) fn magnitude_f32(x: &ImageF32, y: &ImageF32) -> OpResult<ImageF32> {
+ map2(x, y, "magnitude_f32", |a, b| (a * a + b * b).sqrt())
+}
diff --git a/rust/crates/ml/src/cv/bilateral.rs b/rust/crates/ml/src/cv/bilateral.rs
new file mode 100644
index 0000000000..68855c5c62
--- /dev/null
+++ b/rust/crates/ml/src/cv/bilateral.rs
@@ -0,0 +1,81 @@
+use rayon::prelude::*;
+
+use super::reflect101;
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+pub(crate) fn bilateral_filter_u8(
+ src: &ImageU8,
+ d: i32,
+ sigma_color: f64,
+ sigma_space: f64,
+) -> OpResult<ImageU8> {
+ if src.channels != 1 {
+ return Err("bilateral_filter_u8: expected a single-channel image".to_string());
+ }
+ if sigma_color <= 1e-6 || sigma_space <= 1e-6 {
+ return Ok(src.clone());
+ }
+ let radius = if d <= 0 {
+ (sigma_space * 1.5).round() as i32
+ } else {
+ d / 2
+ }
+ .max(1) as usize;
+ let (w, h) = (src.width as usize, src.height as usize);
+
+ let stride = w + 2 * radius;
+ let mut temp = vec![0u8; stride * (h + 2 * radius)];
+ let cols: Vec<usize> = (0..stride)
+ .map(|x| reflect101(x as i64 - radius as i64, src.width) as usize)
+ .collect();
+ for (y, trow) in temp.chunks_exact_mut(stride).enumerate() {
+ let sy = reflect101(y as i64 - radius as i64, src.height) as usize;
+ let srow = &src.data[sy * w..(sy + 1) * w];
+ for (t, &sx) in trow.iter_mut().zip(cols.iter()) {
+ *t = srow[sx];
+ }
+ }
+
+ let color_coeff = -0.5 / (sigma_color * sigma_color);
+ let space_coeff = -0.5 / (sigma_space * sigma_space);
+ let color_weight: Vec<f32> = (0..256)
+ .map(|i| ((i * i) as f64 * color_coeff).exp() as f32)
+ .collect();
+
+ let ir = radius as i64;
+ let mut taps = Vec::new();
+ for i in -ir..=ir {
+ for j in -ir..=ir {
+ let r2 = (i * i + j * j) as f64;
+ if r2 > (ir * ir) as f64 {
+ continue;
+ }
+ taps.push((
+ ((i + ir) as usize * stride + (j + ir) as usize),
+ (r2 * space_coeff).exp() as f32,
+ ));
+ }
+ }
+
+ let mut out = vec![0u8; w * h];
+ out.par_chunks_exact_mut(w)
+ .enumerate()
+ .for_each(|(y, dst)| {
+ let win = &temp[y * stride..];
+ let centre = radius * stride + radius;
+ for (x, d) in dst.iter_mut().enumerate() {
+ let mid = win[centre + x] as i32;
+ let mut sum = 0.0f32;
+ let mut wsum = 0.0f32;
+ for &(off, sw) in &taps {
+ let val = win[off + x];
+ let weight = sw * color_weight[(val as i32 - mid).unsigned_abs() as usize];
+ wsum += weight;
+ sum += val as f32 * weight;
+ }
+ *d = (sum / wsum).round_ties_even() as u8;
+ }
+ });
+ ImageU8::new(src.width, src.height, 1, out)
+}
diff --git a/rust/crates/ml/src/cv/canny.rs b/rust/crates/ml/src/cv/canny.rs
new file mode 100644
index 0000000000..c38c82201b
--- /dev/null
+++ b/rust/crates/ml/src/cv/canny.rs
@@ -0,0 +1,152 @@
+//! The 75/200 thresholds used downstream are calibrated against this exact
+//! gradient scaling: integer `|dx| + |dy|` of a 3x3 Sobel with replicated
+//! borders, thresholds floored, the low one compared strictly.
+
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+const CANNY_SHIFT: i32 = 15;
+/// `(int)(tan(22.5 deg) * (1 << CANNY_SHIFT) + 0.5)`.
+const TG22: i32 = 13573;
+
+const MAYBE: u8 = 0;
+const NO: u8 = 1;
+const EDGE: u8 = 2;
+
+fn sobel_planes(src: &[u8], w: usize, h: usize, cn: usize, c: usize) -> (Vec<i16>, Vec<i16>) {
+ let mut h_diff = vec![0i16; w * h];
+ let mut h_smooth = vec![0i16; w * h];
+ for y in 0..h {
+ let row = &src[y * w * cn..(y + 1) * w * cn];
+ let at = |x: usize| row[x * cn + c] as i16;
+ let (d, s) = (
+ &mut h_diff[y * w..(y + 1) * w],
+ &mut h_smooth[y * w..(y + 1) * w],
+ );
+ for x in 0..w {
+ let left = at(x.saturating_sub(1));
+ let right = at((x + 1).min(w - 1));
+ d[x] = right - left;
+ s[x] = left + 2 * at(x) + right;
+ }
+ }
+
+ let mut dx = vec![0i16; w * h];
+ let mut dy = vec![0i16; w * h];
+ for y in 0..h {
+ let up = y.saturating_sub(1) * w;
+ let down = (y + 1).min(h - 1) * w;
+ let here = y * w;
+ for x in 0..w {
+ dx[here + x] = h_diff[up + x] + 2 * h_diff[here + x] + h_diff[down + x];
+ dy[here + x] = h_smooth[down + x] - h_smooth[up + x];
+ }
+ }
+ (dx, dy)
+}
+
+fn floor_threshold(t: f64) -> i32 {
+ t.floor().clamp(i32::MIN as f64, i32::MAX as f64) as i32
+}
+
+pub(crate) fn canny(src: &ImageU8, threshold1: f64, threshold2: f64) -> OpResult<ImageU8> {
+ let (w, h) = (src.width as usize, src.height as usize);
+ let cn = src.channels as usize;
+
+ let (low_thresh, high_thresh) = if threshold1 > threshold2 {
+ (threshold2, threshold1)
+ } else {
+ (threshold1, threshold2)
+ };
+ let low = floor_threshold(low_thresh);
+ let high = floor_threshold(high_thresh);
+
+ let (mut dx, mut dy) = sobel_planes(&src.data, w, h, cn, 0);
+ let mut norm: Vec<i32> = dx
+ .iter()
+ .zip(dy.iter())
+ .map(|(&x, &y)| (x as i32).abs() + (y as i32).abs())
+ .collect();
+ for c in 1..cn {
+ let (cdx, cdy) = sobel_planes(&src.data, w, h, cn, c);
+ for i in 0..w * h {
+ let m = (cdx[i] as i32).abs() + (cdy[i] as i32).abs();
+ if m > norm[i] {
+ norm[i] = m;
+ dx[i] = cdx[i];
+ dy[i] = cdy[i];
+ }
+ }
+ }
+
+ // The one-pixel `NO` frame keeps the unguarded hysteresis flood inside.
+ let step = w + 2;
+ let mut mag = vec![0i32; step * (h + 2)];
+ for y in 0..h {
+ mag[(y + 1) * step + 1..(y + 1) * step + 1 + w].copy_from_slice(&norm[y * w..(y + 1) * w]);
+ }
+ let mut map = vec![NO; step * (h + 2)];
+ let mut stack: Vec<usize> = Vec::new();
+
+ for y in 0..h {
+ for x in 0..w {
+ let p = (y + 1) * step + x + 1;
+ let m = mag[p];
+ if m > low {
+ let (xs, ys) = (dx[y * w + x] as i32, dy[y * w + x] as i32);
+ let ax = xs.abs();
+ let ay = ys.abs() << CANNY_SHIFT;
+ let tg22x = ax * TG22;
+ let keep = if ay < tg22x {
+ m > mag[p - 1] && m >= mag[p + 1]
+ } else if ay > tg22x + (ax << (CANNY_SHIFT + 1)) {
+ m > mag[p - step] && m >= mag[p + step]
+ } else {
+ let s: isize = if (xs ^ ys) < 0 { -1 } else { 1 };
+ let up = (p as isize - step as isize - s) as usize;
+ let down = (p as isize + step as isize + s) as usize;
+ m > mag[up] && m > mag[down]
+ };
+ if keep {
+ if m > high {
+ map[p] = EDGE;
+ stack.push(p);
+ } else {
+ map[p] = MAYBE;
+ }
+ continue;
+ }
+ }
+ map[p] = NO;
+ }
+ }
+
+ let offsets = [
+ -(step as isize) - 1,
+ -(step as isize),
+ -(step as isize) + 1,
+ -1,
+ 1,
+ step as isize - 1,
+ step as isize,
+ step as isize + 1,
+ ];
+ while let Some(p) = stack.pop() {
+ for offset in offsets {
+ let q = (p as isize + offset) as usize;
+ if map[q] == MAYBE {
+ map[q] = EDGE;
+ stack.push(q);
+ }
+ }
+ }
+
+ let mut out = vec![0u8; w * h];
+ for y in 0..h {
+ let row = &map[(y + 1) * step + 1..(y + 1) * step + 1 + w];
+ for (o, &v) in out[y * w..(y + 1) * w].iter_mut().zip(row.iter()) {
+ *o = if v == EDGE { 255 } else { 0 };
+ }
+ }
+ ImageU8::new(src.width, src.height, 1, out)
+}
diff --git a/rust/crates/ml/src/cv/channels.rs b/rust/crates/ml/src/cv/channels.rs
new file mode 100644
index 0000000000..db4531f780
--- /dev/null
+++ b/rust/crates/ml/src/cv/channels.rs
@@ -0,0 +1,94 @@
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+// 15-bit fixed-point BT.601 luma weights.
+const GRAY_SHIFT: i32 = 15;
+const RY15: i32 = 9798;
+const GY15: i32 = 19235;
+const BY15: i32 = 3735;
+
+fn require_channels(src: &ImageU8, channels: i32, op: &str) -> OpResult<()> {
+ if src.channels != channels {
+ return Err(format!(
+ "{op}: expected {channels} channel(s), got {}",
+ src.channels
+ ));
+ }
+ Ok(())
+}
+
+pub(crate) fn bgr_to_rgb(src: &ImageU8) -> OpResult<ImageU8> {
+ require_channels(src, 3, "bgr_to_rgb")?;
+ let mut data = vec![0u8; src.data.len()];
+ super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {
+ for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {
+ out[0] = px[2];
+ out[1] = px[1];
+ out[2] = px[0];
+ }
+ });
+ ImageU8::new(src.width, src.height, 3, data)
+}
+
+pub(crate) fn gray_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {
+ require_channels(src, 1, "gray_to_bgr")?;
+ let mut data = vec![0u8; src.data.len() * 3];
+ super::pointwise(&mut data, 3, &src.data, 1, |data, srcd| {
+ for (out, &v) in data.chunks_exact_mut(3).zip(srcd.iter()) {
+ out[0] = v;
+ out[1] = v;
+ out[2] = v;
+ }
+ });
+ ImageU8::new(src.width, src.height, 3, data)
+}
+
+pub(crate) fn bgr_to_gray(src: &ImageU8) -> OpResult<ImageU8> {
+ require_channels(src, 3, "bgr_to_gray")?;
+ let round = 1i32 << (GRAY_SHIFT - 1);
+ let mut data = vec![0u8; src.data.len() / 3];
+ super::pointwise(&mut data, 1, &src.data, 3, |data, srcd| {
+ for (out, px) in data.iter_mut().zip(srcd.chunks_exact(3)) {
+ let sum = px[0] as i32 * BY15 + px[1] as i32 * GY15 + px[2] as i32 * RY15;
+ *out = ((sum + round) >> GRAY_SHIFT) as u8;
+ }
+ });
+ ImageU8::new(src.width, src.height, 1, data)
+}
+
+pub(crate) fn split_u8(src: &ImageU8) -> OpResult<Vec<ImageU8>> {
+ let cn = src.channels as usize;
+ let mut planes = Vec::with_capacity(cn);
+ for c in 0..cn {
+ let mut data = vec![0u8; src.data.len() / cn];
+ super::pointwise(&mut data, 1, &src.data, cn, |data, srcd| {
+ for (out, px) in data.iter_mut().zip(srcd.chunks_exact(cn)) {
+ *out = px[c];
+ }
+ });
+ planes.push(ImageU8::new(src.width, src.height, 1, data)?);
+ }
+ Ok(planes)
+}
+
+pub(crate) fn merge_u8(channels: &[ImageU8]) -> OpResult<ImageU8> {
+ let first = channels
+ .first()
+ .ok_or_else(|| "merge_u8: no input planes".to_string())?;
+ for plane in channels {
+ if plane.channels != 1 {
+ return Err("merge_u8: every input plane must be single-channel".to_string());
+ }
+ if plane.width != first.width || plane.height != first.height {
+ return Err("merge_u8: input planes have different sizes".to_string());
+ }
+ }
+ let cn = channels.len();
+ let mut data = vec![0u8; first.data.len() * cn];
+ for (c, plane) in channels.iter().enumerate() {
+ for (i, &v) in plane.data.iter().enumerate() {
+ data[i * cn + c] = v;
+ }
+ }
+ ImageU8::new(first.width, first.height, cn as i32, data)
+}
diff --git a/rust/crates/ml/src/cv/contours.rs b/rust/crates/ml/src/cv/contours.rs
new file mode 100644
index 0000000000..73252cbd50
--- /dev/null
+++ b/rust/crates/ml/src/cv/contours.rs
@@ -0,0 +1,25 @@
+use image::GrayImage;
+
+use crate::cv::OpResult;
+use crate::cv::image::{Contour, ImageU8};
+
+pub(crate) fn find_contours(src: &ImageU8) -> OpResult<Vec<Contour>> {
+ if src.channels != 1 {
+ return Err(format!(
+ "find_contours: expected a single-channel image, got {}",
+ src.channels
+ ));
+ }
+ let img = GrayImage::from_raw(src.width as u32, src.height as u32, src.data.clone())
+ .ok_or_else(|| "find_contours: source buffer mismatch".to_string())?;
+ Ok(imageproc::contours::find_contours::<i32>(&img)
+ .into_iter()
+ .map(|c| {
+ let area = imageproc::geometry::contour_area(&c.points);
+ Contour {
+ points: c.points.into_iter().map(|p| (p.x, p.y)).collect(),
+ area,
+ }
+ })
+ .collect())
+}
diff --git a/rust/crates/ml/src/cv/convert.rs b/rust/crates/ml/src/cv/convert.rs
new file mode 100644
index 0000000000..38d38a6521
--- /dev/null
+++ b/rust/crates/ml/src/cv/convert.rs
@@ -0,0 +1,23 @@
+use super::saturate_u8_f32;
+use crate::cv::OpResult;
+use crate::cv::image::{ImageF32, ImageU8};
+
+pub(crate) fn u8_to_f32(src: &ImageU8) -> OpResult<ImageF32> {
+ let mut data = vec![0.0f32; src.data.len()];
+ super::pointwise(&mut data, 1, &src.data, 1, |o, s| {
+ for (o, &v) in o.iter_mut().zip(s) {
+ *o = v as f32;
+ }
+ });
+ ImageF32::new(src.width, src.height, src.channels, data)
+}
+
+pub(crate) fn f32_to_u8(src: &ImageF32) -> OpResult<ImageU8> {
+ let mut data = vec![0u8; src.data.len()];
+ super::pointwise(&mut data, 1, &src.data, 1, |o, s| {
+ for (o, &v) in o.iter_mut().zip(s) {
+ *o = saturate_u8_f32(v);
+ }
+ });
+ ImageU8::new(src.width, src.height, src.channels, data)
+}
diff --git a/rust/crates/ml/src/cv/draw.rs b/rust/crates/ml/src/cv/draw.rs
new file mode 100644
index 0000000000..4db69545fa
--- /dev/null
+++ b/rust/crates/ml/src/cv/draw.rs
@@ -0,0 +1,48 @@
+use image::{GrayImage, Luma};
+use imageproc::drawing::draw_polygon_mut;
+use imageproc::point::Point;
+
+use super::saturate_u8_f64;
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+pub(crate) fn fill_poly(
+ width: i32,
+ height: i32,
+ polygon: &[(i32, i32)],
+ value: f64,
+) -> OpResult<ImageU8> {
+ if width <= 0 || height <= 0 {
+ return Err(format!("fill_poly: invalid mask size {width}x{height}"));
+ }
+ let mut canvas = GrayImage::new(width as u32, height as u32);
+ let color = Luma([saturate_u8_f64(value)]);
+
+ // imageproc wants an open path with distinct first/last points.
+ let mut pts: Vec<Point<i32>> = Vec::with_capacity(polygon.len());
+ for &(x, y) in polygon {
+ if pts.last() != Some(&Point::new(x, y)) {
+ pts.push(Point::new(x, y));
+ }
+ }
+ while pts.len() > 1 && pts.first() == pts.last() {
+ pts.pop();
+ }
+
+ match pts.as_slice() {
+ [] => {}
+ [p] => {
+ if (0..width).contains(&p.x) && (0..height).contains(&p.y) {
+ canvas.put_pixel(p.x as u32, p.y as u32, color);
+ }
+ }
+ [a, b] => imageproc::drawing::draw_line_segment_mut(
+ &mut canvas,
+ (a.x as f32, a.y as f32),
+ (b.x as f32, b.y as f32),
+ color,
+ ),
+ _ => draw_polygon_mut(&mut canvas, &pts, color),
+ }
+ ImageU8::new(width, height, 1, canvas.into_raw())
+}
diff --git a/rust/crates/ml/src/cv/filter.rs b/rust/crates/ml/src/cv/filter.rs
new file mode 100644
index 0000000000..a305273c40
--- /dev/null
+++ b/rust/crates/ml/src/cv/filter.rs
@@ -0,0 +1,116 @@
+use crate::cv::OpResult;
+use crate::cv::image::{ImageF32, ImageU8};
+
+use super::reflect101;
+
+/// Taps sum to 256 (8 fractional bits).
+fn gaussian_taps(n: i32) -> OpResult<&'static [u32]> {
+ Ok(match n {
+ 1 => &[256],
+ 3 => &[64, 128, 64],
+ 5 => &[16, 64, 96, 64, 16],
+ 7 => &[8, 28, 56, 72, 56, 28, 8],
+ _ => return Err(format!("gaussian_blur_u8: unsupported ksize {n}")),
+ })
+}
+
+pub(crate) fn gaussian_blur_u8(src: &ImageU8, ksize: i32) -> OpResult<ImageU8> {
+ if src.channels != 1 {
+ return Err("gaussian_blur_u8: expected a single-channel image".to_string());
+ }
+ let kw = if src.width == 1 { 1 } else { ksize };
+ let kh = if src.height == 1 { 1 } else { ksize };
+ if kw == 1 && kh == 1 {
+ return Ok(src.clone());
+ }
+ let kx = gaussian_taps(kw)?;
+ let ky = gaussian_taps(kh)?;
+ let (w, h) = (src.width as usize, src.height as usize);
+ let (ax, ay) = ((kw / 2) as i64, (kh / 2) as i64);
+
+ // Horizontal pass into u16 (taps sum to 256), vertical into u32.
+ let mut hsum = vec![0u16; w * h];
+ for y in 0..h {
+ let row = &src.data[y * w..(y + 1) * w];
+ let out = &mut hsum[y * w..(y + 1) * w];
+ for (x, o) in out.iter_mut().enumerate() {
+ let mut acc = 0u16;
+ for (t, &tap) in kx.iter().enumerate() {
+ let sx = reflect101(x as i64 + t as i64 - ax, src.width) as usize;
+ acc += tap as u16 * row[sx] as u16;
+ }
+ *o = acc;
+ }
+ }
+ let mut out = vec![0u8; w * h];
+ let mut acc = vec![0u32; w];
+ for y in 0..h {
+ let dst = &mut out[y * w..(y + 1) * w];
+ acc.iter_mut().for_each(|a| *a = 0);
+ for (t, &tap) in ky.iter().enumerate() {
+ let sy = reflect101(y as i64 + t as i64 - ay, src.height) as usize;
+ for (a, &v) in acc.iter_mut().zip(&hsum[sy * w..(sy + 1) * w]) {
+ *a += tap * v as u32;
+ }
+ }
+ for (d, &a) in dst.iter_mut().zip(acc.iter()) {
+ *d = ((a + (1 << 15)) >> 16).min(255) as u8;
+ }
+ }
+ ImageU8::new(src.width, src.height, 1, out)
+}
+
+pub(crate) fn box_filter_f32(src: &ImageF32, kw: i32, kh: i32) -> OpResult<ImageF32> {
+ if kw <= 0 || kh <= 0 {
+ return Err(format!("box_filter_f32: invalid kernel {kw}x{kh}"));
+ }
+ if src.channels != 1 {
+ return Err("box_filter_f32: expected a single-channel image".to_string());
+ }
+ let (w, h) = (src.width as usize, src.height as usize);
+ let (ax, ay) = ((kw / 2) as i64, (kh / 2) as i64);
+ let scale = 1.0 / (kw as f64 * kh as f64);
+
+ let mut hsum = vec![0.0f64; w * h];
+ let ext_idx: Vec<usize> = (0..w + kw as usize - 1)
+ .map(|j| reflect101(j as i64 - ax, src.width) as usize)
+ .collect();
+ let mut ext = vec![0.0f64; w + kw as usize - 1];
+ for y in 0..h {
+ let row = &src.data[y * w..(y + 1) * w];
+ for (e, &sx) in ext.iter_mut().zip(ext_idx.iter()) {
+ *e = row[sx] as f64;
+ }
+ let out = &mut hsum[y * w..(y + 1) * w];
+ let mut s: f64 = ext[..kw as usize].iter().sum();
+ out[0] = s;
+ for x in 1..w {
+ s += ext[x + kw as usize - 1] - ext[x - 1];
+ out[x] = s;
+ }
+ }
+
+ let mut out = vec![0.0f32; w * h];
+ let mut col = vec![0.0f64; w];
+ for t in 0..kh as usize - 1 {
+ let sy = reflect101(t as i64 - ay, src.height) as usize;
+ for (c, &v) in col.iter_mut().zip(&hsum[sy * w..(sy + 1) * w]) {
+ *c += v;
+ }
+ }
+ for y in 0..h {
+ let enter = reflect101(y as i64 + kh as i64 - 1 - ay, src.height) as usize;
+ for (c, &v) in col.iter_mut().zip(&hsum[enter * w..(enter + 1) * w]) {
+ *c += v;
+ }
+ let dst = &mut out[y * w..(y + 1) * w];
+ for (d, &c) in dst.iter_mut().zip(col.iter()) {
+ *d = (c * scale) as f32;
+ }
+ let leave = reflect101(y as i64 - ay, src.height) as usize;
+ for (c, &v) in col.iter_mut().zip(&hsum[leave * w..(leave + 1) * w]) {
+ *c -= v;
+ }
+ }
+ ImageF32::new(src.width, src.height, 1, out)
+}
diff --git a/rust/crates/ml/src/cv/image.rs b/rust/crates/ml/src/cv/image.rs
new file mode 100644
index 0000000000..a341ae86ce
--- /dev/null
+++ b/rust/crates/ml/src/cv/image.rs
@@ -0,0 +1,148 @@
+//! Row-major, channel-interleaved, fully packed (no row padding).
+
+use super::OpResult;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct ImageU8 {
+ pub width: i32,
+ pub height: i32,
+ pub channels: i32,
+ pub data: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct ImageF32 {
+ pub width: i32,
+ pub height: i32,
+ pub channels: i32,
+ pub data: Vec<f32>,
+}
+
+fn check_len(width: i32, height: i32, channels: i32, len: usize) -> OpResult<()> {
+ if width <= 0 || height <= 0 || channels <= 0 {
+ return Err(format!(
+ "invalid image geometry {width}x{height}x{channels}"
+ ));
+ }
+ let expected = width as usize * height as usize * channels as usize;
+ if len != expected {
+ return Err(format!(
+ "image buffer length {len} does not match {width}x{height}x{channels} ({expected})"
+ ));
+ }
+ Ok(())
+}
+
+impl ImageU8 {
+ pub(crate) fn new(width: i32, height: i32, channels: i32, data: Vec<u8>) -> OpResult<Self> {
+ check_len(width, height, channels, data.len())?;
+ Ok(Self {
+ width,
+ height,
+ channels,
+ data,
+ })
+ }
+
+ pub(crate) fn zeros(width: i32, height: i32, channels: i32) -> OpResult<Self> {
+ if width <= 0 || height <= 0 || channels <= 0 {
+ return Err(format!(
+ "invalid image geometry {width}x{height}x{channels}"
+ ));
+ }
+ let len = width as usize * height as usize * channels as usize;
+ Ok(Self {
+ width,
+ height,
+ channels,
+ data: vec![0u8; len],
+ })
+ }
+
+ pub(crate) fn size(&self) -> (i32, i32) {
+ (self.width, self.height)
+ }
+
+ pub(crate) fn same_geometry(&self, other: &ImageU8) -> bool {
+ self.width == other.width && self.height == other.height && self.channels == other.channels
+ }
+}
+
+impl ImageF32 {
+ pub(crate) fn new(width: i32, height: i32, channels: i32, data: Vec<f32>) -> OpResult<Self> {
+ check_len(width, height, channels, data.len())?;
+ Ok(Self {
+ width,
+ height,
+ channels,
+ data,
+ })
+ }
+
+ pub(crate) fn zeros(width: i32, height: i32, channels: i32) -> OpResult<Self> {
+ if width <= 0 || height <= 0 || channels <= 0 {
+ return Err(format!(
+ "invalid image geometry {width}x{height}x{channels}"
+ ));
+ }
+ let len = width as usize * height as usize * channels as usize;
+ Ok(Self {
+ width,
+ height,
+ channels,
+ data: vec![0.0f32; len],
+ })
+ }
+
+ pub(crate) fn pixels(&self) -> usize {
+ self.width as usize * self.height as usize
+ }
+
+ pub(crate) fn same_geometry(&self, other: &ImageF32) -> bool {
+ self.width == other.width && self.height == other.height && self.channels == other.channels
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum ImageRef<'a> {
+ U8(&'a ImageU8),
+ F32(&'a ImageF32),
+}
+
+#[derive(Clone, Debug)]
+pub(crate) enum Image {
+ U8(ImageU8),
+ F32(ImageF32),
+}
+
+impl ImageRef<'_> {
+ pub(crate) fn size(&self) -> (i32, i32) {
+ match self {
+ ImageRef::U8(i) => i.size(),
+ ImageRef::F32(i) => (i.width, i.height),
+ }
+ }
+}
+
+impl Image {
+ pub(crate) fn into_u8(self) -> OpResult<ImageU8> {
+ match self {
+ Image::U8(i) => Ok(i),
+ Image::F32(_) => Err("expected an 8-bit image, got 32-bit float".to_string()),
+ }
+ }
+
+ pub(crate) fn into_f32(self) -> OpResult<ImageF32> {
+ match self {
+ Image::F32(i) => Ok(i),
+ Image::U8(_) => Err("expected a 32-bit float image, got 8-bit".to_string()),
+ }
+ }
+}
+
+/// `area` is the shoelace area.
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct Contour {
+ pub points: Vec<(i32, i32)>,
+ pub area: f64,
+}
diff --git a/rust/crates/ml/src/cv/lab.rs b/rust/crates/ml/src/cv/lab.rs
new file mode 100644
index 0000000000..53c88ac7ac
--- /dev/null
+++ b/rust/crates/ml/src/cv/lab.rs
@@ -0,0 +1,239 @@
+//! 8-bit BGR <-> Lab, as fixed-point dot products over precomputed tables.
+//!
+//! L, a and b are all scaled onto 0..255 (a and b biased by 128); the
+//! pipeline's chroma and luminance thresholds are calibrated in these units.
+
+use std::sync::OnceLock;
+
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+const LAB_SHIFT: i32 = 12;
+const LAB_SHIFT2: i32 = 15;
+const GAMMA_SHIFT: i32 = 3;
+const INV_GAMMA_SHIFT: i32 = 12;
+const INV_GAMMA_TAB_SIZE: usize = 1 << 12;
+const LAB_CBRT_TAB_SIZE: usize = 256 * 3 / 2 * (1 << GAMMA_SHIFT);
+const BASE_SHIFT: i32 = 14;
+const LAB_BASE: i32 = 1 << BASE_SHIFT;
+const MIN_AB_VALUE: i32 = -8145;
+const AB_TO_XZ_LEN: usize = (LAB_BASE as usize) * 9 / 4;
+const LAB2RGB_SHIFT: i32 = LAB_SHIFT + (BASE_SHIFT - INV_GAMMA_SHIFT);
+
+// sRGB <-> XYZ (D65), rows normalized by the white point during table build.
+const SRGB2XYZ_D65: [f64; 9] = [
+ 0.412453, 0.35758, 0.180423, 0.212671, 0.71516, 0.072169, 0.019334, 0.119193, 0.950227,
+];
+const XYZ2SRGB_D65: [f64; 9] = [
+ 3.240479, -1.53715, -0.498535, -0.969256, 1.875991, 0.041556, 0.055648, -0.204043, 1.057311,
+];
+const D65: [f64; 3] = [0.950456, 1.0, 1.088754];
+
+const LTHRESH: f32 = 216.0 / 24389.0;
+const LSCALE: f32 = 841.0 / 108.0;
+const LBIAS: f32 = 16.0 / 116.0;
+
+const GAMMA_THRESHOLD: f64 = 809.0 / 20000.0;
+const GAMMA_INV_THRESHOLD: f64 = 7827.0 / 2500000.0;
+const GAMMA_LOW_SCALE: f64 = 323.0 / 25.0;
+const GAMMA_POWER: f64 = 12.0 / 5.0;
+const GAMMA_XSHIFT: f64 = 11.0 / 200.0;
+
+#[inline]
+fn descale(x: i32, n: i32) -> i32 {
+ (x + (1 << (n - 1))) >> n
+}
+
+#[inline]
+fn round_i32(x: f64) -> i32 {
+ x.round_ties_even() as i32
+}
+
+#[inline]
+fn saturate_u8(x: i32) -> u8 {
+ x.clamp(0, 255) as u8
+}
+
+fn apply_gamma(x: f64) -> f64 {
+ if x <= GAMMA_THRESHOLD {
+ x / GAMMA_LOW_SCALE
+ } else {
+ ((x + GAMMA_XSHIFT) / (1.0 + GAMMA_XSHIFT)).powf(GAMMA_POWER)
+ }
+}
+
+fn apply_inv_gamma(x: f64) -> f64 {
+ if x <= GAMMA_INV_THRESHOLD {
+ x * GAMMA_LOW_SCALE
+ } else {
+ x.powf(1.0 / GAMMA_POWER) * (1.0 + GAMMA_XSHIFT) - GAMMA_XSHIFT
+ }
+}
+
+struct LabTabs {
+ srgb_gamma: [u16; 256],
+ lab_cbrt: [u16; LAB_CBRT_TAB_SIZE],
+ srgb_inv_gamma: Box<[u16]>,
+ lab_to_yf: [u16; 512],
+ ab_to_xz: Box<[i32]>,
+ /// Ordered so a BGR pixel is consumed in storage order.
+ fwd_coeffs: [i32; 9],
+ /// Rows are R, G, B.
+ inv_coeffs: [i32; 9],
+}
+
+fn build_tabs() -> LabTabs {
+ let mut srgb_gamma = [0u16; 256];
+ let int_scale = (255 * (1 << GAMMA_SHIFT)) as f64;
+ for (i, out) in srgb_gamma.iter_mut().enumerate() {
+ *out = round_i32(int_scale * apply_gamma(i as f64 / 255.0)) as u16;
+ }
+
+ let mut srgb_inv_gamma = vec![0u16; INV_GAMMA_TAB_SIZE];
+ for (i, out) in srgb_inv_gamma.iter_mut().enumerate() {
+ let x = i as f64 / INV_GAMMA_TAB_SIZE as f64;
+ *out = round_i32(255.0 * apply_inv_gamma(x)) as u16;
+ }
+
+ let mut lab_cbrt = [0u16; LAB_CBRT_TAB_SIZE];
+ let cb_tab_scale = 1.0f32 / (255.0 * (1 << GAMMA_SHIFT) as f32);
+ let lshift2 = (1 << LAB_SHIFT2) as f32;
+ for (i, out) in lab_cbrt.iter_mut().enumerate() {
+ let x = cb_tab_scale * i as f32;
+ let f = if x < LTHRESH {
+ x.mul_add(LSCALE, LBIAS)
+ } else {
+ x.cbrt()
+ };
+ *out = round_i32((lshift2 * f) as f64) as u16;
+ }
+
+ let mut lab_to_yf = [0u16; 512];
+ for i in 0..256i32 {
+ const LUT_BASE: i32 = 1 << 14;
+ let (y, ify) = if i <= 20 {
+ (
+ round_i32((i * LUT_BASE * 20 * 9) as f64 / (17 * 29 * 29 * 29) as f64),
+ round_i32(LUT_BASE as f64 * (16.0 / 116.0 + (i * 5) as f64 / (3 * 17 * 29) as f64)),
+ )
+ } else {
+ let fy =
+ (i * 100 * LUT_BASE) as f64 / (255 * 116) as f64 + (16 * LUT_BASE) as f64 / 116.0;
+ (
+ round_i32(fy * fy * fy / (LUT_BASE * LUT_BASE) as f64),
+ round_i32(fy),
+ )
+ };
+ lab_to_yf[i as usize * 2] = y as u16;
+ lab_to_yf[i as usize * 2 + 1] = ify as u16;
+ }
+
+ // Integer division truncates toward zero; `i` starts negative.
+ let mut ab_to_xz = vec![0i32; AB_TO_XZ_LEN];
+ let bias = LAB_BASE * 16 / 116 * 108 / 841;
+ for (k, out) in ab_to_xz.iter_mut().enumerate() {
+ let i = k as i32 + MIN_AB_VALUE;
+ *out = if i <= 3390 {
+ i * 108 / 841 - bias
+ } else {
+ i * i / LAB_BASE * i / LAB_BASE
+ };
+ }
+
+ let lshift = (1 << LAB_SHIFT) as f64;
+ let mut fwd_coeffs = [0i32; 9];
+ let mut inv_coeffs = [0i32; 9];
+ for i in 0..3 {
+ let wp = D65[i];
+ fwd_coeffs[i * 3 + 2] = round_i32(lshift * SRGB2XYZ_D65[i * 3] / wp);
+ fwd_coeffs[i * 3 + 1] = round_i32(lshift * SRGB2XYZ_D65[i * 3 + 1] / wp);
+ fwd_coeffs[i * 3] = round_i32(lshift * SRGB2XYZ_D65[i * 3 + 2] / wp);
+
+ inv_coeffs[i] = round_i32(lshift * XYZ2SRGB_D65[i] * wp);
+ inv_coeffs[i + 3] = round_i32(lshift * XYZ2SRGB_D65[i + 3] * wp);
+ inv_coeffs[i + 6] = round_i32(lshift * XYZ2SRGB_D65[i + 6] * wp);
+ }
+
+ LabTabs {
+ srgb_gamma,
+ lab_cbrt,
+ srgb_inv_gamma: srgb_inv_gamma.into_boxed_slice(),
+ lab_to_yf,
+ ab_to_xz: ab_to_xz.into_boxed_slice(),
+ fwd_coeffs,
+ inv_coeffs,
+ }
+}
+
+fn tabs() -> &'static LabTabs {
+ static TABS: OnceLock<LabTabs> = OnceLock::new();
+ TABS.get_or_init(build_tabs)
+}
+
+fn require_3ch(src: &ImageU8, op: &str) -> OpResult<()> {
+ if src.channels != 3 {
+ return Err(format!("{op}: expected 3 channel(s), got {}", src.channels));
+ }
+ Ok(())
+}
+
+pub(crate) fn bgr_to_lab(src: &ImageU8) -> OpResult<ImageU8> {
+ require_3ch(src, "bgr_to_lab")?;
+ let t = tabs();
+ let c = t.fwd_coeffs;
+ let l_scale = (116 * 255 + 50) / 100;
+ let l_shift = -((16 * 255 * (1 << LAB_SHIFT2) + 50) / 100);
+ let ab_shift = 128 * (1 << LAB_SHIFT2);
+
+ let mut data = vec![0u8; src.data.len()];
+ super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {
+ for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {
+ let v0 = t.srgb_gamma[px[0] as usize] as i32;
+ let v1 = t.srgb_gamma[px[1] as usize] as i32;
+ let v2 = t.srgb_gamma[px[2] as usize] as i32;
+ let fx =
+ t.lab_cbrt[descale(v0 * c[0] + v1 * c[1] + v2 * c[2], LAB_SHIFT) as usize] as i32;
+ let fy =
+ t.lab_cbrt[descale(v0 * c[3] + v1 * c[4] + v2 * c[5], LAB_SHIFT) as usize] as i32;
+ let fz =
+ t.lab_cbrt[descale(v0 * c[6] + v1 * c[7] + v2 * c[8], LAB_SHIFT) as usize] as i32;
+
+ out[0] = saturate_u8(descale(l_scale * fy + l_shift, LAB_SHIFT2));
+ out[1] = saturate_u8(descale(500 * (fx - fy) + ab_shift, LAB_SHIFT2));
+ out[2] = saturate_u8(descale(200 * (fy - fz) + ab_shift, LAB_SHIFT2));
+ }
+ });
+ ImageU8::new(src.width, src.height, 3, data)
+}
+
+pub(crate) fn lab_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {
+ require_3ch(src, "lab_to_bgr")?;
+ let t = tabs();
+ let c = t.inv_coeffs;
+ let a_bias = 128 * LAB_BASE / 500;
+ let b_bias = 128 * LAB_BASE / 200 - 1;
+ let top = INV_GAMMA_TAB_SIZE as i32 - 1;
+
+ let mut data = vec![0u8; src.data.len()];
+ super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {
+ for (out, px) in data.chunks_exact_mut(3).zip(srcd.chunks_exact(3)) {
+ let y = t.lab_to_yf[px[0] as usize * 2] as i32;
+ let ify = t.lab_to_yf[px[0] as usize * 2 + 1] as i32;
+
+ let adiv = px[1] as i32 * LAB_BASE / 500 - a_bias;
+ let bdiv = px[2] as i32 * LAB_BASE / 200 - b_bias;
+
+ let x = t.ab_to_xz[(ify + adiv - MIN_AB_VALUE) as usize];
+ let z = t.ab_to_xz[(ify - bdiv - MIN_AB_VALUE) as usize];
+
+ let r = descale(c[0] * x + c[1] * y + c[2] * z, LAB2RGB_SHIFT).clamp(0, top);
+ let g = descale(c[3] * x + c[4] * y + c[5] * z, LAB2RGB_SHIFT).clamp(0, top);
+ let b = descale(c[6] * x + c[7] * y + c[8] * z, LAB2RGB_SHIFT).clamp(0, top);
+
+ out[0] = saturate_u8(t.srgb_inv_gamma[b as usize] as i32);
+ out[1] = saturate_u8(t.srgb_inv_gamma[g as usize] as i32);
+ out[2] = saturate_u8(t.srgb_inv_gamma[r as usize] as i32);
+ }
+ });
+ ImageU8::new(src.width, src.height, 3, data)
+}
diff --git a/rust/crates/ml/src/cv/masking.rs b/rust/crates/ml/src/cv/masking.rs
new file mode 100644
index 0000000000..e1adb777f9
--- /dev/null
+++ b/rust/crates/ml/src/cv/masking.rs
@@ -0,0 +1,92 @@
+use crate::cv::OpResult;
+use crate::cv::image::{ImageF32, ImageU8};
+
+pub(crate) fn threshold_binary_u8(src: &ImageU8, thresh: f64, maxval: f64) -> OpResult<ImageU8> {
+ let ithresh = thresh.floor() as i64;
+ let imaxval = maxval.round_ties_even().clamp(0.0, 255.0) as u8;
+ ImageU8::new(
+ src.width,
+ src.height,
+ src.channels,
+ src.data
+ .iter()
+ .map(|&v| if v as i64 > ithresh { imaxval } else { 0 })
+ .collect(),
+ )
+}
+
+pub(crate) fn threshold_binary_f32(src: &ImageF32, thresh: f64, maxval: f64) -> OpResult<ImageF32> {
+ let t = thresh as f32;
+ let m = maxval as f32;
+ ImageF32::new(
+ src.width,
+ src.height,
+ src.channels,
+ src.data
+ .iter()
+ .map(|&v| if v > t { m } else { 0.0 })
+ .collect(),
+ )
+}
+
+pub(crate) fn in_range_u8(src: &ImageU8, lower: f64, upper: f64) -> OpResult<ImageU8> {
+ if src.channels != 1 {
+ return Err("in_range_u8: expected a single-channel image".to_string());
+ }
+ let lo = lower.round_ties_even().clamp(0.0, 255.0) as u8;
+ let hi = upper.round_ties_even().clamp(0.0, 255.0) as u8;
+ ImageU8::new(
+ src.width,
+ src.height,
+ 1,
+ src.data
+ .iter()
+ .map(|&v| if lo <= v && v <= hi { 255 } else { 0 })
+ .collect(),
+ )
+}
+
+pub(crate) fn bitwise_and_u8(a: &ImageU8, b: &ImageU8) -> OpResult<ImageU8> {
+ if !a.same_geometry(b) {
+ return Err("bitwise_and_u8: operands have different geometry".to_string());
+ }
+ ImageU8::new(
+ a.width,
+ a.height,
+ a.channels,
+ a.data
+ .iter()
+ .zip(b.data.iter())
+ .map(|(&x, &y)| x & y)
+ .collect(),
+ )
+}
+
+pub(crate) fn count_non_zero(src: &ImageU8) -> OpResult<i32> {
+ if src.channels != 1 {
+ return Err(format!(
+ "count_non_zero: expected a single-channel image, got {}",
+ src.channels
+ ));
+ }
+ Ok(src.data.iter().filter(|&&v| v != 0).count() as i32)
+}
+
+pub(crate) fn copy_to_masked(src: &ImageU8, dst: &ImageU8, mask: &ImageU8) -> OpResult<ImageU8> {
+ if !src.same_geometry(dst) {
+ return Err("copy_to_masked: source and destination geometry differ".to_string());
+ }
+ if mask.channels != 1 || mask.width != src.width || mask.height != src.height {
+ return Err("copy_to_masked: mask must be single-channel and the same size".to_string());
+ }
+ let cn = src.channels as usize;
+ let mut out = dst.data.clone();
+ super::pointwise3(&mut out, cn, &src.data, cn, &mask.data, 1, |o, s, m| {
+ for (i, &m) in m.iter().enumerate() {
+ if m != 0 {
+ o[i * cn..(i + 1) * cn].copy_from_slice(&s[i * cn..(i + 1) * cn]);
+ }
+ }
+ });
+ ImageU8::new(src.width, src.height, src.channels, out)
+}
diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rs
new file mode 100644
index 0000000000..0c4459a49f
--- /dev/null
+++ b/rust/crates/ml/src/cv/mod.rs
@@ -0,0 +1,129 @@
+mod arith;
+mod bilateral;
+mod canny;
+mod channels;
+mod contours;
+mod convert;
+mod draw;
+mod filter;
+pub(crate) mod image;
+mod lab;
+mod masking;
+mod morph;
+mod resize;
+mod stats;
+mod structuring;
+mod transform;
+mod warp;
+
+use rayon::prelude::*;
+
+use self::image::{Image, ImageRef};
+
+pub(crate) type OpResult<T> = Result<T, String>;
+
+pub(crate) use arith::{
+ add_f32_scalar, add_weighted_f32, exp_f32, log_f32, magnitude_f32, max_f32_scalar,
+ min_f32_scalar, multiply_f32, multiply_f32_scalar, subtract_f32, subtract_f32_scalar,
+};
+pub(crate) use bilateral::bilateral_filter_u8;
+pub(crate) use canny::canny;
+pub(crate) use channels::{bgr_to_gray, bgr_to_rgb, gray_to_bgr, merge_u8, split_u8};
+pub(crate) use contours::find_contours;
+pub(crate) use convert::{f32_to_u8, u8_to_f32};
+pub(crate) use draw::fill_poly;
+pub(crate) use filter::{box_filter_f32, gaussian_blur_u8};
+pub(crate) use lab::{bgr_to_lab, lab_to_bgr};
+pub(crate) use masking::{
+ bitwise_and_u8, copy_to_masked, count_non_zero, in_range_u8, threshold_binary_f32,
+ threshold_binary_u8,
+};
+pub(crate) use morph::{morphology_close, morphology_erode, morphology_open};
+pub(crate) use stats::{
+ hist_256_f32, hist_256_u8, mean_f32, mean_u8c3_masked, min_max_loc_f32, percentile_f32, sum_f32,
+};
+pub(crate) use structuring::ellipse_kernel;
+pub(crate) use transform::rotate_u8;
+pub(crate) use warp::warp_perspective;
+
+use resize::Interp;
+
+pub(crate) fn resize_bilinear(src: ImageRef<'_>, width: i32, height: i32) -> OpResult<Image> {
+ resize::resize(src, width, height, Interp::Bilinear)
+}
+
+pub(crate) fn resize_area(src: ImageRef<'_>, width: i32, height: i32) -> OpResult<Image> {
+ resize::resize(src, width, height, Interp::Area)
+}
+
+pub(crate) fn resize_bicubic(src: ImageRef<'_>, width: i32, height: i32) -> OpResult<Image> {
+ resize::resize(src, width, height, Interp::Bicubic)
+}
+
+const PARALLEL_MIN_PIXELS: usize = 65_536;
+const PIXELS_PER_CHUNK: usize = 65_536;
+
+pub(crate) fn pointwise<T: Sync, U: Send>(
+ out: &mut [U],
+ out_stride: usize,
+ src: &[T],
+ src_stride: usize,
+ f: impl Fn(&mut [U], &[T]) + Send + Sync,
+) {
+ if out.len() / out_stride.max(1) < PARALLEL_MIN_PIXELS {
+ f(out, src);
+ return;
+ }
+ out.par_chunks_mut(PIXELS_PER_CHUNK * out_stride)
+ .zip(src.par_chunks(PIXELS_PER_CHUNK * src_stride))
+ .for_each(|(o, s)| f(o, s));
+}
+
+pub(crate) fn pointwise3<T: Sync, V: Sync, U: Send>(
+ out: &mut [U],
+ out_stride: usize,
+ a: &[T],
+ a_stride: usize,
+ b: &[V],
+ b_stride: usize,
+ f: impl Fn(&mut [U], &[T], &[V]) + Send + Sync,
+) {
+ if out.len() / out_stride.max(1) < PARALLEL_MIN_PIXELS {
+ f(out, a, b);
+ return;
+ }
+ out.par_chunks_mut(PIXELS_PER_CHUNK * out_stride)
+ .zip(a.par_chunks(PIXELS_PER_CHUNK * a_stride))
+ .zip(b.par_chunks(PIXELS_PER_CHUNK * b_stride))
+ .for_each(|((o, a), b)| f(o, a, b));
+}
+
+pub(crate) fn saturate_u8_f32(v: f32) -> u8 {
+ v.round_ties_even().clamp(0.0, 255.0) as u8
+}
+
+pub(crate) fn saturate_u8_f64(v: f64) -> u8 {
+ v.round_ties_even().clamp(0.0, 255.0) as u8
+}
+
+/// Mirrors an out-of-range coordinate back into `[0, len)` without repeating
+/// the edge pixel.
+pub(crate) fn reflect101(mut p: i64, len: i32) -> i64 {
+ let len = len as i64;
+ if p >= 0 && p < len {
+ return p;
+ }
+ if len == 1 {
+ return 0;
+ }
+ loop {
+ if p < 0 {
+ p = -p;
+ } else {
+ p = len - 1 - (p - len) - 1;
+ }
+ if p >= 0 && p < len {
+ return p;
+ }
+ }
+}
diff --git a/rust/crates/ml/src/cv/morph.rs b/rust/crates/ml/src/cv/morph.rs
new file mode 100644
index 0000000000..9f384e4ca6
--- /dev/null
+++ b/rust/crates/ml/src/cv/morph.rs
@@ -0,0 +1,78 @@
+use rayon::prelude::*;
+
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+const PARALLEL_MIN_PIXELS: usize = 200_000;
+
+fn morph(src: &ImageU8, kernel: &ImageU8, erode: bool) -> OpResult<ImageU8> {
+ if src.channels != 1 || kernel.channels != 1 {
+ return Err("morphology: image and kernel must be single-channel".to_string());
+ }
+ if kernel.width * kernel.height == 1 {
+ return Ok(src.clone());
+ }
+ let pad: u8 = if erode { 255 } else { 0 };
+ let (w, h) = (src.width as usize, src.height as usize);
+ let (kw, kh) = (kernel.width as usize, kernel.height as usize);
+
+ let pw = w + kw - 1;
+ let mut padded = vec![pad; pw * (h + kh - 1)];
+ for y in 0..h {
+ padded[(y + kh / 2) * pw + kw / 2..(y + kh / 2) * pw + kw / 2 + w]
+ .copy_from_slice(&src.data[y * w..(y + 1) * w]);
+ }
+
+ let offsets: Vec<usize> = (0..kh * kw)
+ .filter(|&i| kernel.data[i] != 0)
+ .map(|i| (i / kw) * pw + i % kw)
+ .collect();
+ if offsets.is_empty() {
+ return ImageU8::new(src.width, src.height, 1, vec![pad; w * h]);
+ }
+
+ let mut out = vec![0u8; w * h];
+ // Tap-major keeps the min/max folds vectorizable.
+ let row = |y: usize, dst: &mut [u8]| {
+ let base = y * pw;
+ dst.copy_from_slice(&padded[base + offsets[0]..base + offsets[0] + w]);
+ for &off in &offsets[1..] {
+ let src_slice = &padded[base + off..base + off + w];
+ if erode {
+ for (d, &s) in dst.iter_mut().zip(src_slice) {
+ *d = (*d).min(s);
+ }
+ } else {
+ for (d, &s) in dst.iter_mut().zip(src_slice) {
+ *d = (*d).max(s);
+ }
+ }
+ }
+ };
+ if w * h >= PARALLEL_MIN_PIXELS {
+ out.par_chunks_mut(w)
+ .enumerate()
+ .for_each(|(y, dst)| row(y, dst));
+ } else {
+ for (y, dst) in out.chunks_mut(w).enumerate() {
+ row(y, dst);
+ }
+ }
+ ImageU8::new(src.width, src.height, 1, out)
+}
+
+pub(crate) fn morphology_erode(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
+ morph(src, kernel, true)
+}
+
+fn morphology_dilate(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
+ morph(src, kernel, false)
+}
+
+pub(crate) fn morphology_open(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
+ morphology_dilate(&morphology_erode(src, kernel)?, kernel)
+}
+
+pub(crate) fn morphology_close(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
+ morphology_erode(&morphology_dilate(src, kernel)?, kernel)
+}
diff --git a/rust/crates/ml/src/cv/resize.rs b/rust/crates/ml/src/cv/resize.rs
new file mode 100644
index 0000000000..7b33fb55ae
--- /dev/null
+++ b/rust/crates/ml/src/cv/resize.rs
@@ -0,0 +1,106 @@
+//! The `Interpolation` algs sample without low-pass filtering; downstream
+//! calibration depends on that.
+
+use fast_image_resize::{
+ FilterType, PixelType, ResizeAlg, ResizeOptions, Resizer,
+ images::{Image as FirImage, ImageRef as FirImageRef},
+};
+
+use crate::cv::OpResult;
+use crate::cv::image::{Image, ImageF32, ImageRef, ImageU8};
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub(crate) enum Interp {
+ Bilinear,
+ Area,
+ Bicubic,
+}
+
+fn pixel_type(channels: i32, is_f32: bool) -> OpResult<PixelType> {
+ Ok(match (is_f32, channels) {
+ (false, 1) => PixelType::U8,
+ (false, 3) => PixelType::U8x3,
+ (true, 1) => PixelType::F32,
+ (true, 3) => PixelType::F32x3,
+ (is_f32, n) => {
+ return Err(format!(
+ "resize: unsupported {n}-channel {} image",
+ if is_f32 { "f32" } else { "u8" }
+ ));
+ }
+ })
+}
+
+fn run(
+ src_bytes: &[u8],
+ src_size: (i32, i32),
+ pixel_type: PixelType,
+ width: i32,
+ height: i32,
+ alg: ResizeAlg,
+) -> OpResult<Vec<u8>> {
+ let src = FirImageRef::new(src_size.0 as u32, src_size.1 as u32, src_bytes, pixel_type)
+ .map_err(|e| format!("resize: bad source: {e}"))?;
+ let mut dst = FirImage::new(width as u32, height as u32, pixel_type);
+ Resizer::new()
+ .resize(&src, &mut dst, Some(&ResizeOptions::new().resize_alg(alg)))
+ .map_err(|e| format!("resize failed: {e}"))?;
+ Ok(dst.into_vec())
+}
+
+pub(crate) fn resize(
+ src: ImageRef<'_>,
+ width: i32,
+ height: i32,
+ interp: Interp,
+) -> OpResult<Image> {
+ let (sw, sh) = src.size();
+ if width <= 0 || height <= 0 {
+ return Err(format!("resize: invalid destination size {width}x{height}"));
+ }
+ if width == sw && height == sh {
+ return Ok(match src {
+ ImageRef::U8(i) => Image::U8(i.clone()),
+ ImageRef::F32(i) => Image::F32(i.clone()),
+ });
+ }
+
+ let alg = match interp {
+ Interp::Bilinear => ResizeAlg::Interpolation(FilterType::Bilinear),
+ Interp::Bicubic => ResizeAlg::Interpolation(FilterType::CatmullRom),
+ Interp::Area if width >= sw && height >= sh => {
+ ResizeAlg::Interpolation(FilterType::Bilinear)
+ }
+ Interp::Area => ResizeAlg::Convolution(FilterType::Box),
+ };
+
+ match src {
+ ImageRef::U8(s) => {
+ let data = run(
+ &s.data,
+ s.size(),
+ pixel_type(s.channels, false)?,
+ width,
+ height,
+ alg,
+ )?;
+ Ok(Image::U8(ImageU8::new(width, height, s.channels, data)?))
+ }
+ ImageRef::F32(s) => {
+ let bytes: Vec<u8> = s.data.iter().flat_map(|v| v.to_ne_bytes()).collect();
+ let out = run(
+ &bytes,
+ (s.width, s.height),
+ pixel_type(s.channels, true)?,
+ width,
+ height,
+ alg,
+ )?;
+ let data: Vec<f32> = out
+ .chunks_exact(4)
+ .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]]))
+ .collect();
+ Ok(Image::F32(ImageF32::new(width, height, s.channels, data)?))
+ }
+ }
+}
diff --git a/rust/crates/ml/src/cv/stats.rs b/rust/crates/ml/src/cv/stats.rs
new file mode 100644
index 0000000000..d88b7348d7
--- /dev/null
+++ b/rust/crates/ml/src/cv/stats.rs
@@ -0,0 +1,90 @@
+use crate::cv::OpResult;
+use crate::cv::image::{ImageF32, ImageU8};
+
+fn sum_channel0(src: &ImageF32) -> f64 {
+ let cn = src.channels as usize;
+ if cn == 1 {
+ src.data.iter().map(|&v| v as f64).sum()
+ } else {
+ src.data.chunks_exact(cn).map(|px| px[0] as f64).sum()
+ }
+}
+
+pub(crate) fn sum_f32(src: &ImageF32) -> OpResult<f64> {
+ Ok(sum_channel0(src))
+}
+
+pub(crate) fn mean_f32(src: &ImageF32) -> OpResult<f64> {
+ Ok(sum_channel0(src) / src.pixels() as f64)
+}
+
+pub(crate) fn mean_u8c3_masked(src: &ImageU8, mask: &ImageU8) -> OpResult<[f64; 3]> {
+ if src.channels != 3 {
+ return Err(format!(
+ "mean_u8c3_masked: expected a 3-channel image, got {}",
+ src.channels
+ ));
+ }
+ if mask.channels != 1 || mask.width != src.width || mask.height != src.height {
+ return Err("mean_u8c3_masked: mask must be single-channel and the same size".to_string());
+ }
+ let mut sums = [0i64; 3];
+ let mut count = 0i64;
+ for (px, &m) in src.data.chunks_exact(3).zip(mask.data.iter()) {
+ if m != 0 {
+ sums[0] += px[0] as i64;
+ sums[1] += px[1] as i64;
+ sums[2] += px[2] as i64;
+ count += 1;
+ }
+ }
+ if count == 0 {
+ return Ok([0.0; 3]);
+ }
+ Ok(sums.map(|s| s as f64 / count as f64))
+}
+
+pub(crate) fn min_max_loc_f32(src: &ImageF32) -> OpResult<(f64, f64)> {
+ let mut min_val = f32::INFINITY;
+ let mut max_val = f32::NEG_INFINITY;
+ for &v in &src.data {
+ if v < min_val {
+ min_val = v;
+ }
+ if v > max_val {
+ max_val = v;
+ }
+ }
+ Ok((min_val as f64, max_val as f64))
+}
+
+pub(crate) fn hist_256_u8(src: &ImageU8) -> OpResult<Vec<f64>> {
+ let cn = src.channels as usize;
+ let mut bins = [0u32; 256];
+ for px in src.data.chunks_exact(cn) {
+ bins[px[0] as usize] += 1;
+ }
+ Ok(bins.iter().map(|&c| c as f64).collect())
+}
+
+pub(crate) fn hist_256_f32(src: &ImageF32) -> OpResult<Vec<f64>> {
+ let cn = src.channels as usize;
+ let mut bins = [0u32; 256];
+ for px in src.data.chunks_exact(cn) {
+ let v = px[0] as f64;
+ if (0.0..256.0).contains(&v) {
+ bins[v as usize] += 1;
+ }
+ }
+ Ok(bins.iter().map(|&c| c as f64).collect())
+}
+
+pub(crate) fn percentile_f32(src: &ImageF32, p: f64) -> OpResult<f32> {
+ let mut values = src.data.clone();
+ if values.is_empty() {
+ return Err("percentile_f32: empty image".to_string());
+ }
+ let index = ((values.len() as f64 * p) as usize).min(values.len() - 1);
+ let (_, value, _) = values.select_nth_unstable_by(index, f32::total_cmp);
+ Ok(*value)
+}
diff --git a/rust/crates/ml/src/cv/structuring.rs b/rust/crates/ml/src/cv/structuring.rs
new file mode 100644
index 0000000000..12d8cd91cb
--- /dev/null
+++ b/rust/crates/ml/src/cv/structuring.rs
@@ -0,0 +1,37 @@
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+pub(crate) fn ellipse_kernel(ksize: i32) -> OpResult<ImageU8> {
+ if ksize <= 0 {
+ return Err(format!("ellipse_kernel: invalid ksize {ksize}"));
+ }
+ let mut element = ImageU8::zeros(ksize, ksize, 1)?;
+ let rect = ksize == 1;
+ let r = ksize / 2;
+ let inv_r2 = if r != 0 {
+ 1.0 / (r as f64 * r as f64)
+ } else {
+ 0.0
+ };
+ let c = ksize / 2;
+
+ for i in 0..ksize {
+ let (mut j1, mut j2) = (0i32, 0i32);
+ if rect {
+ j2 = ksize;
+ } else {
+ let dy = i - r;
+ if dy.abs() <= r {
+ let dx = (c as f64 * (((r * r - dy * dy) as f64) * inv_r2).sqrt()).round_ties_even()
+ as i32;
+ j1 = (c - dx).max(0);
+ j2 = (c + dx + 1).min(ksize);
+ }
+ }
+ let row = (i * ksize) as usize;
+ for j in j1.max(0)..j2 {
+ element.data[row + j as usize] = 1;
+ }
+ }
+ Ok(element)
+}
diff --git a/rust/crates/ml/src/cv/transform.rs b/rust/crates/ml/src/cv/transform.rs
new file mode 100644
index 0000000000..9293afb36f
--- /dev/null
+++ b/rust/crates/ml/src/cv/transform.rs
@@ -0,0 +1,58 @@
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+pub(crate) fn rotate_u8(src: &ImageU8, degrees: i32) -> OpResult<ImageU8> {
+ let cn = src.channels as usize;
+ let (w, h) = (src.width as usize, src.height as usize);
+ let degrees = degrees.rem_euclid(360);
+ let (out_w, out_h) = match degrees {
+ 0 | 180 => (src.width, src.height),
+ 90 | 270 => (src.height, src.width),
+ other => {
+ return Err(format!(
+ "Only 0, 90, 180, 270 degrees are supported, got {other}"
+ ));
+ }
+ };
+
+ let mut data = vec![0u8; src.data.len()];
+ // A per-pixel `copy_from_slice` is a `memcpy` call per pixel; the common
+ // channel counts are spelled out instead.
+ let move_px = |dst: &mut [u8], d: usize, s: usize| match cn {
+ 1 => dst[d] = src.data[s],
+ 3 => {
+ dst[d] = src.data[s];
+ dst[d + 1] = src.data[s + 1];
+ dst[d + 2] = src.data[s + 2];
+ }
+ _ => dst[d..d + cn].copy_from_slice(&src.data[s..s + cn]),
+ };
+
+ match degrees {
+ 0 => data.copy_from_slice(&src.data),
+ 180 => {
+ for y in 0..h {
+ for x in 0..w {
+ let s = ((h - 1 - y) * w + (w - 1 - x)) * cn;
+ let d = (y * w + x) * cn;
+ move_px(&mut data, d, s);
+ }
+ }
+ }
+ _ => {
+ let ow = out_w as usize;
+ let oh = out_h as usize;
+ for y in 0..oh {
+ for x in 0..ow {
+ let (sx, sy) = if degrees == 90 {
+ (y, h - 1 - x)
+ } else {
+ (w - 1 - y, x)
+ };
+ move_px(&mut data, (y * ow + x) * cn, (sy * w + sx) * cn);
+ }
+ }
+ }
+ }
+ ImageU8::new(out_w, out_h, src.channels, data)
+}
diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rs
new file mode 100644
index 0000000000..122d8a3f7f
--- /dev/null
+++ b/rust/crates/ml/src/cv/warp.rs
@@ -0,0 +1,40 @@
+use image::{Rgb, RgbImage};
+use imageproc::geometric_transformations::{Interpolation, Projection, warp_into};
+
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+pub(crate) fn warp_perspective(
+ src: &ImageU8,
+ src_corners: [(f64, f64); 4],
+ dst_corners: [(f64, f64); 4],
+ width: i32,
+ height: i32,
+) -> OpResult<ImageU8> {
+ if width <= 0 || height <= 0 {
+ return Err(format!(
+ "warp_perspective: invalid destination size {width}x{height}"
+ ));
+ }
+ if src.channels != 3 {
+ return Err(format!(
+ "warp_perspective: expected a 3-channel image, got {}",
+ src.channels
+ ));
+ }
+ let narrow = |c: [(f64, f64); 4]| c.map(|(x, y)| (x as f32, y as f32));
+ let projection = Projection::from_control_points(narrow(src_corners), narrow(dst_corners))
+ .ok_or_else(|| "warp_perspective: the corner pairs are degenerate".to_string())?;
+
+ let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())
+ .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;
+ let mut out = RgbImage::new(width as u32, height as u32);
+ warp_into(
+ &source,
+ &projection,
+ Interpolation::Bilinear,
+ Rgb([0, 0, 0]),
+ &mut out,
+ );
+ ImageU8::new(width, height, 3, out.into_raw())
+}
diff --git a/rust/crates/ml/src/lib.rs b/rust/crates/ml/src/lib.rs
index eaf9fa9654..317864d02d 100644
--- a/rust/crates/ml/src/lib.rs
+++ b/rust/crates/ml/src/lib.rs
@@ -1,5 +1,6 @@
pub mod assets;
mod clip;
+mod cv;
mod diagnostics;
pub mod error;
pub mod face;
@@ -12,6 +13,7 @@ mod pet;
mod postprocess;
mod preprocess;
mod runtime;
+pub mod scan;
pub mod types;
#[cfg(feature = "usearch")]
pub mod vector_db;
diff --git a/rust/crates/ml/src/scan/codec.rs b/rust/crates/ml/src/scan/codec.rs
new file mode 100644
index 0000000000..ffa1086ed2
--- /dev/null
+++ b/rust/crates/ml/src/scan/codec.rs
@@ -0,0 +1,41 @@
+use ente_image::decode::decode_image_from_bytes;
+use ente_image::image_compression::{EncodedImageFormat, encode_rgb};
+
+use super::scanner::ScanError;
+use crate::cv::image::ImageU8;
+
+/// EXIF orientation comes pre-applied by the decoder.
+pub(crate) fn decode_bgr(bytes: &[u8]) -> Result<ImageU8, ScanError> {
+ let decoded = decode_image_from_bytes(bytes)
+ .map_err(|err| ScanError::Codec(format!("failed to decode image: {err}")))?;
+ let width = i32::try_from(decoded.dimensions.width)
+ .map_err(|_| ScanError::Codec("image width does not fit in i32".to_string()))?;
+ let height = i32::try_from(decoded.dimensions.height)
+ .map_err(|_| ScanError::Codec("image height does not fit in i32".to_string()))?;
+
+ let mut bgr = decoded.rgb;
+ for px in bgr.chunks_exact_mut(3) {
+ px.swap(0, 2);
+ }
+ ImageU8::new(width, height, 3, bgr).map_err(ScanError::Codec)
+}
+
+pub(crate) fn encode_jpeg(image: &ImageU8, quality: u8) -> Result<Vec<u8>, ScanError> {
+ if image.channels != 3 {
+ return Err(ScanError::Codec(format!(
+ "cannot encode a {}-channel image",
+ image.channels
+ )));
+ }
+ let mut rgb = image.data.clone();
+ for px in rgb.chunks_exact_mut(3) {
+ px.swap(0, 2);
+ }
+ encode_rgb(
+ &rgb,
+ image.width as u32,
+ image.height as u32,
+ EncodedImageFormat::Jpeg { quality },
+ )
+ .map_err(|err| ScanError::Codec(err.to_string()))
+}
diff --git a/rust/crates/ml/src/scan/color.rs b/rust/crates/ml/src/scan/color.rs
new file mode 100644
index 0000000000..182e2e3c04
--- /dev/null
+++ b/rust/crates/ml/src/scan/color.rs
@@ -0,0 +1,146 @@
+use super::OpResult;
+use super::detection::resize_for_max_pixels;
+use super::geometry::{Quad, norm};
+use super::mask::Mask;
+use crate::cv;
+use crate::cv::image::{ImageF32, ImageRef, ImageU8};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ColorMode {
+ Color,
+ Grayscale,
+}
+
+const CHROMA_THRESHOLD: f64 = 17.5;
+const PROPORTION_THRESHOLD: f64 = 0.0003;
+const LUMINANCE_MIN: f64 = 40.0;
+const LUMINANCE_MAX: f64 = 180.0;
+
+/// `quad` is in original-image coordinates.
+pub(crate) fn auto_color_mode(img: &ImageU8, mask: &Mask, quad: &Quad) -> OpResult<ColorMode> {
+ let resized_img = resize_for_max_pixels(img, 1024.0 * 768.0)?;
+ let work_size = resized_img.size();
+
+ let doc_mask = document_mask(mask, quad, img.size(), work_size)?;
+ let white_balanced = apply_gray_world_to_document(&resized_img, &doc_mask)?;
+
+ let lab = cv::bgr_to_lab(&white_balanced)?;
+ let channels = cv::split_u8(&lab)?;
+ let luminance = &channels[0];
+ let a = &channels[1];
+ let b = &channels[2];
+
+ let chroma = chroma(a, b)?;
+
+ let color_mask_f = cv::threshold_binary_f32(&chroma, CHROMA_THRESHOLD, 255.0)?;
+ let color_mask = cv::f32_to_u8(&color_mask_f)?;
+
+ let luminance_mask = cv::in_range_u8(luminance, LUMINANCE_MIN, LUMINANCE_MAX)?;
+
+ let tmp = cv::bitwise_and_u8(&color_mask, &luminance_mask)?;
+ let restricted_mask = cv::bitwise_and_u8(&tmp, &doc_mask)?;
+
+ let colored_pixels = cv::count_non_zero(&restricted_mask)?;
+ let total_pixels = cv::count_non_zero(&doc_mask)?;
+
+ if total_pixels == 0 {
+ return Ok(ColorMode::Grayscale);
+ }
+ let proportion = colored_pixels as f64 / total_pixels as f64;
+ Ok(if proportion > PROPORTION_THRESHOLD {
+ ColorMode::Color
+ } else {
+ ColorMode::Grayscale
+ })
+}
+
+fn chroma(a: &ImageU8, b: &ImageU8) -> OpResult<ImageF32> {
+ let a_float = cv::u8_to_f32(a)?;
+ let b_float = cv::u8_to_f32(b)?;
+ let a_shifted = cv::subtract_f32_scalar(&a_float, 128.0)?;
+ let b_shifted = cv::subtract_f32_scalar(&b_float, 128.0)?;
+ cv::magnitude_f32(&a_shifted, &b_shifted)
+}
+
+fn erode_border(mask: &ImageU8, quad: &Quad) -> OpResult<ImageU8> {
+ let min_dim = quad
+ .edges()
+ .iter()
+ .map(|&(from, to)| norm(from, to))
+ .fold(f64::INFINITY, f64::min);
+ let mut k = (min_dim * 0.02 + 0.5).floor() as i32;
+ k = k.clamp(3, 15);
+ if k % 2 == 0 {
+ k += 1;
+ }
+
+ let kernel = cv::ellipse_kernel(k)?;
+ cv::morphology_erode(mask, &kernel)
+}
+
+fn document_mask(
+ mask: &Mask,
+ quad: &Quad,
+ orig_size: (i32, i32),
+ work_size: (i32, i32),
+) -> OpResult<ImageU8> {
+ let mask_image = mask.to_image()?;
+ let resized_mask =
+ cv::resize_area(ImageRef::U8(&mask_image), work_size.0, work_size.1)?.into_u8()?;
+
+ let resized_quad = quad.scaled_to(
+ orig_size.0 as f64,
+ orig_size.1 as f64,
+ work_size.0 as f64,
+ work_size.1 as f64,
+ );
+
+ let eroded_mask = erode_border(&resized_mask, &resized_quad)?;
+
+ let pts: Vec<(i32, i32)> = resized_quad
+ .corners()
+ .iter()
+ .map(|p| (p.x as i32, p.y as i32))
+ .collect();
+ let quad_mask = cv::fill_poly(eroded_mask.width, eroded_mask.height, &pts, 255.0)?;
+
+ cv::bitwise_and_u8(&eroded_mask, &quad_mask)
+}
+
+pub(crate) fn apply_gray_world_to_document(img: &ImageU8, doc_mask: &ImageU8) -> OpResult<ImageU8> {
+ assert_eq!(img.channels, 3, "gray-world expects an 8UC3 image");
+
+ let non_zero = cv::count_non_zero(doc_mask)?;
+ if non_zero == 0 {
+ return Ok(img.clone());
+ }
+
+ let mean = cv::mean_u8c3_masked(img, doc_mask)?;
+ let eps = 1e-6;
+ let mean_b = if mean[0] < eps { eps } else { mean[0] };
+ let mean_g = if mean[1] < eps { eps } else { mean[1] };
+ let mean_r = if mean[2] < eps { eps } else { mean[2] };
+
+ let mean_gray = (mean_b + mean_g + mean_r) / 3.0;
+ let scales = [
+ mean_gray / mean_b,
+ mean_gray / mean_g,
+ mean_gray / mean_r,
+ 0.0,
+ ];
+
+ let img_f = cv::u8_to_f32(img)?;
+ let scaled_f = cv::multiply_f32_scalar(&img_f, scales)?;
+ let scaled8 = cv::f32_to_u8(&scaled_f)?;
+
+ cv::copy_to_masked(&scaled8, img, doc_mask)
+}
+
+pub(crate) fn quad_to_image(quad: &Quad, mask: &Mask, image_size: (i32, i32)) -> Quad {
+ quad.scaled_to(
+ mask.width as f64,
+ mask.height as f64,
+ image_size.0 as f64,
+ image_size.1 as f64,
+ )
+}
diff --git a/rust/crates/ml/src/scan/contour_orientation.rs b/rust/crates/ml/src/scan/contour_orientation.rs
new file mode 100644
index 0000000000..3a457d44b7
--- /dev/null
+++ b/rust/crates/ml/src/scan/contour_orientation.rs
@@ -0,0 +1,329 @@
+use super::geometry::{Point, average};
+
+const SMOOTH_WINDOW: i32 = 5;
+const MIN_SIDE_LENGTH_RATIO: f64 = 0.02;
+
+#[derive(Clone, Copy, Debug)]
+struct Line {
+ p: Point,
+ d: Point,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct ContourSegment {
+ start: usize,
+ end: usize,
+ angle: f64,
+ length: f64,
+}
+
+pub(crate) fn find_quad_from_contour_orientation(contour: &[Point]) -> Option<Vec<Point>> {
+ let max_angle_var = 5.0f64.to_radians();
+ let merge_angle = 7.0f64.to_radians();
+
+ if contour.len() < 20 {
+ return None;
+ }
+
+ let angles = compute_smoothed_angles(contour, SMOOTH_WINDOW);
+
+ // Open polyline: n-1 terms.
+ let perimeter: f64 = contour
+ .windows(2)
+ .map(|w| (w[1].x - w[0].x).hypot(w[1].y - w[0].y))
+ .sum();
+
+ let min_length = perimeter * MIN_SIDE_LENGTH_RATIO;
+ let segments = extract_segments(contour, &angles, max_angle_var, min_length);
+ let merged = merge_segments(&segments, merge_angle);
+ let dominant = select_dominant_segments(&merged, 4, 25.0f64.to_radians());
+
+ if dominant.len() != 4 {
+ return None;
+ }
+
+ let lines: Vec<Line> = dominant
+ .iter()
+ .map(|s| {
+ let points: Vec<Point> = if s.start < s.end {
+ contour[s.start..s.end].to_vec()
+ } else {
+ let mut v = contour[s.start..].to_vec();
+ v.extend_from_slice(&contour[..s.end]);
+ v
+ };
+ fit_line(&points)
+ })
+ .collect();
+
+ let mut corners = Vec::with_capacity(4);
+ for i in 0..4 {
+ corners.push(intersect_lines(lines[i], lines[(i + 1) % 4])?);
+ }
+ Some(corners)
+}
+
+fn normalize_angle(a: f64) -> f64 {
+ let mut x = a;
+ while x <= -std::f64::consts::PI {
+ x += 2.0 * std::f64::consts::PI;
+ }
+ while x > std::f64::consts::PI {
+ x -= 2.0 * std::f64::consts::PI;
+ }
+ x
+}
+
+fn angle_diff(a: f64, b: f64) -> f64 {
+ normalize_angle(a - b).abs()
+}
+
+fn angle_mean(a: f64, b: f64) -> f64 {
+ let x = a.cos() + b.cos();
+ let y = a.sin() + b.sin();
+ y.atan2(x)
+}
+
+/// Total least squares direction from the second-moment matrix.
+fn fit_line(points: &[Point]) -> Line {
+ let cx = average(points.iter().map(|p| p.x));
+ let cy = average(points.iter().map(|p| p.y));
+
+ let mut xx = 0.0;
+ let mut xy = 0.0;
+ let mut yy = 0.0;
+ for p in points {
+ let dx = p.x - cx;
+ let dy = p.y - cy;
+ xx += dx * dx;
+ xy += dx * dy;
+ yy += dy * dy;
+ }
+
+ let theta = 0.5 * (2.0 * xy).atan2(xx - yy);
+ Line {
+ p: Point::new(cx, cy),
+ d: Point::new(theta.cos(), theta.sin()),
+ }
+}
+
+fn intersect_lines(l1: Line, l2: Line) -> Option<Point> {
+ let x1 = l1.p.x;
+ let y1 = l1.p.y;
+ let x2 = x1 + l1.d.x;
+ let y2 = y1 + l1.d.y;
+
+ let x3 = l2.p.x;
+ let y3 = l2.p.y;
+ let x4 = x3 + l2.d.x;
+ let y4 = y3 + l2.d.y;
+
+ let denom = (x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4);
+ if denom.abs() < 1e-6 {
+ return None;
+ }
+
+ let px = ((x1 * y2 - y1 * x2) * (x3 - x4) - (x1 - x2) * (x3 * y4 - y3 * x4)) / denom;
+ let py = ((x1 * y2 - y1 * x2) * (y3 - y4) - (y1 - y2) * (x3 * y4 - y3 * x4)) / denom;
+ Some(Point::new(px, py))
+}
+
+fn extract_segments(
+ contour: &[Point],
+ angles: &[f64],
+ max_angle_var: f64,
+ min_length: f64,
+) -> Vec<ContourSegment> {
+ let n = contour.len();
+ let mut result = Vec::new();
+
+ let start_index = find_best_start_index(angles);
+
+ let segment_length = |s: usize, e: usize| -> f64 {
+ let mut len = 0.0;
+ let mut i = s;
+ while i != e {
+ let j = (i + 1) % n;
+ len += (contour[j].x - contour[i].x).hypot(contour[j].y - contour[i].y);
+ i = j;
+ }
+ len
+ };
+
+ let mut start = start_index;
+ let mut ref_angle = angles[start_index];
+
+ let mut steps = 1usize;
+ while steps <= n {
+ let idx = (start_index + steps) % n;
+ if steps < n && angle_diff(angles[idx], ref_angle) < max_angle_var {
+ ref_angle = angle_mean(ref_angle, angles[idx]);
+ } else {
+ let len = segment_length(start, idx);
+ if len >= min_length {
+ result.push(ContourSegment {
+ start,
+ end: idx,
+ angle: ref_angle,
+ length: len,
+ });
+ }
+ start = idx;
+ ref_angle = angles[idx];
+ }
+ steps += 1;
+ }
+
+ result
+}
+
+fn find_best_start_index(angles: &[f64]) -> usize {
+ let n = angles.len();
+ let mut best_index = 0usize;
+ let mut best_delta = 0.0;
+ for i in 0..n {
+ let j = (i + 1) % n;
+ let d = angle_diff(angles[i], angles[j]);
+ if d > best_delta {
+ best_delta = d;
+ best_index = j;
+ }
+ }
+ best_index
+}
+
+fn compute_smoothed_angles(contour: &[Point], window: i32) -> Vec<f64> {
+ let n = contour.len();
+ let ni = n as i32;
+
+ let mut angles = vec![0.0f64; n];
+ for i in 0..n {
+ let p0 = contour[(i + n - 1) % n];
+ let p1 = contour[(i + 1) % n];
+ angles[i] = (p1.y - p0.y).atan2(p1.x - p0.x);
+ }
+
+ let cos_a: Vec<f64> = angles.iter().map(|a| a.cos()).collect();
+ let sin_a: Vec<f64> = angles.iter().map(|a| a.sin()).collect();
+
+ let mut smooth = vec![0.0f64; n];
+ let mut sx = 0.0;
+ let mut sy = 0.0;
+ for k in -window..=window {
+ // n >= 20 and window == 5, so `k + n` is always positive.
+ let idx = ((k + ni) % ni) as usize;
+ sx += cos_a[idx];
+ sy += sin_a[idx];
+ }
+ smooth[0] = sy.atan2(sx);
+
+ for (i, out) in smooth.iter_mut().enumerate().skip(1) {
+ let ii = i as i32;
+ let out_idx = ((ii - window - 1 + ni) % ni) as usize;
+ let in_idx = ((ii + window) % ni) as usize;
+ sx -= cos_a[out_idx];
+ sy -= sin_a[out_idx];
+ sx += cos_a[in_idx];
+ sy += sin_a[in_idx];
+ *out = sy.atan2(sx);
+ }
+ smooth
+}
+
+fn merge_segments(segments: &[ContourSegment], angle_threshold: f64) -> Vec<ContourSegment> {
+ if segments.is_empty() {
+ return Vec::new();
+ }
+ if segments.len() <= 4 {
+ return segments.to_vec();
+ }
+
+ let mut merged: Vec<ContourSegment> = Vec::new();
+ let mut cur = segments[0];
+ for p in &segments[1..] {
+ if angle_diff(p.angle, cur.angle) < angle_threshold {
+ cur = ContourSegment {
+ start: cur.start,
+ end: p.end,
+ angle: angle_mean(cur.angle, p.angle),
+ length: cur.length + p.length,
+ };
+ } else {
+ merged.push(cur);
+ cur = *p;
+ }
+ }
+ merged.push(cur);
+ merged
+}
+
+/// Both sorts are stable, which decides ties.
+fn select_dominant_segments(
+ segments: &[ContourSegment],
+ max_count: usize,
+ min_angle_separation: f64,
+) -> Vec<ContourSegment> {
+ let mut sorted = segments.to_vec();
+ sorted.sort_by(|a, b| b.length.total_cmp(&a.length));
+
+ let mut selected: Vec<ContourSegment> = Vec::new();
+ for p in &sorted {
+ let too_close = selected
+ .iter()
+ .any(|s| angle_diff(p.angle, s.angle) < min_angle_separation);
+ if !too_close {
+ selected.push(*p);
+ if selected.len() == max_count {
+ break;
+ }
+ }
+ }
+
+ selected.sort_by_key(|s| s.start);
+ selected
+}
+
+#[cfg(test)]
+mod tests {
+ use super::find_quad_from_contour_orientation;
+ use crate::scan::geometry::{Point, norm};
+
+ #[test]
+ fn contour_orientation_recovers_a_rotated_rectangle() {
+ let corners = [
+ Point::new(40.0, 30.0),
+ Point::new(220.0, 50.0),
+ Point::new(200.0, 190.0),
+ Point::new(20.0, 170.0),
+ ];
+ let mut contour = Vec::new();
+ for i in 0..4 {
+ let a = corners[i];
+ let b = corners[(i + 1) % 4];
+ let steps = 60;
+ for s in 0..steps {
+ let t = s as f64 / steps as f64;
+ contour.push(Point::new(a.x + (b.x - a.x) * t, a.y + (b.y - a.y) * t));
+ }
+ }
+ let quad =
+ find_quad_from_contour_orientation(&contour).expect("quad from contour orientation");
+ assert_eq!(quad.len(), 4);
+ for corner in corners {
+ let best = quad
+ .iter()
+ .map(|p| norm(*p, corner))
+ .fold(f64::INFINITY, f64::min);
+ assert!(
+ best < 1.0,
+ "corner {corner:?} not recovered (min dist {best})"
+ );
+ }
+ }
+
+ #[test]
+ fn contour_orientation_rejects_short_contours() {
+ let contour: Vec<Point> = (0..19).map(|i| Point::new(i as f64, 0.0)).collect();
+ assert!(find_quad_from_contour_orientation(&contour).is_none());
+ }
+}
diff --git a/rust/crates/ml/src/scan/detection.rs b/rust/crates/ml/src/scan/detection.rs
new file mode 100644
index 0000000000..2b546dd7b7
--- /dev/null
+++ b/rust/crates/ml/src/scan/detection.rs
@@ -0,0 +1,244 @@
+use super::OpResult;
+use super::color::ColorMode;
+use super::contour_orientation::find_quad_from_contour_orientation;
+use super::geometry::{ImageSize, Point, Quad, create_quad};
+use super::mask::Mask;
+use super::perspective::estimate_real_dimensions;
+use super::postprocess::enhance_captured_image;
+use super::quad_score::score_quad_against_probmap;
+use crate::cv;
+use crate::cv::image::{Contour, ImageRef, ImageU8};
+
+const THRESHOLDS: [f64; 7] = [0.5, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
+const LIVE_THRESHOLDS: [f64; 1] = [0.9];
+
+pub(crate) fn size_trunc(width: f64, height: f64) -> (i32, i32) {
+ (width as i32, height as i32)
+}
+
+/// The returned quad is in mask coordinates.
+pub(crate) fn detect_document_quad(
+ mask: &Mask,
+ original_size: ImageSize,
+ is_live_analysis: bool,
+) -> OpResult<Option<Quad>> {
+ let mask_image = mask.to_image()?;
+ let thresholds: &[f64] = if is_live_analysis {
+ &LIVE_THRESHOLDS
+ } else {
+ &THRESHOLDS
+ };
+
+ let mut vertices =
+ find_quad_from_orientation_with_adaptive_threshold(&mask_image, original_size, thresholds)?;
+
+ if vertices.is_none() && !is_live_analysis {
+ // Fallback: min-area rectangle of the biggest contour.
+ if let Some(biggest) = biggest_contour(&mask_image)? {
+ vertices = min_area_rect(&biggest.points, mask.width, mask.height);
+ }
+ }
+
+ Ok(match vertices {
+ Some(v) if v.len() == 4 => Some(create_quad(&v)),
+ _ => None,
+ })
+}
+
+fn find_quad_from_orientation_with_adaptive_threshold(
+ mask_image: &ImageU8,
+ original_size: ImageSize,
+ thresholds: &[f64],
+) -> OpResult<Option<Vec<Point>>> {
+ let probmap_smooth = cv::gaussian_blur_u8(mask_image, 3)?;
+
+ let kernel = cv::ellipse_kernel(5)?;
+ let prob_float = cv::u8_to_f32(mask_image)?;
+ let mut best_quad: Option<Vec<Point>> = None;
+ let mut best_score = 0.0f64;
+
+ for &thr in thresholds {
+ let bin = cv::threshold_binary_u8(&probmap_smooth, thr * 255.0, 255.0)?;
+ let closed = cv::morphology_close(&bin, &kernel)?;
+
+ if let Some(quad) = find_quad_from_orientation(&closed, original_size)?
+ && is_valid_quad(&quad)
+ {
+ let score = score_quad_against_probmap(&quad, &prob_float, 0.02)?;
+ if score > best_score {
+ best_score = score;
+ best_quad = Some(quad);
+ }
+ }
+ }
+
+ Ok(best_quad)
+}
+
+/// Deliberately no upper bound: corners past the mask edge are legitimate
+/// for edge-touching documents.
+pub(crate) fn is_valid_quad(quad: &[Point]) -> bool {
+ quad.iter().all(|p| p.x >= 0.0 && p.y >= 0.0)
+}
+
+fn find_quad_from_orientation(
+ mask_image: &ImageU8,
+ original_size: ImageSize,
+) -> OpResult<Option<Vec<Point>>> {
+ let Some(contour) = biggest_contour(mask_image)? else {
+ return Ok(None);
+ };
+
+ let scale_x = original_size.width / mask_image.width as f64;
+ let scale_y = original_size.height / mask_image.height as f64;
+
+ let scaled: Vec<Point> = contour
+ .points
+ .iter()
+ .map(|&(x, y)| Point::new(x as f64 * scale_x, y as f64 * scale_y))
+ .collect();
+
+ Ok(find_quad_from_contour_orientation(&scaled).map(|corners| {
+ corners
+ .iter()
+ .map(|p| Point::new(p.x / scale_x, p.y / scale_y))
+ .collect()
+ }))
+}
+
+pub(crate) fn min_area_rect(points: &[(i32, i32)], width: i32, height: i32) -> Option<Vec<Point>> {
+ if points.len() < 3 {
+ return None;
+ }
+ let pts: Vec<imageproc::point::Point<i32>> = points
+ .iter()
+ .map(|&(x, y)| imageproc::point::Point::new(x, y))
+ .collect();
+ let rect = imageproc::geometry::min_area_rect(&pts);
+ if imageproc::geometry::contour_area(&rect) == 0.0 {
+ return None;
+ }
+ Some(
+ rect.iter()
+ .map(|p| {
+ Point::new(
+ (p.x as f64).clamp(0.0, width as f64 - 1.0),
+ (p.y as f64).clamp(0.0, height as f64 - 1.0),
+ )
+ })
+ .collect(),
+ )
+}
+
+pub(crate) fn biggest_contour(image: &ImageU8) -> OpResult<Option<Contour>> {
+ let refined = refine_mask(image)?;
+ let blurred = cv::gaussian_blur_u8(&refined, 5)?;
+ let edges = cv::canny(&blurred, 75.0, 200.0)?;
+ let contours = cv::find_contours(&edges)?;
+
+ let mut biggest: Option<Contour> = None;
+ let mut max_area = 0.0f64;
+ for contour in contours {
+ let area = contour.area.abs();
+ if area > max_area {
+ max_area = area;
+ biggest = Some(contour);
+ }
+ }
+ Ok(biggest)
+}
+
+fn refine_mask(original: &ImageU8) -> OpResult<ImageU8> {
+ let binary_mask = cv::threshold_binary_u8(original, 128.0, 255.0)?;
+ let kernel = cv::ellipse_kernel(5)?;
+ let closed = cv::morphology_close(&binary_mask, &kernel)?;
+ cv::morphology_open(&closed, &kernel)
+}
+
+pub(crate) fn resize_for_max_pixels(img: &ImageU8, max_pixels: f64) -> OpResult<ImageU8> {
+ let orig_pixels = img.width as i64 * img.height as i64;
+ if orig_pixels as f64 <= max_pixels {
+ return Ok(img.clone());
+ }
+ let scale = (max_pixels / orig_pixels as f64).sqrt();
+ let (width, height) = size_trunc(img.width as f64 * scale, img.height as f64 * scale);
+ cv::resize_area(ImageRef::U8(img), width, height)?.into_u8()
+}
+
+pub(crate) fn extract_document(
+ input: &ImageU8,
+ quad: &Quad,
+ rotation_degrees: i32,
+ color_mode: ColorMode,
+ max_pixels: f64,
+) -> OpResult<ImageU8> {
+ let estimated = estimate_real_dimensions(quad, input.width, input.height);
+ let (target_width, target_height) = estimated.to_pixel_dimensions(quad);
+
+ let corners = quad.corners();
+ let src_corners = [
+ (corners[0].x, corners[0].y),
+ (corners[1].x, corners[1].y),
+ (corners[2].x, corners[2].y),
+ (corners[3].x, corners[3].y),
+ ];
+ let dst_corners = [
+ (0.0, 0.0),
+ (target_width, 0.0),
+ (target_width, target_height),
+ (0.0, target_height),
+ ];
+
+ let (out_width, out_height) = size_trunc(target_width, target_height);
+ let warped = cv::warp_perspective(input, src_corners, dst_corners, out_width, out_height)?;
+
+ let resized = resize_for_max_pixels(&warped, max_pixels)?;
+ let enhanced = enhance_captured_image(&resized, color_mode)?;
+ cv::rotate_u8(&enhanced, rotation_degrees)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::min_area_rect;
+
+ fn assert_close(a: f64, b: f64, eps: f64) {
+ assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");
+ }
+
+ #[test]
+ fn min_area_rect_of_axis_aligned_box() {
+ let polygon = vec![(10, 20), (60, 20), (60, 90), (10, 90), (35, 55)];
+ let rect = min_area_rect(&polygon, 256, 256).expect("rect");
+ assert_eq!(rect.len(), 4);
+ let xs: Vec<f64> = rect.iter().map(|p| p.x).collect();
+ let ys: Vec<f64> = rect.iter().map(|p| p.y).collect();
+ assert_close(xs.iter().cloned().fold(f64::INFINITY, f64::min), 10.0, 1e-9);
+ assert_close(
+ xs.iter().cloned().fold(f64::NEG_INFINITY, f64::max),
+ 60.0,
+ 1e-9,
+ );
+ assert_close(ys.iter().cloned().fold(f64::INFINITY, f64::min), 20.0, 1e-9);
+ assert_close(
+ ys.iter().cloned().fold(f64::NEG_INFINITY, f64::max),
+ 90.0,
+ 1e-9,
+ );
+ }
+
+ #[test]
+ fn min_area_rect_clips_to_image_bounds() {
+ let polygon = vec![(-30, -30), (500, -30), (500, 500), (-30, 500)];
+ let rect = min_area_rect(&polygon, 256, 256).expect("rect");
+ for p in &rect {
+ assert!((0.0..=255.0).contains(&p.x), "x out of bounds: {p:?}");
+ assert!((0.0..=255.0).contains(&p.y), "y out of bounds: {p:?}");
+ }
+ }
+
+ #[test]
+ fn min_area_rect_rejects_degenerate_input() {
+ assert!(min_area_rect(&[(0, 0), (1, 1)], 256, 256).is_none());
+ assert!(min_area_rect(&[(0, 0), (5, 5), (10, 10)], 256, 256).is_none());
+ }
+}
diff --git a/rust/crates/ml/src/scan/geometry.rs b/rust/crates/ml/src/scan/geometry.rs
new file mode 100644
index 0000000000..50f40b4630
--- /dev/null
+++ b/rust/crates/ml/src/scan/geometry.rs
@@ -0,0 +1,194 @@
+pub(crate) fn average(values: impl Iterator<Item = f64>) -> f64 {
+ let mut sum = 0.0;
+ let mut count = 0usize;
+ for v in values {
+ sum += v;
+ count += 1;
+ }
+ if count == 0 {
+ f64::NAN
+ } else {
+ sum / count as f64
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct Point {
+ pub x: f64,
+ pub y: f64,
+}
+
+impl Point {
+ pub fn new(x: f64, y: f64) -> Self {
+ Self { x, y }
+ }
+
+ pub(crate) fn scaled(&self, scale_x: f64, scale_y: f64) -> Point {
+ Point::new(self.x * scale_x, self.y * scale_y)
+ }
+}
+
+pub(crate) fn norm(p1: Point, p2: Point) -> f64 {
+ (p2.x - p1.x).hypot(p2.y - p1.y)
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ImageSize {
+ pub width: f64,
+ pub height: f64,
+}
+
+impl ImageSize {
+ pub(crate) fn new(width: f64, height: f64) -> Self {
+ Self { width, height }
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct Quad {
+ pub top_left: Point,
+ pub top_right: Point,
+ pub bottom_right: Point,
+ pub bottom_left: Point,
+}
+
+impl Quad {
+ pub fn corners(&self) -> [Point; 4] {
+ [
+ self.top_left,
+ self.top_right,
+ self.bottom_right,
+ self.bottom_left,
+ ]
+ }
+
+ pub(crate) fn edges(&self) -> [(Point, Point); 4] {
+ [
+ (self.top_left, self.top_right),
+ (self.top_right, self.bottom_right),
+ (self.bottom_right, self.bottom_left),
+ (self.bottom_left, self.top_left),
+ ]
+ }
+
+ /// A negative `iterations` falls into the identity branch (truncating
+ /// `%`).
+ pub(crate) fn rotate90(&self, iterations: i32, image_size: ImageSize) -> Quad {
+ let rotate = |p: Point| -> Point {
+ match iterations % 4 {
+ 1 => Point::new(image_size.height - p.y, p.x),
+ 2 => Point::new(image_size.width - p.x, image_size.height - p.y),
+ 3 => Point::new(p.y, image_size.width - p.x),
+ _ => p,
+ }
+ };
+ create_quad(&[
+ rotate(self.top_left),
+ rotate(self.top_right),
+ rotate(self.bottom_right),
+ rotate(self.bottom_left),
+ ])
+ }
+
+ pub(crate) fn scaled_to(
+ &self,
+ from_width: f64,
+ from_height: f64,
+ to_width: f64,
+ to_height: f64,
+ ) -> Quad {
+ let scale_x = to_width / from_width;
+ let scale_y = to_height / from_height;
+ Quad {
+ top_left: self.top_left.scaled(scale_x, scale_y),
+ top_right: self.top_right.scaled(scale_x, scale_y),
+ bottom_right: self.bottom_right.scaled(scale_x, scale_y),
+ bottom_left: self.bottom_left.scaled(scale_x, scale_y),
+ }
+ }
+}
+
+pub(crate) fn create_quad(vertices: &[Point]) -> Quad {
+ assert_eq!(vertices.len(), 4, "create_quad requires exactly 4 vertices");
+ let cx = average(vertices.iter().map(|p| p.x));
+ let cy = average(vertices.iter().map(|p| p.y));
+
+ let mut sorted = vertices.to_vec();
+ sorted.sort_by(|a, b| {
+ let aa = (a.y - cy).atan2(a.x - cx);
+ let bb = (b.y - cy).atan2(b.x - cx);
+ aa.total_cmp(&bb)
+ });
+
+ Quad {
+ top_left: sorted[0],
+ top_right: sorted[1],
+ bottom_right: sorted[2],
+ bottom_left: sorted[3],
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{ImageSize, Point, Quad, create_quad, norm};
+
+ fn assert_close(a: f64, b: f64, eps: f64) {
+ assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");
+ }
+
+ #[test]
+ fn create_quad_orders_corners_by_angle_from_centroid() {
+ // With y down, atan2 ascending starts at the top-left.
+ let vertices = vec![
+ Point::new(100.0, 100.0),
+ Point::new(0.0, 100.0),
+ Point::new(0.0, 0.0),
+ Point::new(100.0, 0.0),
+ ];
+ let quad = create_quad(&vertices);
+ assert_eq!(quad.top_left, Point::new(0.0, 0.0));
+ assert_eq!(quad.top_right, Point::new(100.0, 0.0));
+ assert_eq!(quad.bottom_right, Point::new(100.0, 100.0));
+ assert_eq!(quad.bottom_left, Point::new(0.0, 100.0));
+ }
+
+ #[test]
+ fn create_quad_is_idempotent() {
+ let quad = create_quad(&[
+ Point::new(10.0, 12.0),
+ Point::new(90.0, 8.0),
+ Point::new(95.0, 70.0),
+ Point::new(5.0, 74.0),
+ ]);
+ let again = create_quad(&quad.corners());
+ assert_eq!(quad.top_left, again.top_left);
+ assert_eq!(quad.top_right, again.top_right);
+ assert_eq!(quad.bottom_right, again.bottom_right);
+ assert_eq!(quad.bottom_left, again.bottom_left);
+ }
+
+ #[test]
+ fn scaled_to_maps_mask_space_to_image_space() {
+ let quad = Quad {
+ top_left: Point::new(0.0, 0.0),
+ top_right: Point::new(128.0, 0.0),
+ bottom_right: Point::new(128.0, 256.0),
+ bottom_left: Point::new(0.0, 256.0),
+ };
+ let scaled = quad.scaled_to(256.0, 256.0, 1024.0, 512.0);
+ assert_close(scaled.top_right.x, 512.0, 1e-9);
+ assert_close(scaled.bottom_right.y, 512.0, 1e-9);
+ }
+
+ #[test]
+ fn norm_matches_hypot() {
+ assert_close(norm(Point::new(0.0, 0.0), Point::new(3.0, 4.0)), 5.0, 1e-12);
+ }
+
+ #[test]
+ fn image_size_keeps_doubles() {
+ let size = ImageSize::new(1024.0, 768.0);
+ assert_close(size.width, 1024.0, 0.0);
+ assert_close(size.height, 768.0, 0.0);
+ }
+}
diff --git a/rust/crates/ml/src/scan/mask.rs b/rust/crates/ml/src/scan/mask.rs
new file mode 100644
index 0000000000..286219878e
--- /dev/null
+++ b/rust/crates/ml/src/scan/mask.rs
@@ -0,0 +1,26 @@
+use super::OpResult;
+use crate::cv::image::ImageU8;
+
+pub(crate) struct Mask {
+ pub width: i32,
+ pub height: i32,
+ binary: Vec<u8>,
+}
+
+impl Mask {
+ pub(crate) fn from_probmap(probmap: &[u8], width: i32, height: i32) -> Self {
+ let binary = probmap
+ .iter()
+ .map(|&v| if v >= 128 { 255u8 } else { 0u8 })
+ .collect();
+ Self {
+ width,
+ height,
+ binary,
+ }
+ }
+
+ pub(crate) fn to_image(&self) -> OpResult<ImageU8> {
+ ImageU8::new(self.width, self.height, 1, self.binary.clone())
+ }
+}
diff --git a/rust/crates/ml/src/scan/mod.rs b/rust/crates/ml/src/scan/mod.rs
new file mode 100644
index 0000000000..87e4497c73
--- /dev/null
+++ b/rust/crates/ml/src/scan/mod.rs
@@ -0,0 +1,72 @@
+mod codec;
+mod color;
+mod contour_orientation;
+mod detection;
+mod geometry;
+mod mask;
+mod perspective;
+mod postprocess;
+mod quad_score;
+mod scanner;
+mod segmentation;
+mod yuv;
+
+pub use color::ColorMode;
+pub use geometry::{Point, Quad};
+pub use scanner::{ReprocessOptions, ScanError, ScanOptions, ScanResult, ScannerSession};
+pub use segmentation::MASK_SIDE;
+pub use yuv::PlaneLayout;
+
+use std::path::PathBuf;
+
+use ente_assets::download::CancellationToken;
+use ente_assets::{Asset, AssetFile, AssetStore};
+
+/// Downloads are verified against this before the path reaches
+/// [`ScannerSession::new`].
+pub const SEGMENTATION_MODEL_SHA256: &str =
+ "36b8eeadd42592af496bf2e125a6aad9bebcbca1bda2ac19fa22e108574217a3";
+
+// Temporary dev hosting; will move to models.ente.com.
+const SEGMENTATION_MODEL_URL: &str = "https://entedevassets.priem.dev/document_segmentation.onnx";
+const SEGMENTATION_MODEL_FILE: &str = "document_segmentation.onnx";
+
+fn segmentation_model_asset() -> Asset {
+ Asset::file(
+ vec!["models".to_string(), "document_segmentation".to_string()],
+ AssetFile {
+ name: SEGMENTATION_MODEL_FILE.to_string(),
+ url: SEGMENTATION_MODEL_URL.to_string(),
+ sha256: SEGMENTATION_MODEL_SHA256.to_string(),
+ },
+ )
+ .expect("valid segmentation model asset")
+}
+
+pub async fn ensure_segmentation_model(
+ store: &AssetStore,
+) -> Result<PathBuf, ente_assets::download::Error> {
+ let asset = segmentation_model_asset();
+ if store.is_downloaded(&asset) {
+ log::info!("segmentation model: using cached copy");
+ } else {
+ log::info!("segmentation model: downloading from {SEGMENTATION_MODEL_URL}");
+ let start = std::time::Instant::now();
+ store
+ .download(
+ std::slice::from_ref(&asset),
+ |_| {},
+ CancellationToken::default(),
+ )
+ .await?;
+ log::info!(
+ "segmentation model: downloaded in {}ms",
+ start.elapsed().as_millis()
+ );
+ }
+ Ok(store
+ .file_path(&asset, SEGMENTATION_MODEL_FILE)
+ .expect("segmentation model file"))
+}
+
+pub(crate) use crate::cv::OpResult;
diff --git a/rust/crates/ml/src/scan/perspective.rs b/rust/crates/ml/src/scan/perspective.rs
new file mode 100644
index 0000000000..0033a1bee9
--- /dev/null
+++ b/rust/crates/ml/src/scan/perspective.rs
@@ -0,0 +1,181 @@
+//! Aspect-ratio estimation from the quad's vanishing points.
+
+use super::geometry::{Point, Quad, norm};
+
+#[derive(Clone, Copy, Debug)]
+struct Vector3D {
+ x: f64,
+ y: f64,
+ z: f64,
+}
+
+impl Vector3D {
+ fn new(x: f64, y: f64, z: f64) -> Self {
+ Self { x, y, z }
+ }
+
+ fn sub(self, other: Vector3D) -> Vector3D {
+ Vector3D::new(self.x - other.x, self.y - other.y, self.z - other.z)
+ }
+
+ fn scale(self, t: f64) -> Vector3D {
+ Vector3D::new(self.x * t, self.y * t, self.z * t)
+ }
+
+ fn dot(self, other: Vector3D) -> f64 {
+ self.x * other.x + self.y * other.y + self.z * other.z
+ }
+
+ fn cross(self, other: Vector3D) -> Vector3D {
+ Vector3D::new(
+ self.y * other.z - self.z * other.y,
+ self.z * other.x - self.x * other.z,
+ self.x * other.y - self.y * other.x,
+ )
+ }
+
+ fn norm(self) -> f64 {
+ (self.x * self.x + self.y * self.y + self.z * self.z).sqrt()
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) struct EstimatedDimensions {
+ pub(crate) width: f64,
+ pub(crate) height: f64,
+}
+
+impl EstimatedDimensions {
+ pub(crate) fn to_pixel_dimensions(self, quad: &Quad) -> (f64, f64) {
+ let w =
+ (norm(quad.top_left, quad.top_right) + norm(quad.bottom_left, quad.bottom_right)) / 2.0;
+ let h =
+ (norm(quad.top_left, quad.bottom_left) + norm(quad.top_right, quad.bottom_right)) / 2.0;
+ let projected_area = w * h;
+
+ let ratio = self.height / self.width;
+ let target_width = (projected_area / ratio).sqrt();
+ let target_height = target_width * ratio;
+ (target_width, target_height)
+ }
+}
+
+pub(crate) fn estimate_real_dimensions(
+ quad: &Quad,
+ image_width: i32,
+ image_height: i32,
+) -> EstimatedDimensions {
+ let average_sides = || EstimatedDimensions {
+ width: (norm(quad.top_left, quad.top_right) + norm(quad.bottom_left, quad.bottom_right))
+ / 2.0,
+ height: (norm(quad.top_left, quad.bottom_left) + norm(quad.top_right, quad.bottom_right))
+ / 2.0,
+ };
+
+ let to_h = |p: Point| Vector3D::new(p.x, p.y, 1.0);
+ let line_through = |p1: Point, p2: Point| to_h(p1).cross(to_h(p2));
+
+ let v1h = line_through(quad.top_left, quad.top_right)
+ .cross(line_through(quad.bottom_left, quad.bottom_right));
+ let v2h = line_through(quad.top_left, quad.bottom_left)
+ .cross(line_through(quad.top_right, quad.bottom_right));
+
+ if v1h.z.abs() < 1e-6 || v2h.z.abs() < 1e-6 {
+ return average_sides();
+ }
+
+ let cx = image_width as f64 / 2.0;
+ let cy = image_height as f64 / 2.0;
+
+ let v1 = Point::new(v1h.x / v1h.z - cx, v1h.y / v1h.z - cy);
+ let v2 = Point::new(v2h.x / v2h.z - cx, v2h.y / v2h.z - cy);
+
+ let f2 = -(v1.x * v2.x + v1.y * v2.y);
+ if f2 <= 0.0 {
+ return average_sides();
+ }
+ let f = f2.sqrt();
+
+ if f > (image_width.max(image_height)) as f64 * 1.2 {
+ return average_sides();
+ }
+
+ let d1 = Vector3D::new(v1.x, v1.y, f);
+ let d2 = Vector3D::new(v2.x, v2.y, f);
+ let n = d1.cross(d2);
+
+ let ray = |p: Point| Vector3D::new((p.x - cx) / f, (p.y - cy) / f, 1.0);
+
+ let corner3d = |p: Point| {
+ let r = ray(p);
+ r.scale(1.0 / n.dot(r))
+ };
+
+ let x_tl = corner3d(quad.top_left);
+ let x_tr = corner3d(quad.top_right);
+ let x_br = corner3d(quad.bottom_right);
+ let x_bl = corner3d(quad.bottom_left);
+
+ let real_w = (x_tr.sub(x_tl).norm() + x_br.sub(x_bl).norm()) / 2.0;
+ let real_h = (x_bl.sub(x_tl).norm() + x_br.sub(x_tr).norm()) / 2.0;
+
+ EstimatedDimensions {
+ width: real_w,
+ height: real_h,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{EstimatedDimensions, estimate_real_dimensions};
+ use crate::scan::geometry::{Point, Quad};
+
+ fn assert_close(a: f64, b: f64, eps: f64) {
+ assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");
+ }
+
+ fn rect_quad(w: f64, h: f64) -> Quad {
+ Quad {
+ top_left: Point::new(0.0, 0.0),
+ top_right: Point::new(w, 0.0),
+ bottom_right: Point::new(w, h),
+ bottom_left: Point::new(0.0, h),
+ }
+ }
+
+ #[test]
+ fn estimate_real_dimensions_falls_back_for_parallel_sides() {
+ let quad = rect_quad(400.0, 300.0);
+ let dims = estimate_real_dimensions(&quad, 1000, 800);
+ assert_eq!(
+ dims,
+ EstimatedDimensions {
+ width: 400.0,
+ height: 300.0
+ }
+ );
+ }
+
+ #[test]
+ fn estimate_real_dimensions_falls_back_for_weak_perspective() {
+ // f exceeds max(w,h)*1.2, so the fallback applies.
+ let quad = Quad {
+ top_left: Point::new(100.0, 100.0),
+ top_right: Point::new(900.0, 100.0),
+ bottom_right: Point::new(899.0, 700.0),
+ bottom_left: Point::new(101.0, 700.0),
+ };
+ let dims = estimate_real_dimensions(&quad, 1000, 800);
+ assert_close(dims.width, 799.0, 1e-9);
+ assert_close(dims.height, 600.0008333, 1e-6);
+ }
+
+ #[test]
+ fn to_pixel_dimensions_preserves_area_and_ratio() {
+ let quad = rect_quad(400.0, 300.0);
+ let dims = estimate_real_dimensions(&quad, 1000, 800);
+ let (w, h) = dims.to_pixel_dimensions(&quad);
+ assert_close(w * h, 400.0 * 300.0, 1e-6);
+ assert_close(h / w, 300.0 / 400.0, 1e-12);
+ }
+}
diff --git a/rust/crates/ml/src/scan/postprocess.rs b/rust/crates/ml/src/scan/postprocess.rs
new file mode 100644
index 0000000000..a3453cfe23
--- /dev/null
+++ b/rust/crates/ml/src/scan/postprocess.rs
@@ -0,0 +1,334 @@
+use super::OpResult;
+use super::color::ColorMode;
+use super::detection::size_trunc;
+use crate::cv;
+use crate::cv::image::{ImageF32, ImageRef, ImageU8};
+
+pub(crate) fn enhance_captured_image(img: &ImageU8, color_mode: ColorMode) -> OpResult<ImageU8> {
+ match color_mode {
+ ColorMode::Color => multi_scale_retinex_on_l(img),
+ ColorMode::Grayscale => enhance_grayscale_image(img),
+ }
+}
+
+const SHADOW_MAP_LONG_EDGE: f64 = 256.0;
+const SHADOW_KERNEL_FRAC: f64 = 1.0 / 16.0;
+const SHADOW_MAX_GAIN: f32 = 2.8;
+const SHADOW_PAPER_PERCENTILE: f64 = 0.90;
+
+/// Flat-field illumination correction on a luminance channel: estimate the
+/// background map (closing + blur at map scale), then divide toward the
+/// paper level. Below `paper / SHADOW_MAX_GAIN` the gain ramps back to 1:
+/// regions that much darker than paper are content (photos, figures), not
+/// cast shadows, and must not be brightened.
+fn remove_shadows(channel: &ImageU8) -> OpResult<ImageU8> {
+ let (width, height) = channel.size();
+ let long_edge = width.max(height) as f64;
+ let (map_w, map_h) = if long_edge > SHADOW_MAP_LONG_EDGE {
+ let scale = SHADOW_MAP_LONG_EDGE / long_edge;
+ size_trunc(width as f64 * scale, height as f64 * scale)
+ } else {
+ (width, height)
+ };
+ let small = cv::resize_area(ImageRef::U8(channel), map_w, map_h)?.into_u8()?;
+
+ let k = ((map_w.max(map_h) as f64 * SHADOW_KERNEL_FRAC) as i32).max(3) | 1;
+ let kernel = cv::ellipse_kernel(k)?;
+ let closed = cv::morphology_close(&small, &kernel)?;
+
+ let closed_f = cv::u8_to_f32(&closed)?;
+ let map_small = cv::box_filter_f32(&cv::box_filter_f32(&closed_f, k, k)?, k, k)?;
+
+ let paper = cv::percentile_f32(&map_small, SHADOW_PAPER_PERCENTILE)?;
+ if paper < 1.0 {
+ return Ok(channel.clone());
+ }
+ let map = cv::resize_bilinear(ImageRef::F32(&map_small), width, height)?.into_f32()?;
+
+ let knee = paper / SHADOW_MAX_GAIN;
+ let mut out = ImageU8::zeros(width, height, 1)?;
+ cv::pointwise3(
+ &mut out.data,
+ 1,
+ &channel.data,
+ 1,
+ &map.data,
+ 1,
+ |out, src, map| {
+ for ((o, &v), &m) in out.iter_mut().zip(src).zip(map) {
+ let m = m.max(1.0);
+ let gain = if m >= knee {
+ paper / m
+ } else {
+ 1.0 + (SHADOW_MAX_GAIN - 1.0) * (m / knee)
+ };
+ *o = cv::saturate_u8_f32(v as f32 * gain);
+ }
+ },
+ );
+ Ok(out)
+}
+
+fn multi_scale_retinex_on_l(bgr: &ImageU8) -> OpResult<ImageU8> {
+ let lab = cv::bgr_to_lab(bgr)?;
+ let mut lab_channels = cv::split_u8(&lab)?;
+ let l = remove_shadows(&lab_channels[0])?;
+
+ let l_float_raw = cv::u8_to_f32(&l)?;
+ let l_float = cv::add_f32_scalar(&l_float_raw, 1.0)?;
+
+ let scale_factor = 2.0;
+ // `max_dim_small` deliberately uses the untruncated sizes.
+ let small_w = l_float.width as f64 / scale_factor;
+ let small_h = l_float.height as f64 / scale_factor;
+ let (small_cols, small_rows) = size_trunc(small_w, small_h);
+
+ let l_small = cv::resize_area(ImageRef::F32(&l_float), small_cols, small_rows)?.into_f32()?;
+
+ let log_l_small = cv::log_f32(&l_small)?;
+
+ let max_dim_small = small_w.max(small_h);
+ let kernel_sizes = [
+ max_dim_small / 80.0,
+ max_dim_small / 10.0,
+ max_dim_small / 2.0,
+ ];
+
+ let weight = 1.0 / kernel_sizes.len() as f64;
+ let mut retinex_small = ImageF32::zeros(l_small.width, l_small.height, 1)?;
+
+ for ks in kernel_sizes {
+ let k = (ks as i32).max(3) | 1;
+ let blur_log = cv::box_filter_f32(&log_l_small, k, k)?;
+ let diff = cv::subtract_f32(&log_l_small, &blur_log)?;
+ retinex_small = cv::add_weighted_f32(&retinex_small, 1.0, &diff, weight, 0.0)?;
+ }
+
+ let (min_val, max_val) = cv::min_max_loc_f32(&retinex_small)?;
+ let mut retinex_norm_small = cv::subtract_f32_scalar(&retinex_small, min_val)?;
+
+ let range = max_val - min_val;
+ if range > 1e-6 {
+ retinex_norm_small =
+ cv::multiply_f32_scalar(&retinex_norm_small, [1.0 / range, 0.0, 0.0, 0.0])?;
+ }
+
+ let retinex_norm = cv::resize_bicubic(
+ ImageRef::F32(&retinex_norm_small),
+ l_float.width,
+ l_float.height,
+ )?
+ .into_f32()?;
+
+ let l_original_float = cv::u8_to_f32(&l)?;
+
+ let mean_l = cv::mean_f32(&l_original_float)?;
+ let amplitude = 60.0;
+
+ let corrected_l = cv::multiply_f32_scalar(&retinex_norm, [amplitude, 0.0, 0.0, 0.0])?;
+ let corrected_l = cv::add_f32_scalar(&corrected_l, mean_l - amplitude / 2.0)?;
+
+ let alpha = 0.6;
+ let corrected_l =
+ cv::add_weighted_f32(&l_original_float, 1.0 - alpha, &corrected_l, alpha, 0.0)?;
+
+ let p_low_orig = percentile_l(&l_original_float, 0.001)?;
+ let p_low = percentile_l(&corrected_l, 0.001)?;
+ let p_high = percentile_l(&corrected_l, 0.995)?;
+
+ let target_low = p_low.min(p_low_orig);
+ let target_high = 245.0;
+ let scale = (target_high - target_low) / (p_high - p_low + 1e-6);
+
+ let shifted = cv::subtract_f32_scalar(&corrected_l, p_low)?;
+ let stretched = cv::multiply_f32_scalar(&shifted, [scale, 0.0, 0.0, 0.0])?;
+ let offset = cv::add_f32_scalar(&stretched, target_low)?;
+
+ let clamped_high = cv::min_f32_scalar(&offset, 255.0)?;
+ let clamped = cv::max_f32_scalar(&clamped_high, 0.0)?;
+
+ lab_channels[0] = cv::f32_to_u8(&clamped)?;
+
+ let merged = cv::merge_u8(&lab_channels)?;
+ cv::lab_to_bgr(&merged)
+}
+
+/// Values outside [0,256) are not counted, but the denominator stays the
+/// full pixel count.
+pub(crate) fn percentile_l(l: &ImageF32, p: f64) -> OpResult<f64> {
+ let hist = cv::hist_256_f32(l)?;
+ let total = l.pixels() as f64;
+ let mut sum = 0.0;
+ for (i, count) in hist.iter().enumerate() {
+ sum += count;
+ if sum / total >= p {
+ return Ok(i as f64);
+ }
+ }
+ Ok(255.0)
+}
+
+fn enhance_grayscale_image(img: &ImageU8) -> OpResult<ImageU8> {
+ let gray = match img.channels {
+ 3 => cv::bgr_to_gray(img)?,
+ 1 => img.clone(),
+ other => {
+ return Err(format!(
+ "grayscale enhancement expects a 1- or 3-channel image, got {other}"
+ ));
+ }
+ };
+ let gray = remove_shadows(&gray)?;
+
+ let max_dim = gray.width.max(gray.height) as f64;
+
+ let img_float_raw = cv::u8_to_f32(&gray)?;
+ let img_float = cv::add_f32_scalar(&img_float_raw, 1.0)?;
+ let log_img = cv::log_f32(&img_float)?;
+
+ // Unlike the color path, no odd-kernel forcing.
+ let kernel_sizes = [max_dim / 6.0, max_dim / 50.0];
+ let weight = 1.0 / kernel_sizes.len() as f64;
+ let mut retinex = ImageF32::zeros(gray.width, gray.height, 1)?;
+
+ for kernel_size in kernel_sizes {
+ // Unlike the color path, blurs the linear image and logs afterwards.
+ let (kw, kh) = size_trunc(kernel_size, kernel_size);
+ let blur_raw = cv::box_filter_f32(&img_float, kw, kh)?;
+ let blur = cv::add_f32_scalar(&blur_raw, 1.0)?;
+ let log_blur = cv::log_f32(&blur)?;
+ let diff = cv::subtract_f32(&log_img, &log_blur)?;
+ retinex = cv::add_weighted_f32(&retinex, 1.0, &diff, weight, 0.0)?;
+ }
+
+ let retinex_exp = cv::exp_f32(&retinex)?;
+
+ let p_low = cv::percentile_f32(&retinex_exp, 0.004)? as f64;
+ let p_high = cv::percentile_f32(&retinex_exp, 0.99)? as f64;
+
+ let normalized = cv::subtract_f32_scalar(&retinex_exp, p_low)?;
+ let scale = if p_high > p_low {
+ 255.0 / (p_high - p_low)
+ } else {
+ 1.0
+ };
+ let scaled = cv::multiply_f32_scalar(&normalized, [scale, 0.0, 0.0, 0.0])?;
+ let clamped_high = cv::min_f32_scalar(&scaled, 255.0)?;
+ let clamped = cv::max_f32_scalar(&clamped_high, 0.0)?;
+
+ let result8u = cv::f32_to_u8(&clamped)?;
+
+ // The mode over [180,255] estimates the background level.
+ let hist = cv::hist_256_u8(&result8u)?;
+ let mut mode_val = 220usize;
+ let mut mode_count = 0.0f64;
+ for (i, &c) in hist.iter().enumerate().take(256).skip(180) {
+ if c > mode_count {
+ mode_count = c;
+ mode_val = i;
+ }
+ }
+
+ let stretched8u = if mode_val >= 254 {
+ // Retinex over-amplified; normalize the original grayscale instead.
+ let gray_f = cv::u8_to_f32(&gray)?;
+ let g_low = cv::percentile_f32(&gray_f, 0.01)? as f64;
+ let g_high = cv::percentile_f32(&gray_f, 0.99)? as f64;
+
+ let shifted = cv::subtract_f32_scalar(&gray_f, g_low)?;
+ let scaled =
+ cv::multiply_f32_scalar(&shifted, [255.0 / (g_high - g_low + 1e-6), 0.0, 0.0, 0.0])?;
+ let min_clamped = cv::min_f32_scalar(&scaled, 255.0)?;
+ let max_clamped = cv::max_f32_scalar(&min_clamped, 0.0)?;
+ cv::f32_to_u8(&max_clamped)?
+ } else {
+ let stretched_f = cv::u8_to_f32(&result8u)?;
+ let multiplied =
+ cv::multiply_f32_scalar(&stretched_f, [255.0 / mode_val as f64, 0.0, 0.0, 0.0])?;
+ let min_clamped = cv::min_f32_scalar(&multiplied, 255.0)?;
+ cv::f32_to_u8(&min_clamped)?
+ };
+
+ let denoised = cv::bilateral_filter_u8(&stretched8u, 9, 20.0, 10.0)?;
+ cv::gray_to_bgr(&denoised)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::remove_shadows;
+ use crate::cv::image::ImageU8;
+
+ fn background_mean(img: &ImageU8, x0: i32, x1: i32) -> f64 {
+ let mut sum = 0.0;
+ let mut count = 0.0;
+ for y in 20..img.height - 20 {
+ for x in x0..x1 {
+ if x % 17 != 3 || y % 11 != 5 {
+ sum += img.data[(y * img.width + x) as usize] as f64;
+ count += 1.0;
+ }
+ }
+ }
+ sum / count
+ }
+
+ #[test]
+ fn remove_shadows_lifts_shadowed_paper_and_keeps_text_dark() {
+ let (w, h) = (400, 300);
+ let mut data = vec![230u8; (w * h) as usize];
+ for y in 0..h {
+ for x in 0..w {
+ let i = (y * w + x) as usize;
+ if x % 17 == 3 && y % 11 == 5 {
+ data[i] = 40;
+ }
+ if x < w / 2 {
+ data[i] = (data[i] as f64 * 0.55) as u8;
+ }
+ }
+ }
+ let img = ImageU8::new(w, h, 1, data).unwrap();
+ let out = remove_shadows(&img).unwrap();
+
+ let shadow_bg = background_mean(&out, 20, w / 4);
+ let lit_bg = background_mean(&out, 3 * w / 4, w - 20);
+ assert!(
+ (shadow_bg - lit_bg).abs() < 12.0,
+ "shadowed paper {shadow_bg:.1} should match lit paper {lit_bg:.1}"
+ );
+
+ let mut min_shadow_half = 255u8;
+ for y in 20..h - 20 {
+ for x in 20..w / 4 {
+ min_shadow_half = min_shadow_half.min(out.data[(y * w + x) as usize]);
+ }
+ }
+ assert!(
+ min_shadow_half < 90,
+ "text in the lifted shadow should stay dark, got {min_shadow_half}"
+ );
+ }
+
+ #[test]
+ fn remove_shadows_is_near_identity_on_evenly_lit_paper() {
+ let (w, h) = (400, 300);
+ let mut data = vec![225u8; (w * h) as usize];
+ for y in 0..h {
+ for x in 0..w {
+ if x % 17 == 3 && y % 11 == 5 {
+ data[(y * w + x) as usize] = 40;
+ }
+ }
+ }
+ let img = ImageU8::new(w, h, 1, data.clone()).unwrap();
+ let out = remove_shadows(&img).unwrap();
+ let mean_delta: f64 = out
+ .data
+ .iter()
+ .zip(&data)
+ .map(|(&a, &b)| (a as f64 - b as f64).abs())
+ .sum::<f64>()
+ / data.len() as f64;
+ assert!(mean_delta < 4.0, "expected near-identity, got {mean_delta}");
+ }
+}
diff --git a/rust/crates/ml/src/scan/quad_score.rs b/rust/crates/ml/src/scan/quad_score.rs
new file mode 100644
index 0000000000..dd2c62141d
--- /dev/null
+++ b/rust/crates/ml/src/scan/quad_score.rs
@@ -0,0 +1,32 @@
+use super::OpResult;
+use super::geometry::Point;
+use crate::cv;
+use crate::cv::image::{ImageF32, ImageU8};
+
+pub(crate) fn make_polygon_mask(width: i32, height: i32, polygon: &[Point]) -> OpResult<ImageU8> {
+ let pts: Vec<(i32, i32)> = polygon.iter().map(|p| (p.x as i32, p.y as i32)).collect();
+ cv::fill_poly(width, height, &pts, 1.0)
+}
+
+/// `probmap` values are 0..255 (the binarized mask as f32), not normalized
+/// probabilities.
+pub(crate) fn score_quad_against_probmap(
+ quad: &[Point],
+ probmap: &ImageF32,
+ min_quad_area_ratio: f64,
+) -> OpResult<f64> {
+ let mask = make_polygon_mask(probmap.width, probmap.height, quad)?;
+ let mask_float = cv::u8_to_f32(&mask)?;
+ let masked = cv::multiply_f32(probmap, &mask_float)?;
+
+ let sum_masked = cv::sum_f32(&masked)?;
+ let sum_mask = cv::sum_f32(&mask_float)?;
+ let mean_prob = sum_masked / sum_mask;
+ let area_ratio = sum_mask / (probmap.height * probmap.width) as f64;
+
+ Ok(if area_ratio < min_quad_area_ratio {
+ 0.0
+ } else {
+ mean_prob * (0.7 + 0.3 * area_ratio)
+ })
+}
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rs
new file mode 100644
index 0000000000..28750f1360
--- /dev/null
+++ b/rust/crates/ml/src/scan/scanner.rs
@@ -0,0 +1,274 @@
+use thiserror::Error;
+
+use super::codec;
+use super::color::{ColorMode, auto_color_mode, quad_to_image};
+use super::detection::{detect_document_quad, extract_document, resize_for_max_pixels};
+use super::geometry::{ImageSize, Quad};
+use super::mask::Mask;
+use super::segmentation::{MASK_SIDE, Segmenter};
+use super::yuv::{PlaneLayout, rgba_to_bgr, yuv420_to_bgr};
+use crate::cv;
+use crate::cv::image::ImageU8;
+
+const DEFAULT_MAX_PIXELS: u32 = 2_000_000;
+const DEFAULT_JPEG_QUALITY: u8 = 75;
+
+#[derive(Debug, Clone, PartialEq, Eq, Error)]
+pub enum ScanError {
+ #[error("invalid input: {0}")]
+ InvalidInput(String),
+ #[error("model load: {0}")]
+ ModelLoad(String),
+ #[error("codec: {0}")]
+ Codec(String),
+ #[error("pipeline: {0}")]
+ Pipeline(String),
+}
+
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct ScanOptions {
+ pub color_mode_override: Option<ColorMode>,
+ pub max_pixels: Option<u32>,
+ /// Must be a multiple of 90.
+ pub rotation_degrees: i32,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct ReprocessOptions {
+ /// Same coordinate space as [`ScanResult::quad`]: the decoded source
+ /// image.
+ pub quad: Quad,
+ /// Must be a multiple of 90.
+ pub rotation_degrees: i32,
+ pub color_mode: ColorMode,
+ pub max_pixels: Option<u32>,
+ pub jpeg_quality: Option<u8>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct ScanResult {
+ /// In decoded-source coordinates (EXIF applied, before
+ /// `rotation_degrees`); `None` when nothing was detected and the whole
+ /// frame was returned.
+ pub quad: Option<Quad>,
+ pub color_mode: ColorMode,
+ pub output_width: u32,
+ pub output_height: u32,
+ /// The size [`ScanResult::quad`] is relative to.
+ pub source_width: u32,
+ pub source_height: u32,
+ /// JPEG-encoded.
+ pub processed_image: Vec<u8>,
+}
+
+pub struct ScannerSession {
+ segmenter: Segmenter,
+}
+
+impl ScannerSession {
+ /// The caller is expected to have verified the model file against
+ /// [`SEGMENTATION_MODEL_SHA256`](super::SEGMENTATION_MODEL_SHA256).
+ pub fn new(model_path: &str) -> Result<Self, ScanError> {
+ let start = std::time::Instant::now();
+ let result = Self::load(model_path);
+ match &result {
+ Ok(_) => log::info!(
+ "session ready: {} loaded in {}ms",
+ basename(model_path),
+ start.elapsed().as_millis()
+ ),
+ Err(error) => log::error!("session init failed: {error}"),
+ }
+ result
+ }
+
+ fn load(model_path: &str) -> Result<Self, ScanError> {
+ if !std::path::Path::new(model_path).is_file() {
+ return Err(ScanError::ModelLoad(format!(
+ "no model file at {model_path}"
+ )));
+ }
+ Ok(Self {
+ segmenter: Segmenter::new(model_path)?,
+ })
+ }
+
+ fn segment(&self, bgr: &ImageU8) -> Result<Mask, ScanError> {
+ let probmap = self
+ .segmenter
+ .probability_map_u8(bgr)
+ .map_err(ScanError::Pipeline)?;
+ Ok(Mask::from_probmap(&probmap, MASK_SIDE, MASK_SIDE))
+ }
+
+ /// The returned quad is in mask space (256x256), already rotated by
+ /// `rotation_degrees`: the caller rotates the overlay quad, not the
+ /// frame.
+ pub fn live_detect_rgba(
+ &self,
+ rgba: &[u8],
+ width: u32,
+ height: u32,
+ rotation_degrees: i32,
+ ) -> Result<Option<Quad>, ScanError> {
+ let bgr =
+ rgba_to_bgr(rgba, to_i32(width)?, to_i32(height)?).map_err(ScanError::InvalidInput)?;
+ self.live_detect(&bgr, rotation_degrees)
+ }
+
+ /// Quad semantics as in [`Self::live_detect_rgba`].
+ pub fn live_detect_yuv420(
+ &self,
+ y: &[u8],
+ u: &[u8],
+ v: &[u8],
+ layout: PlaneLayout,
+ rotation_degrees: i32,
+ ) -> Result<Option<Quad>, ScanError> {
+ let bgr = yuv420_to_bgr(y, u, v, layout).map_err(ScanError::InvalidInput)?;
+ self.live_detect(&bgr, rotation_degrees)
+ }
+
+ fn live_detect(&self, bgr: &ImageU8, rotation_degrees: i32) -> Result<Option<Quad>, ScanError> {
+ let mask = self.segment(bgr)?;
+ let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);
+ let quad_in_mask =
+ detect_document_quad(&mask, original_size, true).map_err(ScanError::Pipeline)?;
+ let mask_size = ImageSize::new(mask.width as f64, mask.height as f64);
+ Ok(quad_in_mask.map(|q| q.rotate90(rotation_degrees / 90, mask_size)))
+ }
+
+ pub fn process_capture(
+ &self,
+ image_bytes: &[u8],
+ options: &ScanOptions,
+ ) -> Result<ScanResult, ScanError> {
+ let start = std::time::Instant::now();
+ let result = self.process_capture_inner(image_bytes, options);
+ match &result {
+ Ok(scan) => log::info!(
+ "capture: {}x{} -> {}x{} {:?}, quad {}, {}ms",
+ scan.source_width,
+ scan.source_height,
+ scan.output_width,
+ scan.output_height,
+ scan.color_mode,
+ if scan.quad.is_some() { "found" } else { "none" },
+ start.elapsed().as_millis()
+ ),
+ Err(error) => log::error!("capture failed: {error}"),
+ }
+ result
+ }
+
+ fn process_capture_inner(
+ &self,
+ image_bytes: &[u8],
+ options: &ScanOptions,
+ ) -> Result<ScanResult, ScanError> {
+ let bgr = codec::decode_bgr(image_bytes)?;
+ let max_pixels = options.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS) as f64;
+
+ let mask = self.segment(&bgr)?;
+ let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);
+ let quad_in_mask =
+ detect_document_quad(&mask, original_size, false).map_err(ScanError::Pipeline)?;
+
+ let Some(quad_in_mask) = quad_in_mask else {
+ let resized = resize_for_max_pixels(&bgr, max_pixels).map_err(ScanError::Pipeline)?;
+ let page =
+ cv::rotate_u8(&resized, options.rotation_degrees).map_err(ScanError::Pipeline)?;
+ return finish(None, ColorMode::Color, &bgr, &page, DEFAULT_JPEG_QUALITY);
+ };
+
+ let quad = quad_to_image(&quad_in_mask, &mask, bgr.size());
+ let auto = auto_color_mode(&bgr, &mask, &quad).map_err(ScanError::Pipeline)?;
+ let color_mode = options.color_mode_override.unwrap_or(auto);
+ let page = extract_document(
+ &bgr,
+ &quad,
+ options.rotation_degrees,
+ color_mode,
+ max_pixels,
+ )
+ .map_err(ScanError::Pipeline)?;
+
+ finish(Some(quad), color_mode, &bgr, &page, DEFAULT_JPEG_QUALITY)
+ }
+
+ /// No inference runs.
+ pub fn reprocess(
+ &self,
+ source_bytes: &[u8],
+ options: &ReprocessOptions,
+ ) -> Result<ScanResult, ScanError> {
+ let start = std::time::Instant::now();
+ let result = self.reprocess_inner(source_bytes, options);
+ match &result {
+ Ok(scan) => log::info!(
+ "reprocess: -> {}x{} in {}ms",
+ scan.output_width,
+ scan.output_height,
+ start.elapsed().as_millis()
+ ),
+ Err(error) => log::error!("reprocess failed: {error}"),
+ }
+ result
+ }
+
+ fn reprocess_inner(
+ &self,
+ source_bytes: &[u8],
+ options: &ReprocessOptions,
+ ) -> Result<ScanResult, ScanError> {
+ let bgr = codec::decode_bgr(source_bytes)?;
+ let page = extract_document(
+ &bgr,
+ &options.quad,
+ options.rotation_degrees,
+ options.color_mode,
+ options.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS) as f64,
+ )
+ .map_err(ScanError::Pipeline)?;
+
+ finish(
+ Some(options.quad),
+ options.color_mode,
+ &bgr,
+ &page,
+ options.jpeg_quality.unwrap_or(DEFAULT_JPEG_QUALITY),
+ )
+ }
+}
+
+fn basename(path: &str) -> &str {
+ std::path::Path::new(path)
+ .file_name()
+ .and_then(|name| name.to_str())
+ .unwrap_or(path)
+}
+
+fn to_i32(value: u32) -> Result<i32, ScanError> {
+ i32::try_from(value)
+ .map_err(|_| ScanError::InvalidInput(format!("{value} does not fit in i32")))
+}
+
+fn finish(
+ quad: Option<Quad>,
+ color_mode: ColorMode,
+ source: &ImageU8,
+ page: &ImageU8,
+ jpeg_quality: u8,
+) -> Result<ScanResult, ScanError> {
+ let processed_image = codec::encode_jpeg(page, jpeg_quality)?;
+
+ Ok(ScanResult {
+ quad,
+ color_mode,
+ output_width: page.width as u32,
+ output_height: page.height as u32,
+ source_width: source.width as u32,
+ source_height: source.height as u32,
+ processed_image,
+ })
+}
diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rs
new file mode 100644
index 0000000000..afdffa63e4
--- /dev/null
+++ b/rust/crates/ml/src/scan/segmentation.rs
@@ -0,0 +1,85 @@
+use std::sync::Mutex;
+
+use ort::ep::CPU;
+use ort::session::Session;
+use ort::session::builder::GraphOptimizationLevel;
+use ort::value::TensorRef;
+
+use super::OpResult;
+use super::scanner::ScanError;
+use crate::cv;
+use crate::cv::image::{ImageRef, ImageU8};
+
+pub const MASK_SIDE: i32 = 256;
+
+pub(crate) struct Segmenter {
+ // `Session::run` needs `&mut self`.
+ session: Mutex<Session>,
+}
+
+impl Segmenter {
+ /// CPU-only on purpose: the model is tiny and the thresholded downstream
+ /// pipeline needs deterministic output across devices.
+ pub(crate) fn new(model_path: &str) -> Result<Self, ScanError> {
+ let build = || -> Result<Session, ort::Error> {
+ Session::builder()?
+ .with_optimization_level(GraphOptimizationLevel::All)?
+ .with_intra_threads(1)?
+ .with_inter_threads(1)?
+ .with_execution_providers([CPU::default().with_arena_allocator(true).build()])?
+ .commit_from_file(model_path)
+ };
+ let session = build().map_err(|err| ScanError::ModelLoad(err.to_string()))?;
+ if session.inputs().len() != 1 {
+ return Err(ScanError::ModelLoad(format!(
+ "expected exactly 1 model input, got {}",
+ session.inputs().len()
+ )));
+ }
+ Ok(Self {
+ session: Mutex::new(session),
+ })
+ }
+
+ pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {
+ let rgb = cv::bgr_to_rgb(bgr)?;
+ let resized = cv::resize_bilinear(ImageRef::U8(&rgb), MASK_SIDE, MASK_SIDE)?.into_u8()?;
+
+ let input: Vec<f32> = resized
+ .data
+ .iter()
+ .map(|&v| (v as f32 - 127.5) / 127.5)
+ .collect();
+
+ let side = MASK_SIDE as i64;
+ let mut session = match self.session.lock() {
+ Ok(guard) => guard,
+ Err(poisoned) => poisoned.into_inner(),
+ };
+ let mut run = || -> Result<Vec<f32>, String> {
+ let tensor = TensorRef::<f32>::from_array_view(([1i64, side, side, 3], &input[..]))
+ .map_err(|err| err.to_string())?;
+ let outputs = session
+ .run(ort::inputs![tensor])
+ .map_err(|err| err.to_string())?;
+ if outputs.len() == 0 {
+ return Err("model produced no outputs".to_string());
+ }
+ let (shape, data) = outputs[0]
+ .try_extract_tensor::<f32>()
+ .map_err(|err| err.to_string())?;
+ let expected = (MASK_SIDE * MASK_SIDE) as usize;
+ if data.len() != expected {
+ return Err(format!("unexpected model output shape {shape:?}"));
+ }
+ Ok(data.to_vec())
+ };
+ let data = run()?;
+
+ // The model outputs probabilities directly; no sigmoid.
+ Ok(data
+ .iter()
+ .map(|&v| (v.clamp(0.0, 1.0) * 255.0).round_ties_even() as u8)
+ .collect())
+ }
+}
diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rs
new file mode 100644
index 0000000000..d8351454d3
--- /dev/null
+++ b/rust/crates/ml/src/scan/yuv.rs
@@ -0,0 +1,462 @@
+//! YUV_420_888 camera planes -> packed BGR.
+//!
+//! Reproduces libyuv's limited-range BT.601 fixed-point arithmetic β€” not the
+//! textbook full-range matrix β€” because that is what Android camera stacks
+//! apply when converting frames to RGBA, so YUV and RGBA frames yield the
+//! same quad.
+
+use crate::cv::image::ImageU8;
+
+// Q6 fixed-point BT.601 limited-range coefficients.
+const YG: i32 = 18997; // round(1.164 * 64 * 256 * 256 / 257)
+const YGB: i32 = -1160; // 1.164 * 64 * -16 + 64 * 0.5 (rounding bias folded in)
+const UB: i32 = -128; // max(-128, round(-2.018 * 64)): clamped to fit a signed byte
+const UG: i32 = 25; // round(0.391 * 64)
+const VG: i32 = 52; // round(0.813 * 64)
+const VR: i32 = -102; // round(-1.596 * 64)
+
+// Bias terms; fold in the -128 U/V offset.
+const BB: i32 = UB * 128 + YGB;
+const BG: i32 = UG * 128 + VG * 128 + YGB;
+const BR: i32 = VR * 128 + YGB;
+
+fn ceil_half(value: i32) -> i32 {
+ (value + 1) / 2
+}
+
+fn clamp_u8(value: i32) -> u8 {
+ value.clamp(0, 255) as u8
+}
+
+#[inline]
+pub(crate) fn yuv_to_bgr(y: u8, u: u8, v: u8) -> (u8, u8, u8) {
+ let (y, u, v) = (y as i32, u as i32, v as i32);
+ // 0x0101 replicates 8 bits to 16, keeping the luma multiply in 32 bits.
+ let y1 = ((y as u32 * 0x0101 * YG as u32) >> 16) as i32;
+ let b = clamp_u8((-(u * UB) + y1 + BB) >> 6);
+ let g = clamp_u8((-(u * UG + v * VG) + y1 + BG) >> 6);
+ let r = clamp_u8((-(v * VR) + y1 + BR) >> 6);
+ (b, g, r)
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct PlaneLayout {
+ pub width: i32,
+ pub height: i32,
+ pub y_row_stride: i32,
+ pub uv_row_stride: i32,
+ pub uv_pixel_stride: i32,
+}
+
+fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> usize {
+ if rows <= 0 {
+ return 0;
+ }
+ (rows - 1) as usize * row_stride as usize + last_row_bytes as usize
+}
+
+pub(crate) fn yuv420_to_bgr(
+ y_plane: &[u8],
+ u_plane: &[u8],
+ v_plane: &[u8],
+ layout: PlaneLayout,
+) -> Result<ImageU8, String> {
+ let PlaneLayout {
+ width,
+ height,
+ y_row_stride,
+ uv_row_stride,
+ uv_pixel_stride,
+ } = layout;
+
+ if width <= 0 || height <= 0 {
+ return Err(format!("invalid frame size {width}x{height}"));
+ }
+ if uv_pixel_stride < 1 {
+ return Err(format!("invalid uv pixel stride {uv_pixel_stride}"));
+ }
+ if y_row_stride < width {
+ return Err(format!(
+ "y row stride {y_row_stride} is smaller than the frame width {width}"
+ ));
+ }
+ let cw = ceil_half(width);
+ let ch = ceil_half(height);
+ if uv_row_stride < (cw - 1) * uv_pixel_stride + 1 {
+ return Err(format!(
+ "uv row stride {uv_row_stride} is too small for {cw} chroma samples \
+ at pixel stride {uv_pixel_stride}"
+ ));
+ }
+
+ let need_y = required_len(height, y_row_stride, width);
+ let need_uv = required_len(ch, uv_row_stride, (cw - 1) * uv_pixel_stride + 1);
+ if y_plane.len() < need_y {
+ return Err(format!(
+ "y plane has {} bytes, needs {need_y}",
+ y_plane.len()
+ ));
+ }
+ if u_plane.len() < need_uv {
+ return Err(format!(
+ "u plane has {} bytes, needs {need_uv}",
+ u_plane.len()
+ ));
+ }
+ if v_plane.len() < need_uv {
+ return Err(format!(
+ "v plane has {} bytes, needs {need_uv}",
+ v_plane.len()
+ ));
+ }
+
+ let mut out = vec![0u8; width as usize * height as usize * 3];
+ for row in 0..height {
+ let y_base = row as usize * y_row_stride as usize;
+ let uv_base = (row / 2) as usize * uv_row_stride as usize;
+ let out_base = row as usize * width as usize * 3;
+ for col in 0..width {
+ let uv_index = uv_base + (col / 2) as usize * uv_pixel_stride as usize;
+ let (b, g, r) = yuv_to_bgr(
+ y_plane[y_base + col as usize],
+ u_plane[uv_index],
+ v_plane[uv_index],
+ );
+ let o = out_base + col as usize * 3;
+ out[o] = b;
+ out[o + 1] = g;
+ out[o + 2] = r;
+ }
+ }
+
+ ImageU8::new(width, height, 3, out)
+}
+
+pub(crate) fn rgba_to_bgr(rgba: &[u8], width: i32, height: i32) -> Result<ImageU8, String> {
+ if width <= 0 || height <= 0 {
+ return Err(format!("invalid frame size {width}x{height}"));
+ }
+ let expected = width as usize * height as usize * 4;
+ if rgba.len() != expected {
+ return Err(format!(
+ "RGBA buffer has {} bytes, expected {expected} for {width}x{height}",
+ rgba.len()
+ ));
+ }
+ let mut out = vec![0u8; width as usize * height as usize * 3];
+ for (dst, src) in out.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) {
+ dst[0] = src[2];
+ dst[1] = src[1];
+ dst[2] = src[0];
+ }
+ ImageU8::new(width, height, 3, out)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// `ub` is a parameter: libyuv's effective blue coefficient (-2.0, from
+ /// the `UB` clamp) differs from the documented -2.018.
+ fn reference_yuv_to_bgr(y: u8, u: u8, v: u8, ub: f64) -> (u8, u8, u8) {
+ let yf = (y as f64 - 16.0) * 1.164;
+ let uf = u as f64 - 128.0;
+ let vf = v as f64 - 128.0;
+ let r = yf + 1.596 * vf;
+ let g = yf - 0.391 * uf - 0.813 * vf;
+ let b = yf - ub * uf;
+ let to_u8 = |v: f64| v.round().clamp(0.0, 255.0) as u8;
+ (to_u8(b), to_u8(g), to_u8(r))
+ }
+
+ fn worst_deviation(triples: impl Iterator<Item = (u8, u8, u8)>, ub: f64) -> [i32; 3] {
+ let mut worst = [0i32; 3];
+ for (y, u, v) in triples {
+ let got = yuv_to_bgr(y, u, v);
+ let want = reference_yuv_to_bgr(y, u, v, ub);
+ let pairs = [(got.0, want.0), (got.1, want.1), (got.2, want.2)];
+ for (i, (a, b)) in pairs.iter().enumerate() {
+ worst[i] = worst[i].max((*a as i32 - *b as i32).abs());
+ }
+ }
+ worst
+ }
+
+ fn all_triples() -> impl Iterator<Item = (u8, u8, u8)> {
+ (0..=255u8)
+ .flat_map(|y| (0..=255u8).map(move |u| (y, u)))
+ .flat_map(|(y, u)| (0..=255u8).map(move |v| (y, u, v)))
+ }
+
+ fn strided_triples() -> impl Iterator<Item = (u8, u8, u8)> {
+ let axis = |step: usize| (0..=255u8).step_by(step).chain(std::iter::once(255));
+ axis(6).flat_map(move |y| axis(7).flat_map(move |u| axis(11).map(move |v| (y, u, v))))
+ }
+
+ struct Rng(u64);
+
+ impl Rng {
+ fn next_u8(&mut self) -> u8 {
+ let mut x = self.0;
+ x ^= x >> 12;
+ x ^= x << 25;
+ x ^= x >> 27;
+ self.0 = x;
+ (x.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 33) as u8
+ }
+ }
+
+ #[test]
+ fn anchor_values_match_the_reference_matrix() {
+ assert_eq!(yuv_to_bgr(16, 128, 128), (0, 0, 0));
+ assert_eq!(yuv_to_bgr(235, 128, 128), (255, 255, 255));
+ assert_eq!(yuv_to_bgr(0, 128, 128), (0, 0, 0));
+ assert_eq!(yuv_to_bgr(255, 128, 128), (255, 255, 255));
+ // Y=126 is the mid point of the limited range 16..235.
+ let (b, g, r) = yuv_to_bgr(126, 128, 128);
+ assert_eq!((b, g, r), (128, 128, 128));
+ }
+
+ #[test]
+ fn fixed_point_matches_the_effective_matrix_on_a_subsample() {
+ let worst = worst_deviation(strided_triples(), -2.0);
+ assert!(
+ worst[0] == 0 && worst[1] <= 1 && worst[2] <= 1,
+ "B/G/R deviation from the effective matrix changed: {worst:?}"
+ );
+ }
+
+ #[test]
+ #[ignore = "exhaustive 2^24 sweep; run explicitly"]
+ fn fixed_point_matches_the_effective_matrix_exhaustively() {
+ let worst = worst_deviation(all_triples(), -2.0);
+ assert_eq!(
+ worst,
+ [0, 1, 1],
+ "B/G/R deviation from the effective matrix changed"
+ );
+ }
+
+ #[test]
+ #[ignore = "exhaustive 2^24 sweep; run explicitly"]
+ fn documented_matrix_differs_only_in_blue_by_the_ub_clamp() {
+ let worst = worst_deviation(all_triples(), -2.018);
+ assert_eq!(
+ worst,
+ [3, 1, 1],
+ "B/G/R deviation from the documented matrix changed"
+ );
+ }
+
+ #[test]
+ fn grey_ramp_is_monotonic_and_neutral() {
+ let mut previous = 0u8;
+ for y in 16..=235u8 {
+ let (b, g, r) = yuv_to_bgr(y, 128, 128);
+ assert_eq!(b, g, "neutral chroma must stay grey at Y={y}");
+ assert_eq!(g, r, "neutral chroma must stay grey at Y={y}");
+ assert!(b >= previous, "luma ramp must be monotonic at Y={y}");
+ previous = b;
+ }
+ }
+
+ fn planes_from_pixels(
+ width: i32,
+ height: i32,
+ pixel: impl Fn(i32, i32) -> (u8, u8, u8),
+ ) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
+ let cw = ceil_half(width);
+ let ch = ceil_half(height);
+ let mut y = vec![0u8; (width * height) as usize];
+ let mut u = vec![0u8; (cw * ch) as usize];
+ let mut v = vec![0u8; (cw * ch) as usize];
+ for row in 0..height {
+ for col in 0..width {
+ y[(row * width + col) as usize] = pixel(col, row).0;
+ }
+ }
+ for row in 0..ch {
+ for col in 0..cw {
+ let (_, cu, cv) = pixel(col * 2, row * 2);
+ u[(row * cw + col) as usize] = cu;
+ v[(row * cw + col) as usize] = cv;
+ }
+ }
+ (y, u, v)
+ }
+
+ #[test]
+ fn planar_and_interleaved_layouts_agree() {
+ let (width, height) = (37, 21);
+ let mut rng = Rng(0xC0FF_EE00_1234_5677);
+ let samples: Vec<(u8, u8, u8)> = (0..(width * height))
+ .map(|_| (rng.next_u8(), rng.next_u8(), rng.next_u8()))
+ .collect();
+ let pixel = |x: i32, y: i32| samples[(y * width + x) as usize];
+
+ let (y_plane, u_plane, v_plane) = planes_from_pixels(width, height, pixel);
+ let planar = yuv420_to_bgr(
+ &y_plane,
+ &u_plane,
+ &v_plane,
+ PlaneLayout {
+ width,
+ height,
+ y_row_stride: width,
+ uv_row_stride: ceil_half(width),
+ uv_pixel_stride: 1,
+ },
+ )
+ .expect("planar conversion");
+
+ // Android exposes NV21 as two plane views one byte apart into one
+ // interleaved buffer, both with pixel stride 2.
+ let cw = ceil_half(width);
+ let ch = ceil_half(height);
+ let mut vu = vec![0u8; (cw * ch * 2) as usize];
+ for i in 0..(cw * ch) as usize {
+ vu[i * 2] = v_plane[i];
+ vu[i * 2 + 1] = u_plane[i];
+ }
+ let interleaved = yuv420_to_bgr(
+ &y_plane,
+ &vu[1..],
+ &vu,
+ PlaneLayout {
+ width,
+ height,
+ y_row_stride: width,
+ uv_row_stride: cw * 2,
+ uv_pixel_stride: 2,
+ },
+ )
+ .expect("interleaved conversion");
+ assert_eq!(planar, interleaved);
+
+ let mut uv = vec![0u8; (cw * ch * 2) as usize];
+ for i in 0..(cw * ch) as usize {
+ uv[i * 2] = u_plane[i];
+ uv[i * 2 + 1] = v_plane[i];
+ }
+ let nv12 = yuv420_to_bgr(
+ &y_plane,
+ &uv,
+ &uv[1..],
+ PlaneLayout {
+ width,
+ height,
+ y_row_stride: width,
+ uv_row_stride: cw * 2,
+ uv_pixel_stride: 2,
+ },
+ )
+ .expect("nv12 conversion");
+ assert_eq!(planar, nv12);
+ }
+
+ #[test]
+ fn row_padding_is_honoured() {
+ let (width, height) = (10, 6);
+ let mut rng = Rng(0x1234_5678_9ABC_DEF1);
+ let samples: Vec<(u8, u8, u8)> = (0..(width * height))
+ .map(|_| (rng.next_u8(), rng.next_u8(), rng.next_u8()))
+ .collect();
+ let pixel = |x: i32, y: i32| samples[(y * width + x) as usize];
+ let (y_tight, u_tight, v_tight) = planes_from_pixels(width, height, pixel);
+
+ let tight = yuv420_to_bgr(
+ &y_tight,
+ &u_tight,
+ &v_tight,
+ PlaneLayout {
+ width,
+ height,
+ y_row_stride: width,
+ uv_row_stride: ceil_half(width),
+ uv_pixel_stride: 1,
+ },
+ )
+ .expect("tight conversion");
+
+ let (y_stride, uv_stride) = (width + 7, ceil_half(width) + 3);
+ let cw = ceil_half(width);
+ let ch = ceil_half(height);
+ let mut y_pad = vec![0xA5u8; (y_stride * height) as usize];
+ let mut u_pad = vec![0x5Au8; (uv_stride * ch) as usize];
+ let mut v_pad = vec![0x3Cu8; (uv_stride * ch) as usize];
+ for row in 0..height {
+ for col in 0..width {
+ y_pad[(row * y_stride + col) as usize] = y_tight[(row * width + col) as usize];
+ }
+ }
+ for row in 0..ch {
+ for col in 0..cw {
+ u_pad[(row * uv_stride + col) as usize] = u_tight[(row * cw + col) as usize];
+ v_pad[(row * uv_stride + col) as usize] = v_tight[(row * cw + col) as usize];
+ }
+ }
+ let padded = yuv420_to_bgr(
+ &y_pad,
+ &u_pad,
+ &v_pad,
+ PlaneLayout {
+ width,
+ height,
+ y_row_stride: y_stride,
+ uv_row_stride: uv_stride,
+ uv_pixel_stride: 1,
+ },
+ )
+ .expect("padded conversion");
+ assert_eq!(tight, padded);
+ }
+
+ #[test]
+ fn odd_sizes_and_short_planes_are_rejected_not_panicked() {
+ let layout = PlaneLayout {
+ width: 8,
+ height: 8,
+ y_row_stride: 8,
+ uv_row_stride: 4,
+ uv_pixel_stride: 1,
+ };
+ assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout).is_err());
+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout).is_err());
+ assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout).is_err());
+ assert!(
+ yuv420_to_bgr(
+ &[0u8; 64],
+ &[0u8; 16],
+ &[0u8; 16],
+ PlaneLayout {
+ y_row_stride: 4,
+ ..layout
+ }
+ )
+ .is_err()
+ );
+ assert!(
+ yuv420_to_bgr(
+ &[128u8],
+ &[128u8],
+ &[128u8],
+ PlaneLayout {
+ width: 1,
+ height: 1,
+ y_row_stride: 1,
+ uv_row_stride: 1,
+ uv_pixel_stride: 1,
+ }
+ )
+ .is_ok()
+ );
+ }
+
+ #[test]
+ fn rgba_to_bgr_drops_alpha_and_reverses_channels() {
+ let rgba = vec![1u8, 2, 3, 255, 10, 20, 30, 0];
+ let bgr = rgba_to_bgr(&rgba, 2, 1).expect("conversion");
+ assert_eq!(bgr.data, vec![3, 2, 1, 30, 20, 10]);
+ assert!(rgba_to_bgr(&rgba, 3, 1).is_err());
+ }
+}
diff --git a/rust/crates/ml/tests/scan.rs b/rust/crates/ml/tests/scan.rs
new file mode 100644
index 0000000000..2ec0d97109
--- /dev/null
+++ b/rust/crates/ml/tests/scan.rs
@@ -0,0 +1,179 @@
+//! Point `ENTE_DOC_SEGMENTATION_MODEL` at the model file; without it every
+//! test skips.
+
+use std::io::Cursor;
+
+use ente_ml::scan::{ColorMode, PlaneLayout, Quad, ReprocessOptions, ScanOptions, ScannerSession};
+use image::{ImageFormat, Rgb, RgbImage};
+
+const MODEL_ENV: &str = "ENTE_DOC_SEGMENTATION_MODEL";
+
+fn session() -> Option<ScannerSession> {
+ let Ok(path) = std::env::var(MODEL_ENV) else {
+ eprintln!("skipping: {MODEL_ENV} is not set");
+ return None;
+ };
+ Some(ScannerSession::new(&path).expect("model should load"))
+}
+
+fn synthetic_document() -> RgbImage {
+ let (width, height) = (640u32, 480u32);
+ let mut img = RgbImage::from_pixel(width, height, Rgb([40, 45, 50]));
+ let (x0, y0, x1, y1) = (120u32, 80u32, 520u32, 400u32);
+ for y in y0..y1 {
+ for x in x0..x1 {
+ img.put_pixel(x, y, Rgb([235, 233, 228]));
+ }
+ }
+ for line in 0..8u32 {
+ let ly = y0 + 30 + line * 36;
+ for y in ly..ly + 6 {
+ for x in x0 + 30..x1 - 30 {
+ img.put_pixel(x, y, Rgb([60, 60, 65]));
+ }
+ }
+ }
+ img
+}
+
+fn encode(img: &RgbImage, format: ImageFormat) -> Vec<u8> {
+ let mut bytes = Cursor::new(Vec::new());
+ img.write_to(&mut bytes, format).expect("encode");
+ bytes.into_inner()
+}
+
+fn assert_document_result(result: &ente_ml::scan::ScanResult) {
+ let quad = result.quad.expect("the synthetic document should be found");
+ for p in quad.corners() {
+ assert!(p.x >= 0.0 && p.x <= result.source_width as f64, "{quad:?}");
+ assert!(p.y >= 0.0 && p.y <= result.source_height as f64, "{quad:?}");
+ }
+ assert_eq!(result.source_width, 640);
+ assert_eq!(result.source_height, 480);
+ assert!(result.output_width > 100 && result.output_height > 100);
+ assert_jpeg(&result.processed_image);
+}
+
+fn assert_jpeg(bytes: &[u8]) {
+ assert_eq!(
+ image::guess_format(bytes).expect("recognizable output"),
+ ImageFormat::Jpeg
+ );
+}
+
+#[test]
+fn processes_a_png_capture() {
+ let Some(session) = session() else { return };
+ let bytes = encode(&synthetic_document(), ImageFormat::Png);
+
+ let result = session
+ .process_capture(&bytes, &ScanOptions::default())
+ .expect("process_capture");
+ assert_document_result(&result);
+
+ let decoded = image::load_from_memory(&result.processed_image).expect("valid JPEG output");
+ assert_eq!(decoded.width(), result.output_width);
+ assert_eq!(decoded.height(), result.output_height);
+}
+
+#[test]
+fn processes_a_jpeg_capture_and_reprocesses_it() {
+ let Some(session) = session() else { return };
+ let bytes = encode(&synthetic_document(), ImageFormat::Jpeg);
+
+ let result = session
+ .process_capture(&bytes, &ScanOptions::default())
+ .expect("process_capture");
+ assert_document_result(&result);
+ image::load_from_memory(&result.processed_image).expect("valid JPEG output");
+
+ let reprocessed = session
+ .reprocess(
+ &bytes,
+ &ReprocessOptions {
+ quad: result.quad.expect("quad"),
+ rotation_degrees: 90,
+ color_mode: ColorMode::Grayscale,
+ max_pixels: Some(500_000),
+ jpeg_quality: Some(60),
+ },
+ )
+ .expect("reprocess");
+ assert_eq!(reprocessed.color_mode, ColorMode::Grayscale);
+ assert_jpeg(&reprocessed.processed_image);
+ assert!(reprocessed.output_width < reprocessed.output_height);
+ image::load_from_memory(&reprocessed.processed_image).expect("valid JPEG output");
+}
+
+#[test]
+fn live_detect_rgba_and_yuv_agree_on_a_grayscale_frame() {
+ let Some(session) = session() else { return };
+ let doc = synthetic_document();
+ let (width, height) = (doc.width() as i32, doc.height() as i32);
+
+ let mut rgba = Vec::with_capacity((width * height * 4) as usize);
+ let mut y_plane = Vec::with_capacity((width * height) as usize);
+ for px in doc.pixels() {
+ // value = 1.164 * (y - 16), chroma neutral, so the limited-range YUV
+ // round trip is exact.
+ let y = 16 + (px.0[0] as i32 * 219 / 255) as u8;
+ let value = (((y as i32 - 16) * 1164 + 500) / 1000).clamp(0, 255) as u8;
+ rgba.extend_from_slice(&[value, value, value, 255]);
+ y_plane.push(y);
+ }
+ let chroma_len = ((width + 1) / 2 * (height + 1) / 2) as usize;
+ let u_plane = vec![128u8; chroma_len];
+ let v_plane = vec![128u8; chroma_len];
+
+ let from_rgba = session
+ .live_detect_rgba(&rgba, width as u32, height as u32, 0)
+ .expect("live_detect_rgba")
+ .expect("quad from RGBA");
+ let from_yuv = session
+ .live_detect_yuv420(
+ &y_plane,
+ &u_plane,
+ &v_plane,
+ PlaneLayout {
+ width,
+ height,
+ y_row_stride: width,
+ uv_row_stride: (width + 1) / 2,
+ uv_pixel_stride: 1,
+ },
+ 0,
+ )
+ .expect("live_detect_yuv420")
+ .expect("quad from YUV");
+
+ let max_delta = quad_max_delta(&from_rgba, &from_yuv);
+ assert!(
+ max_delta < 2.0,
+ "RGBA and YUV quads diverge by {max_delta} in mask space:\n{from_rgba:?}\n{from_yuv:?}"
+ );
+
+ let rotated = session
+ .live_detect_rgba(&rgba, width as u32, height as u32, 180)
+ .expect("live_detect_rgba rotated")
+ .expect("quad");
+ for p in rotated.corners() {
+ assert!((0.0..=256.0).contains(&p.x) && (0.0..=256.0).contains(&p.y));
+ }
+}
+
+#[test]
+fn rejects_undecodable_bytes_with_a_codec_error() {
+ let Some(session) = session() else { return };
+ let error = session
+ .process_capture(b"not an image", &ScanOptions::default())
+ .expect_err("garbage must not decode");
+ assert!(matches!(error, ente_ml::scan::ScanError::Codec(_)));
+}
+
+fn quad_max_delta(a: &Quad, b: &Quad) -> f64 {
+ a.corners()
+ .iter()
+ .zip(b.corners())
+ .map(|(p, q)| (p.x - q.x).hypot(p.y - q.y))
+ .fold(0.0, f64::max)
+}