Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Stabilize perspective proportions with source edge refinement

laurenspriem4769ac51adae
diff --git a/rust/crates/ml/src/scan/boundary.rs b/rust/crates/ml/src/scan/boundary.rs
index 7630229387..9b8c1f676d 100644
--- a/rust/crates/ml/src/scan/boundary.rs
+++ b/rust/crates/ml/src/scan/boundary.rs
@@ -548,7 +548,21 @@ fn brightness(source: &ImageU8, x: f64, y: f64) -> Option<f64> {
)
}
-fn nearby_color_edge(source: &ImageU8, edge: [Point; 2], band: f64) -> bool {
+struct ColorEdgeScores {
+ values: [[f64; 91]; 48],
+ supported: [bool; 48],
+}
+
+impl ColorEdgeScores {
+ fn supported_positions(&self) -> usize {
+ self.supported
+ .iter()
+ .filter(|&&supported| supported)
+ .count()
+ }
+}
+
+fn color_edge_scores(source: &ImageU8, edge: [Point; 2], band: f64) -> ColorEdgeScores {
let line = Line::through(edge[0], edge[1]);
let sample = |x: f64, y: f64| -> Option<[f64; 3]> {
if x < 0.0 || y < 0.0 || x >= source.width as f64 || y >= source.height as f64 {
@@ -559,57 +573,145 @@ fn nearby_color_edge(source: &ImageU8, edge: [Point; 2], band: f64) -> bool {
source.data[offset + channel] as f64
}))
};
- (0..48)
- .filter(|&i| {
- let t = (i as f64 + 1.0) / 49.0;
- let center = Point {
- x: edge[0].x + (edge[1].x - edge[0].x) * t,
- y: edge[0].y + (edge[1].y - edge[0].y) * t,
+ let mut supported = [false; 48];
+ let values = std::array::from_fn(|i| {
+ let t = (i as f64 + 1.0) / 49.0;
+ let center = Point {
+ x: edge[0].x + (edge[1].x - edge[0].x) * t,
+ y: edge[0].y + (edge[1].y - edge[0].y) * t,
+ };
+ std::array::from_fn(|step| {
+ let offset = (step as f64 - 45.0) * band / 30.0;
+ let Some(a) = sample(
+ center.x + line.nx * (offset - 1.25),
+ center.y + line.ny * (offset - 1.25),
+ ) else {
+ return 0.0;
};
- (-30..=30).any(|step| {
- let offset = step as f64 * band / 30.0;
- let Some(a) = sample(
- center.x + line.nx * (offset - 1.25),
- center.y + line.ny * (offset - 1.25),
- ) else {
- return false;
- };
- let Some(b) = sample(
- center.x + line.nx * (offset + 1.25),
- center.y + line.ny * (offset + 1.25),
- ) else {
- return false;
- };
- let delta = std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]);
- let dot = |a: [f64; 3], b: [f64; 3]| {
- (a[0] * b[0] + 2.0 * a[1] * b[1] + a[2] * b[2]) / 4.0
- };
- let energy = dot(delta, delta);
- let penalty = 1.0 + 0.25 * (offset / band).powi(2);
- if energy < (6.0 * penalty).powi(2) {
- return false;
- }
- [2.5, 4.0, 6.0].into_iter().all(|reach| {
- let Some(a) = sample(
- center.x + line.nx * (offset - reach),
- center.y + line.ny * (offset - reach),
- ) else {
- return false;
- };
- let Some(b) = sample(
- center.x + line.nx * (offset + reach),
- center.y + line.ny * (offset + reach),
- ) else {
- return false;
- };
- let wide = std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]);
- let wide_energy = dot(wide, wide);
- wide_energy >= 36.0 && dot(delta, wide) >= 0.7 * (energy * wide_energy).sqrt()
- })
- })
+ let Some(b) = sample(
+ center.x + line.nx * (offset + 1.25),
+ center.y + line.ny * (offset + 1.25),
+ ) else {
+ return 0.0;
+ };
+ let delta = std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]);
+ let dot =
+ |a: [f64; 3], b: [f64; 3]| (a[0] * b[0] + 2.0 * a[1] * b[1] + a[2] * b[2]) / 4.0;
+ let energy = dot(delta, delta);
+ let penalty = 1.0 + 0.25 * (offset / band).powi(2);
+ if energy < 9.0 {
+ return 0.0;
+ }
+ let Some(inner) = sample(
+ center.x + line.nx * (offset - 0.5),
+ center.y + line.ny * (offset - 0.5),
+ ) else {
+ return 0.0;
+ };
+ let Some(outer) = sample(
+ center.x + line.nx * (offset + 0.5),
+ center.y + line.ny * (offset + 0.5),
+ ) else {
+ return 0.0;
+ };
+ let fine = std::array::from_fn::<_, 3, _>(|i| inner[i] - outer[i]);
+ let wide = [2.5, 4.0, 6.0].map(|reach| {
+ let a = sample(
+ center.x + line.nx * (offset - reach),
+ center.y + line.ny * (offset - reach),
+ )?;
+ let b = sample(
+ center.x + line.nx * (offset + reach),
+ center.y + line.ny * (offset + reach),
+ )?;
+ Some(std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]))
+ });
+ let [Some(a), Some(b), Some(c)] = wide else {
+ return 0.0;
+ };
+ let agrees = |wide: [f64; 3], minimum: f64| {
+ let wide_energy = dot(wide, wide);
+ wide_energy >= minimum && dot(delta, wide) >= 0.7 * (energy * wide_energy).sqrt()
+ };
+ if (15..76).contains(&step)
+ && energy >= (6.0 * penalty).powi(2)
+ && [a, b, c].into_iter().all(|wide| agrees(wide, 36.0))
+ {
+ supported[i] = true;
+ }
+ if [a, b].into_iter().all(|wide| agrees(wide, 9.0)) {
+ (1.0 + dot(fine, fine).sqrt()) / penalty
+ } else {
+ 0.0
+ }
})
- .count()
- >= 32
+ });
+ ColorEdgeScores { values, supported }
+}
+
+fn refine_color_edges(
+ p: [Point; 4],
+ scores: &[ColorEdgeScores; 4],
+ band: f64,
+) -> Option<[Point; 4]> {
+ let mut lines = std::array::from_fn::<_, 4, _>(|i| Line::through(p[i], p[(i + 1) % 4]));
+ for i in 0..4 {
+ let available = scores[i].supported_positions();
+ if available < 32 {
+ return None;
+ }
+ let mut best = None;
+ let mut best_score = 0.0;
+ for start in 0..91 {
+ for end in 0..91 {
+ let mut supported = 0;
+ let mut strength = 0.0;
+ let mut weighted_support = 0.0;
+ for (position, row) in scores[i].values.iter().enumerate() {
+ let t = (position as f64 + 1.0) / 49.0;
+ let offset = (start as f64 * (1.0 - t) + end as f64 * t).round() as usize;
+ let score = row[offset];
+ supported += usize::from(score > 0.0);
+ strength += score / (score + 16.0);
+ if score > 0.0 {
+ weighted_support +=
+ (1.0 + 0.25 * ((offset as f64 - 45.0) / 30.0).powi(2)).recip();
+ }
+ }
+ let score = weighted_support + strength * 0.1;
+ if supported >= 32 && supported * 5 >= available * 4 && score > best_score {
+ best_score = score;
+ best = Some((start, end));
+ }
+ }
+ }
+ let (start, end) = best?;
+ let mut samples = Vec::new();
+ for (position, row) in scores[i].values.iter().enumerate() {
+ let t = (position as f64 + 1.0) / 49.0;
+ let index = (start as f64 * (1.0 - t) + end as f64 * t).round() as usize;
+ if row[index] == 0.0 {
+ continue;
+ }
+ let offset = (index as f64 - 45.0) * band / 30.0;
+ samples.push(Point {
+ x: p[i].x + (p[(i + 1) % 4].x - p[i].x) * t + lines[i].nx * offset,
+ y: p[i].y + (p[(i + 1) % 4].y - p[i].y) * t + lines[i].ny * offset,
+ });
+ }
+ if samples.len() < 32 {
+ return None;
+ }
+ lines[i] = fit_line(&samples, lines[i], band);
+ }
+ let mut refined = p;
+ for i in 0..4 {
+ refined[i] = lines[(i + 3) % 4].intersection(lines[i])?;
+ if distance(refined[i], p[i]) > band * 1.5 {
+ return None;
+ }
+ }
+ Some(refined)
}
pub(super) fn refine_capture(
@@ -715,11 +817,25 @@ pub(super) fn refine_capture(
lines[side] = fitted;
improved += 1;
}
+ let color_samples = std::array::from_fn::<_, 4, _>(|side| {
+ color_edge_scores(&preview, [p[side], p[(side + 1) % 4]], band * 3.0)
+ });
if (supported < 3 || needs_complete_support)
- && !(0..4).all(|side| nearby_color_edge(&preview, [p[side], p[(side + 1) % 4]], band * 3.0))
+ && color_samples
+ .iter()
+ .any(|samples| samples.supported_positions() < 32)
{
return Ok(None);
}
+ if let Some(refined) = refine_color_edges(p, &color_samples, band * 3.0) {
+ let refined = quad_from_points(refined.map(|p| Point {
+ x: p.x / sx,
+ y: p.y / sy,
+ }));
+ if validate_quad(refined, extent).is_ok() {
+ return Ok(Some(refined));
+ }
+ }
if improved == 0 {
return Ok(Some(quad));
}
@@ -1292,6 +1408,69 @@ mod benchmarks {
mod source_evidence_tests {
use super::*;
+ #[test]
+ fn broad_printed_panel_does_not_displace_a_supported_outer_page() -> OpResult<()> {
+ let mut source = ImageU8::new(400, 300, 3, vec![180; 400 * 300 * 3])?;
+ for y in 40..260 {
+ for x in 60..340 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(195);
+ }
+ }
+ for y in 45..255 {
+ for x in 65..335 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(90);
+ }
+ }
+ for y in [100, 150, 200] {
+ for x in 62..65 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(30);
+ }
+ }
+ let quad = quad_from_points(
+ [(60.0, 40.0), (340.0, 40.0), (340.0, 260.0), (60.0, 260.0)]
+ .map(|(x, y)| Point { x, y }),
+ );
+ let refined = refine_capture(&source, quad, false)?.ok_or("page rejected")?;
+ for (actual, expected) in points(refined).into_iter().zip(points(quad)) {
+ assert!(
+ distance(actual, expected) < 2.0,
+ "{actual:?} vs {expected:?}"
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn sustained_chromatic_edges_refine_displaced_page_corners() -> OpResult<()> {
+ let mut source = ImageU8::new(400, 300, 3, [80, 140, 200].repeat(400 * 300))?;
+ let expected = [(80.0, 40.0), (315.0, 60.0), (340.0, 245.0), (60.0, 265.0)]
+ .map(|(x, y)| Point { x, y });
+ for y in 0..300 {
+ for x in 0..400 {
+ let point = Point {
+ x: x as f64,
+ y: y as f64,
+ };
+ if (0..4).all(|i| cross(expected[i], expected[(i + 1) % 4], point) >= 0.0) {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3]
+ .copy_from_slice(&[40, 160, 200]);
+ }
+ }
+ }
+ let displaced = quad_from_points(
+ [(84.0, 45.0), (312.0, 66.0), (344.0, 250.0), (66.0, 260.0)]
+ .map(|(x, y)| Point { x, y }),
+ );
+ let refined = refine_capture(&source, displaced, false)?.ok_or("page rejected")?;
+ for (actual, expected) in points(refined).into_iter().zip(expected) {
+ assert!(
+ distance(actual, expected) < 2.0,
+ "{actual:?} vs {expected:?}"
+ );
+ }
+ Ok(())
+ }
+
#[test]
fn color_edge_rejects_thin_strokes_without_a_background_step() -> OpResult<()> {
let mut source = ImageU8::new(400, 300, 3, vec![220; 400 * 300 * 3])?;
@@ -1299,13 +1478,13 @@ mod source_evidence_tests {
source.data[(100 * 400 + x) * 3..(100 * 400 + x + 1) * 3].fill(30);
}
let edge = [Point { x: 60.0, y: 100.0 }, Point { x: 340.0, y: 100.0 }];
- assert!(!nearby_color_edge(&source, edge, 7.2));
+ assert!(color_edge_scores(&source, edge, 7.2).supported_positions() < 32);
for y in 100..300 {
for x in 0..400 {
source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(150);
}
}
- assert!(nearby_color_edge(&source, edge, 7.2));
+ assert!(color_edge_scores(&source, edge, 7.2).supported_positions() >= 32);
Ok(())
}
diff --git a/rust/crates/ml/src/scan/page.rs b/rust/crates/ml/src/scan/page.rs
index c36d1c0422..893a0406ae 100644
--- a/rust/crates/ml/src/scan/page.rs
+++ b/rust/crates/ml/src/scan/page.rs
@@ -135,6 +135,30 @@ pub(super) struct RenderPlan {
pub aspect_evidence: AspectEvidence,
}
+struct CameraAspect {
+ ratio: f64,
+ focal_squared: f64,
+}
+
+fn camera_aspect(p: [(f64, f64); 4]) -> Option<CameraAspect> {
+ let h = ProjectiveMap::from_unit_quad(p).ok()?.coefficients;
+ let denominator = h[6] * h[7];
+ if denominator == 0.0 {
+ return None;
+ }
+ let focal_squared = -(h[0] * h[1] + h[3] * h[4]) / denominator;
+ if !focal_squared.is_finite() || focal_squared <= 0.0 {
+ return None;
+ }
+ let ratio = ((h[0] * h[0] + h[3] * h[3] + focal_squared * h[6] * h[6])
+ / (h[1] * h[1] + h[4] * h[4] + focal_squared * h[7] * h[7]))
+ .sqrt();
+ (ratio.is_finite() && ratio > 0.0).then_some(CameraAspect {
+ ratio,
+ focal_squared,
+ })
+}
+
fn aspect_ratio(quad: Quad, extent: SourceExtent) -> OpResult<(f64, AspectEvidence)> {
let p = points(quad);
let horizontal = (distance(p[0], p[1]) * distance(p[2], p[3])).sqrt();
@@ -146,27 +170,42 @@ fn aspect_ratio(quad: Quad, extent: SourceExtent) -> OpResult<(f64, AspectEviden
(p.y - extent.height * 0.5) / extent.width,
)
});
- let h = ProjectiveMap::from_unit_quad(normalized)?.coefficients;
- let denominator = h[6] * h[7];
- let focal_squared = -(h[0] * h[1] + h[3] * h[4]) / denominator;
- let diagonal = extent.width.hypot(extent.height) / extent.width;
- if denominator.abs() < 1e-5 || !focal_squared.is_finite() || focal_squared <= 0.0 {
+ let Some(camera) = camera_aspect(normalized) else {
return Ok((edge_ratio, AspectEvidence::OppositeEdges));
+ };
+ let mut sensitivity: f64 = 0.0;
+ let mut focal_uncertainty: f64 = 0.0;
+ for corner in 0..4 {
+ for axis in 0..2 {
+ for offset in [-0.5, 0.5] {
+ let mut perturbed = normalized;
+ if axis == 0 {
+ perturbed[corner].0 += offset / extent.width;
+ } else {
+ perturbed[corner].1 += offset / extent.width;
+ }
+ let Some(estimate) = camera_aspect(perturbed) else {
+ return Ok((edge_ratio, AspectEvidence::OppositeEdges));
+ };
+ sensitivity = sensitivity.max((estimate.ratio / camera.ratio).ln().abs());
+ focal_uncertainty = focal_uncertainty
+ .max((estimate.focal_squared / camera.focal_squared - 1.0).abs());
+ }
+ }
}
- let focal = focal_squared.sqrt() / diagonal;
- let ratio = ((h[0] * h[0] + h[3] * h[3] + focal_squared * h[6] * h[6])
- / (h[1] * h[1] + h[4] * h[4] + focal_squared * h[7] * h[7]))
- .sqrt();
- let focal_support =
- ((focal - 0.35) / 0.35).clamp(0.0, 1.0) * ((4.0 - focal) / 2.0).clamp(0.0, 1.0);
- let perspective_support = (h[6].abs().min(h[7].abs()) / 0.2).clamp(0.0, 1.0);
- let agreement = (1.0 - (ratio / edge_ratio).ln().abs() / 1.8f64.ln()).clamp(0.0, 1.0);
- let confidence = focal_support * perspective_support * agreement;
- if confidence == 0.0 || !ratio.is_finite() {
+ let focal_margin = 1.0 - focal_uncertainty;
+ if focal_margin <= 0.0 {
+ return Ok((edge_ratio, AspectEvidence::OppositeEdges));
+ }
+ let correction = (camera.ratio / edge_ratio).ln();
+ let correction_uncertainty = (sensitivity * correction / 0.01)
+ .hypot(focal_uncertainty / focal_margin * correction / 0.05);
+ let confidence = (-correction_uncertainty.powi(2)).exp();
+ if confidence == 0.0 {
return Ok((edge_ratio, AspectEvidence::OppositeEdges));
}
Ok((
- (edge_ratio.ln() * (1.0 - confidence) + ratio.ln() * confidence).exp(),
+ (edge_ratio.ln() + correction * confidence).exp(),
AspectEvidence::CameraBlend(confidence),
))
}
@@ -382,6 +421,119 @@ mod benchmarks {
mod camera_geometry_tests {
use super::*;
+ fn projected_rectangle(ratio: f64, pitch: f64, yaw: f64, roll: f64, focal: f64) -> Quad {
+ let focal = 1600.0 * focal;
+ let depth = focal * ratio.max(1.0) / (1200.0 * 0.42);
+ quad_from_points(
+ [(-0.5, -0.5), (0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)].map(|(x, y)| {
+ let x = x * ratio;
+ let y_tilted = y * pitch.cos();
+ let z_tilted = y * pitch.sin();
+ let x_tilted = x * yaw.cos() + z_tilted * yaw.sin();
+ let z = depth - x * yaw.sin() + z_tilted * yaw.cos();
+ let x_rotated = x_tilted * roll.cos() - y_tilted * roll.sin();
+ let y_rotated = x_tilted * roll.sin() + y_tilted * roll.cos();
+ Point {
+ x: 800.0 + focal * (x_rotated + 0.07) / z,
+ y: 600.0 + focal * (y_rotated - 0.05) / z,
+ }
+ }),
+ )
+ }
+
+ #[test]
+ fn uncertain_camera_estimates_change_smoothly_with_corner_edits() -> OpResult<()> {
+ let extent = SourceExtent::new(1600, 1200)?;
+ for (coordinates, axis) in [
+ (
+ [
+ (743.5897227842361, 177.50543714500964),
+ (1241.083461791277, 449.33223891071975),
+ (1385.403361869976, 684.2934076953679),
+ (584.6916052512825, 954.1840077098459),
+ ],
+ 0,
+ ),
+ (
+ [
+ (742.7429337520152, 399.2380002234131),
+ (1287.6039534807205, 316.6797717195004),
+ (1509.314852161333, 813.0879147816449),
+ (203.6059315316379, 1005.405351286754),
+ ],
+ 1,
+ ),
+ ] {
+ let p = coordinates.map(|(x, y)| Point { x, y });
+ let mut previous: Option<f64> = None;
+ for step in -100..=100 {
+ let mut edited = p;
+ if axis == 0 {
+ edited[0].x += step as f64 * 0.05;
+ } else {
+ edited[0].y += step as f64 * 0.05;
+ }
+ let plan = RenderPlan::new(quad_from_points(edited), extent, 0, 500_000)?;
+ let ratio = plan.width as f64 / plan.height as f64;
+ if let Some(previous) = previous {
+ assert!(
+ (ratio / previous - 1.0_f64).abs() < 0.02,
+ "step {step}: {previous} -> {ratio}"
+ );
+ }
+ previous = Some(ratio);
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn physical_rectangles_keep_proportions_across_camera_settings() -> OpResult<()> {
+ let extent = SourceExtent::new(1600, 1200)?;
+ for ratio in [0.3, 0.7, 1.0, 1.7, 3.0] {
+ for (pitch, yaw) in [(0.5, 0.3), (0.73, 0.54), (0.58, 0.9), (-0.47, 0.66)] {
+ for roll in [0.0, 0.43] {
+ for focal in [0.7, 1.2, 2.0] {
+ let quad = projected_rectangle(ratio, pitch, yaw, roll, focal);
+ let p = points(quad);
+ let edge_ratio = ((distance(p[0], p[1]) * distance(p[2], p[3]))
+ / (distance(p[1], p[2]) * distance(p[3], p[0])))
+ .sqrt();
+ let (estimated, _) = aspect_ratio(quad, extent)?;
+ assert!(
+ (estimated / ratio).ln().abs()
+ <= (edge_ratio / ratio).ln().abs() + 1e-10
+ );
+ for budget in [60_000, 500_000, 4_000_000] {
+ let original = RenderPlan::new(quad, extent, 0, budget)?;
+ let actual = original.width as f64 / original.height as f64;
+ if yaw < 0.8 && focal <= 1.2 {
+ assert!(
+ (actual / ratio - 1.0).abs() < 0.06,
+ "ratio={ratio} pitch={pitch} yaw={yaw} roll={roll} focal={focal}: {actual} {:?}",
+ original.aspect_evidence
+ );
+ }
+ assert!(
+ original.width as u64 * original.height as u64 <= budget as u64
+ );
+ for rotation in [90, 180, 270] {
+ let rotated = RenderPlan::new(quad, extent, rotation, budget)?;
+ let expected = if rotation == 180 {
+ (original.width, original.height)
+ } else {
+ (original.height, original.width)
+ };
+ assert_eq!((rotated.width, rotated.height), expected);
+ }
+ }
+ }
+ }
+ }
+ }
+ Ok(())
+ }
+
fn camera_quad(width: f64, height: f64, yaw: f64, pitch: f64) -> Quad {
let project = |x: f64, y: f64| {
let x_camera = yaw.cos() * x + yaw.sin() * pitch.sin() * y;