π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Stabilize perspective proportions with source edge refinement
laurenspriem4769ac51adae
diff --git a/rust/crates/ml/src/scan/boundary.rs b/rust/crates/ml/src/scan/boundary.rsindex 7630229387..9b8c1f676d 100644--- a/rust/crates/ml/src/scan/boundary.rs+++ b/rust/crates/ml/src/scan/boundary.rs@@ -548,7 +548,21 @@ fn brightness(source: &ImageU8, x: f64, y: f64) -> Option<f64> {)}-fn nearby_color_edge(source: &ImageU8, edge: [Point; 2], band: f64) -> bool {+struct ColorEdgeScores {+ values: [[f64; 91]; 48],+ supported: [bool; 48],+}++impl ColorEdgeScores {+ fn supported_positions(&self) -> usize {+ self.supported+ .iter()+ .filter(|&&supported| supported)+ .count()+ }+}++fn color_edge_scores(source: &ImageU8, edge: [Point; 2], band: f64) -> ColorEdgeScores {let line = Line::through(edge[0], edge[1]);let sample = |x: f64, y: f64| -> Option<[f64; 3]> {if x < 0.0 || y < 0.0 || x >= source.width as f64 || y >= source.height as f64 {@@ -559,57 +573,145 @@ fn nearby_color_edge(source: &ImageU8, edge: [Point; 2], band: f64) -> bool {source.data[offset + channel] as f64}))};- (0..48)- .filter(|&i| {- let t = (i as f64 + 1.0) / 49.0;- let center = Point {- x: edge[0].x + (edge[1].x - edge[0].x) * t,- y: edge[0].y + (edge[1].y - edge[0].y) * t,+ let mut supported = [false; 48];+ let values = std::array::from_fn(|i| {+ let t = (i as f64 + 1.0) / 49.0;+ let center = Point {+ x: edge[0].x + (edge[1].x - edge[0].x) * t,+ y: edge[0].y + (edge[1].y - edge[0].y) * t,+ };+ std::array::from_fn(|step| {+ let offset = (step as f64 - 45.0) * band / 30.0;+ let Some(a) = sample(+ center.x + line.nx * (offset - 1.25),+ center.y + line.ny * (offset - 1.25),+ ) else {+ return 0.0;};- (-30..=30).any(|step| {- let offset = step as f64 * band / 30.0;- let Some(a) = sample(- center.x + line.nx * (offset - 1.25),- center.y + line.ny * (offset - 1.25),- ) else {- return false;- };- let Some(b) = sample(- center.x + line.nx * (offset + 1.25),- center.y + line.ny * (offset + 1.25),- ) else {- return false;- };- let delta = std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]);- let dot = |a: [f64; 3], b: [f64; 3]| {- (a[0] * b[0] + 2.0 * a[1] * b[1] + a[2] * b[2]) / 4.0- };- let energy = dot(delta, delta);- let penalty = 1.0 + 0.25 * (offset / band).powi(2);- if energy < (6.0 * penalty).powi(2) {- return false;- }- [2.5, 4.0, 6.0].into_iter().all(|reach| {- let Some(a) = sample(- center.x + line.nx * (offset - reach),- center.y + line.ny * (offset - reach),- ) else {- return false;- };- let Some(b) = sample(- center.x + line.nx * (offset + reach),- center.y + line.ny * (offset + reach),- ) else {- return false;- };- let wide = std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]);- let wide_energy = dot(wide, wide);- wide_energy >= 36.0 && dot(delta, wide) >= 0.7 * (energy * wide_energy).sqrt()- })- })+ let Some(b) = sample(+ center.x + line.nx * (offset + 1.25),+ center.y + line.ny * (offset + 1.25),+ ) else {+ return 0.0;+ };+ let delta = std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]);+ let dot =+ |a: [f64; 3], b: [f64; 3]| (a[0] * b[0] + 2.0 * a[1] * b[1] + a[2] * b[2]) / 4.0;+ let energy = dot(delta, delta);+ let penalty = 1.0 + 0.25 * (offset / band).powi(2);+ if energy < 9.0 {+ return 0.0;+ }+ let Some(inner) = sample(+ center.x + line.nx * (offset - 0.5),+ center.y + line.ny * (offset - 0.5),+ ) else {+ return 0.0;+ };+ let Some(outer) = sample(+ center.x + line.nx * (offset + 0.5),+ center.y + line.ny * (offset + 0.5),+ ) else {+ return 0.0;+ };+ let fine = std::array::from_fn::<_, 3, _>(|i| inner[i] - outer[i]);+ let wide = [2.5, 4.0, 6.0].map(|reach| {+ let a = sample(+ center.x + line.nx * (offset - reach),+ center.y + line.ny * (offset - reach),+ )?;+ let b = sample(+ center.x + line.nx * (offset + reach),+ center.y + line.ny * (offset + reach),+ )?;+ Some(std::array::from_fn::<_, 3, _>(|i| a[i] - b[i]))+ });+ let [Some(a), Some(b), Some(c)] = wide else {+ return 0.0;+ };+ let agrees = |wide: [f64; 3], minimum: f64| {+ let wide_energy = dot(wide, wide);+ wide_energy >= minimum && dot(delta, wide) >= 0.7 * (energy * wide_energy).sqrt()+ };+ if (15..76).contains(&step)+ && energy >= (6.0 * penalty).powi(2)+ && [a, b, c].into_iter().all(|wide| agrees(wide, 36.0))+ {+ supported[i] = true;+ }+ if [a, b].into_iter().all(|wide| agrees(wide, 9.0)) {+ (1.0 + dot(fine, fine).sqrt()) / penalty+ } else {+ 0.0+ }})- .count()- >= 32+ });+ ColorEdgeScores { values, supported }+}++fn refine_color_edges(+ p: [Point; 4],+ scores: &[ColorEdgeScores; 4],+ band: f64,+) -> Option<[Point; 4]> {+ let mut lines = std::array::from_fn::<_, 4, _>(|i| Line::through(p[i], p[(i + 1) % 4]));+ for i in 0..4 {+ let available = scores[i].supported_positions();+ if available < 32 {+ return None;+ }+ let mut best = None;+ let mut best_score = 0.0;+ for start in 0..91 {+ for end in 0..91 {+ let mut supported = 0;+ let mut strength = 0.0;+ let mut weighted_support = 0.0;+ for (position, row) in scores[i].values.iter().enumerate() {+ let t = (position as f64 + 1.0) / 49.0;+ let offset = (start as f64 * (1.0 - t) + end as f64 * t).round() as usize;+ let score = row[offset];+ supported += usize::from(score > 0.0);+ strength += score / (score + 16.0);+ if score > 0.0 {+ weighted_support +=+ (1.0 + 0.25 * ((offset as f64 - 45.0) / 30.0).powi(2)).recip();+ }+ }+ let score = weighted_support + strength * 0.1;+ if supported >= 32 && supported * 5 >= available * 4 && score > best_score {+ best_score = score;+ best = Some((start, end));+ }+ }+ }+ let (start, end) = best?;+ let mut samples = Vec::new();+ for (position, row) in scores[i].values.iter().enumerate() {+ let t = (position as f64 + 1.0) / 49.0;+ let index = (start as f64 * (1.0 - t) + end as f64 * t).round() as usize;+ if row[index] == 0.0 {+ continue;+ }+ let offset = (index as f64 - 45.0) * band / 30.0;+ samples.push(Point {+ x: p[i].x + (p[(i + 1) % 4].x - p[i].x) * t + lines[i].nx * offset,+ y: p[i].y + (p[(i + 1) % 4].y - p[i].y) * t + lines[i].ny * offset,+ });+ }+ if samples.len() < 32 {+ return None;+ }+ lines[i] = fit_line(&samples, lines[i], band);+ }+ let mut refined = p;+ for i in 0..4 {+ refined[i] = lines[(i + 3) % 4].intersection(lines[i])?;+ if distance(refined[i], p[i]) > band * 1.5 {+ return None;+ }+ }+ Some(refined)}pub(super) fn refine_capture(@@ -715,11 +817,25 @@ pub(super) fn refine_capture(lines[side] = fitted;improved += 1;}+ let color_samples = std::array::from_fn::<_, 4, _>(|side| {+ color_edge_scores(&preview, [p[side], p[(side + 1) % 4]], band * 3.0)+ });if (supported < 3 || needs_complete_support)- && !(0..4).all(|side| nearby_color_edge(&preview, [p[side], p[(side + 1) % 4]], band * 3.0))+ && color_samples+ .iter()+ .any(|samples| samples.supported_positions() < 32){return Ok(None);}+ if let Some(refined) = refine_color_edges(p, &color_samples, band * 3.0) {+ let refined = quad_from_points(refined.map(|p| Point {+ x: p.x / sx,+ y: p.y / sy,+ }));+ if validate_quad(refined, extent).is_ok() {+ return Ok(Some(refined));+ }+ }if improved == 0 {return Ok(Some(quad));}@@ -1292,6 +1408,69 @@ mod benchmarks {mod source_evidence_tests {use super::*;+ #[test]+ fn broad_printed_panel_does_not_displace_a_supported_outer_page() -> OpResult<()> {+ let mut source = ImageU8::new(400, 300, 3, vec![180; 400 * 300 * 3])?;+ for y in 40..260 {+ for x in 60..340 {+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(195);+ }+ }+ for y in 45..255 {+ for x in 65..335 {+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(90);+ }+ }+ for y in [100, 150, 200] {+ for x in 62..65 {+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(30);+ }+ }+ let quad = quad_from_points(+ [(60.0, 40.0), (340.0, 40.0), (340.0, 260.0), (60.0, 260.0)]+ .map(|(x, y)| Point { x, y }),+ );+ let refined = refine_capture(&source, quad, false)?.ok_or("page rejected")?;+ for (actual, expected) in points(refined).into_iter().zip(points(quad)) {+ assert!(+ distance(actual, expected) < 2.0,+ "{actual:?} vs {expected:?}"+ );+ }+ Ok(())+ }++ #[test]+ fn sustained_chromatic_edges_refine_displaced_page_corners() -> OpResult<()> {+ let mut source = ImageU8::new(400, 300, 3, [80, 140, 200].repeat(400 * 300))?;+ let expected = [(80.0, 40.0), (315.0, 60.0), (340.0, 245.0), (60.0, 265.0)]+ .map(|(x, y)| Point { x, y });+ for y in 0..300 {+ for x in 0..400 {+ let point = Point {+ x: x as f64,+ y: y as f64,+ };+ if (0..4).all(|i| cross(expected[i], expected[(i + 1) % 4], point) >= 0.0) {+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3]+ .copy_from_slice(&[40, 160, 200]);+ }+ }+ }+ let displaced = quad_from_points(+ [(84.0, 45.0), (312.0, 66.0), (344.0, 250.0), (66.0, 260.0)]+ .map(|(x, y)| Point { x, y }),+ );+ let refined = refine_capture(&source, displaced, false)?.ok_or("page rejected")?;+ for (actual, expected) in points(refined).into_iter().zip(expected) {+ assert!(+ distance(actual, expected) < 2.0,+ "{actual:?} vs {expected:?}"+ );+ }+ Ok(())+ }+#[test]fn color_edge_rejects_thin_strokes_without_a_background_step() -> OpResult<()> {let mut source = ImageU8::new(400, 300, 3, vec![220; 400 * 300 * 3])?;@@ -1299,13 +1478,13 @@ mod source_evidence_tests {source.data[(100 * 400 + x) * 3..(100 * 400 + x + 1) * 3].fill(30);}let edge = [Point { x: 60.0, y: 100.0 }, Point { x: 340.0, y: 100.0 }];- assert!(!nearby_color_edge(&source, edge, 7.2));+ assert!(color_edge_scores(&source, edge, 7.2).supported_positions() < 32);for y in 100..300 {for x in 0..400 {source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(150);}}- assert!(nearby_color_edge(&source, edge, 7.2));+ assert!(color_edge_scores(&source, edge, 7.2).supported_positions() >= 32);Ok(())}diff --git a/rust/crates/ml/src/scan/page.rs b/rust/crates/ml/src/scan/page.rsindex c36d1c0422..893a0406ae 100644--- a/rust/crates/ml/src/scan/page.rs+++ b/rust/crates/ml/src/scan/page.rs@@ -135,6 +135,30 @@ pub(super) struct RenderPlan {pub aspect_evidence: AspectEvidence,}+struct CameraAspect {+ ratio: f64,+ focal_squared: f64,+}++fn camera_aspect(p: [(f64, f64); 4]) -> Option<CameraAspect> {+ let h = ProjectiveMap::from_unit_quad(p).ok()?.coefficients;+ let denominator = h[6] * h[7];+ if denominator == 0.0 {+ return None;+ }+ let focal_squared = -(h[0] * h[1] + h[3] * h[4]) / denominator;+ if !focal_squared.is_finite() || focal_squared <= 0.0 {+ return None;+ }+ let ratio = ((h[0] * h[0] + h[3] * h[3] + focal_squared * h[6] * h[6])+ / (h[1] * h[1] + h[4] * h[4] + focal_squared * h[7] * h[7]))+ .sqrt();+ (ratio.is_finite() && ratio > 0.0).then_some(CameraAspect {+ ratio,+ focal_squared,+ })+}+fn aspect_ratio(quad: Quad, extent: SourceExtent) -> OpResult<(f64, AspectEvidence)> {let p = points(quad);let horizontal = (distance(p[0], p[1]) * distance(p[2], p[3])).sqrt();@@ -146,27 +170,42 @@ fn aspect_ratio(quad: Quad, extent: SourceExtent) -> OpResult<(f64, AspectEviden(p.y - extent.height * 0.5) / extent.width,)});- let h = ProjectiveMap::from_unit_quad(normalized)?.coefficients;- let denominator = h[6] * h[7];- let focal_squared = -(h[0] * h[1] + h[3] * h[4]) / denominator;- let diagonal = extent.width.hypot(extent.height) / extent.width;- if denominator.abs() < 1e-5 || !focal_squared.is_finite() || focal_squared <= 0.0 {+ let Some(camera) = camera_aspect(normalized) else {return Ok((edge_ratio, AspectEvidence::OppositeEdges));+ };+ let mut sensitivity: f64 = 0.0;+ let mut focal_uncertainty: f64 = 0.0;+ for corner in 0..4 {+ for axis in 0..2 {+ for offset in [-0.5, 0.5] {+ let mut perturbed = normalized;+ if axis == 0 {+ perturbed[corner].0 += offset / extent.width;+ } else {+ perturbed[corner].1 += offset / extent.width;+ }+ let Some(estimate) = camera_aspect(perturbed) else {+ return Ok((edge_ratio, AspectEvidence::OppositeEdges));+ };+ sensitivity = sensitivity.max((estimate.ratio / camera.ratio).ln().abs());+ focal_uncertainty = focal_uncertainty+ .max((estimate.focal_squared / camera.focal_squared - 1.0).abs());+ }+ }}- let focal = focal_squared.sqrt() / diagonal;- let ratio = ((h[0] * h[0] + h[3] * h[3] + focal_squared * h[6] * h[6])- / (h[1] * h[1] + h[4] * h[4] + focal_squared * h[7] * h[7]))- .sqrt();- let focal_support =- ((focal - 0.35) / 0.35).clamp(0.0, 1.0) * ((4.0 - focal) / 2.0).clamp(0.0, 1.0);- let perspective_support = (h[6].abs().min(h[7].abs()) / 0.2).clamp(0.0, 1.0);- let agreement = (1.0 - (ratio / edge_ratio).ln().abs() / 1.8f64.ln()).clamp(0.0, 1.0);- let confidence = focal_support * perspective_support * agreement;- if confidence == 0.0 || !ratio.is_finite() {+ let focal_margin = 1.0 - focal_uncertainty;+ if focal_margin <= 0.0 {+ return Ok((edge_ratio, AspectEvidence::OppositeEdges));+ }+ let correction = (camera.ratio / edge_ratio).ln();+ let correction_uncertainty = (sensitivity * correction / 0.01)+ .hypot(focal_uncertainty / focal_margin * correction / 0.05);+ let confidence = (-correction_uncertainty.powi(2)).exp();+ if confidence == 0.0 {return Ok((edge_ratio, AspectEvidence::OppositeEdges));}Ok((- (edge_ratio.ln() * (1.0 - confidence) + ratio.ln() * confidence).exp(),+ (edge_ratio.ln() + correction * confidence).exp(),AspectEvidence::CameraBlend(confidence),))}@@ -382,6 +421,119 @@ mod benchmarks {mod camera_geometry_tests {use super::*;+ fn projected_rectangle(ratio: f64, pitch: f64, yaw: f64, roll: f64, focal: f64) -> Quad {+ let focal = 1600.0 * focal;+ let depth = focal * ratio.max(1.0) / (1200.0 * 0.42);+ quad_from_points(+ [(-0.5, -0.5), (0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)].map(|(x, y)| {+ let x = x * ratio;+ let y_tilted = y * pitch.cos();+ let z_tilted = y * pitch.sin();+ let x_tilted = x * yaw.cos() + z_tilted * yaw.sin();+ let z = depth - x * yaw.sin() + z_tilted * yaw.cos();+ let x_rotated = x_tilted * roll.cos() - y_tilted * roll.sin();+ let y_rotated = x_tilted * roll.sin() + y_tilted * roll.cos();+ Point {+ x: 800.0 + focal * (x_rotated + 0.07) / z,+ y: 600.0 + focal * (y_rotated - 0.05) / z,+ }+ }),+ )+ }++ #[test]+ fn uncertain_camera_estimates_change_smoothly_with_corner_edits() -> OpResult<()> {+ let extent = SourceExtent::new(1600, 1200)?;+ for (coordinates, axis) in [+ (+ [+ (743.5897227842361, 177.50543714500964),+ (1241.083461791277, 449.33223891071975),+ (1385.403361869976, 684.2934076953679),+ (584.6916052512825, 954.1840077098459),+ ],+ 0,+ ),+ (+ [+ (742.7429337520152, 399.2380002234131),+ (1287.6039534807205, 316.6797717195004),+ (1509.314852161333, 813.0879147816449),+ (203.6059315316379, 1005.405351286754),+ ],+ 1,+ ),+ ] {+ let p = coordinates.map(|(x, y)| Point { x, y });+ let mut previous: Option<f64> = None;+ for step in -100..=100 {+ let mut edited = p;+ if axis == 0 {+ edited[0].x += step as f64 * 0.05;+ } else {+ edited[0].y += step as f64 * 0.05;+ }+ let plan = RenderPlan::new(quad_from_points(edited), extent, 0, 500_000)?;+ let ratio = plan.width as f64 / plan.height as f64;+ if let Some(previous) = previous {+ assert!(+ (ratio / previous - 1.0_f64).abs() < 0.02,+ "step {step}: {previous} -> {ratio}"+ );+ }+ previous = Some(ratio);+ }+ }+ Ok(())+ }++ #[test]+ fn physical_rectangles_keep_proportions_across_camera_settings() -> OpResult<()> {+ let extent = SourceExtent::new(1600, 1200)?;+ for ratio in [0.3, 0.7, 1.0, 1.7, 3.0] {+ for (pitch, yaw) in [(0.5, 0.3), (0.73, 0.54), (0.58, 0.9), (-0.47, 0.66)] {+ for roll in [0.0, 0.43] {+ for focal in [0.7, 1.2, 2.0] {+ let quad = projected_rectangle(ratio, pitch, yaw, roll, focal);+ let p = points(quad);+ let edge_ratio = ((distance(p[0], p[1]) * distance(p[2], p[3]))+ / (distance(p[1], p[2]) * distance(p[3], p[0])))+ .sqrt();+ let (estimated, _) = aspect_ratio(quad, extent)?;+ assert!(+ (estimated / ratio).ln().abs()+ <= (edge_ratio / ratio).ln().abs() + 1e-10+ );+ for budget in [60_000, 500_000, 4_000_000] {+ let original = RenderPlan::new(quad, extent, 0, budget)?;+ let actual = original.width as f64 / original.height as f64;+ if yaw < 0.8 && focal <= 1.2 {+ assert!(+ (actual / ratio - 1.0).abs() < 0.06,+ "ratio={ratio} pitch={pitch} yaw={yaw} roll={roll} focal={focal}: {actual} {:?}",+ original.aspect_evidence+ );+ }+ assert!(+ original.width as u64 * original.height as u64 <= budget as u64+ );+ for rotation in [90, 180, 270] {+ let rotated = RenderPlan::new(quad, extent, rotation, budget)?;+ let expected = if rotation == 180 {+ (original.width, original.height)+ } else {+ (original.height, original.width)+ };+ assert_eq!((rotated.width, rotated.height), expected);+ }+ }+ }+ }+ }+ }+ Ok(())+ }+fn camera_quad(width: f64, height: f64, yaw: f64, pitch: f64) -> Quad {let project = |x: f64, y: f64| {let x_camera = yaw.cos() * x + yaw.sin() * pitch.sin() * y;