π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Treat small color accents as monochrome document content
laurenspriem340ff06eb9d2
diff --git a/rust/crates/ml/src/scan/appearance.rs b/rust/crates/ml/src/scan/appearance.rsindex e6c67d3c83..2ac98011b8 100644--- a/rust/crates/ml/src/scan/appearance.rs+++ b/rust/crates/ml/src/scan/appearance.rs@@ -9,7 +9,7 @@ const ANALYSIS_LONG_EDGE: usize = 384;const CELL_EDGE: usize = 16;const CHROMA_BIN_COUNT: usize = 48;const PAPER_CHROMA_RADIUS: f32 = 0.032;-const NEUTRAL_CHROMA_SPREAD: f32 = 0.034;+const NEUTRAL_CHROMA_SPREAD: f32 = 0.067;const MIN_PAPER_LUMINANCE: f32 = 0.12;const MAX_CELL_LOG_STEP: f32 = 0.14;const MAX_PAPER_CURVATURE: f32 = 16.0;@@ -42,7 +42,6 @@ struct PaperField {log_luminance: Vec<f32>,confidence: Vec<f32>,target_log_luminance: f32,- adaptation: [f32; 3],}struct TransferTables {@@ -385,22 +384,50 @@ fn estimate_field(analysis: &Analysis) -> Option<PaperField> {.into_iter().map(|value| 1.0 - transition(value as f32, confidence_start, confidence_end)).collect();- let adaptation = if neutral {- let paper_luminance = luminance(chroma);- chroma.map(|channel| (paper_luminance / channel.max(0.001)).clamp(0.88, 1.12))- } else {- [1.0; 3]- };Some(PaperField {width,height,log_luminance: field,confidence,target_log_luminance,- adaptation,})}+fn paper_adaptation(analysis: &Analysis) -> [f32; 3] {+ let Some(chroma) = paper_chromaticity(analysis) else {+ return [1.0; 3];+ };+ let spread = chroma.iter().copied().fold(f32::NEG_INFINITY, f32::max)+ - chroma.iter().copied().fold(f32::INFINITY, f32::min);+ if spread >= NEUTRAL_CHROMA_SPREAD {+ return [1.0; 3];+ }+ let mut supported = [0.0; 9];+ let mut available = [0.0; 9];+ for (index, sample) in analysis.samples.iter().enumerate() {+ let x = index % analysis.width;+ let y = index / analysis.width;+ let region = (y * 3 / analysis.height) * 3 + x * 3 / analysis.width;+ available[region] += sample.coverage;+ if sample.smoothness > 0.5+ && chroma_distance(chromaticity(sample.rgb), chroma) < PAPER_CHROMA_RADIUS+ {+ supported[region] += sample.coverage;+ }+ }+ let broad_support = supported.iter().sum::<f32>() >= available.iter().sum::<f32>() * 0.2;+ let regions = supported+ .iter()+ .zip(available)+ .filter(|(support, available)| available > &0.0 && **support >= available * 0.05)+ .count();+ if !broad_support || regions < 7 {+ return [1.0; 3];+ }+ let paper_luminance = luminance(chroma);+ chroma.map(|channel| (paper_luminance / channel.max(0.001)).clamp(0.88, 1.12))+}+fn neighbors(x: usize, y: usize, width: usize, height: usize) -> [Option<usize>; 4] {[(x > 0).then(|| y * width + x - 1),@@ -551,7 +578,13 @@ fn encode(value: f32, table: &[f32]) -> u8 {(table[low] * (1.0 - fraction) + table[high] * fraction).round() as u8}-fn render(image: &mut ImageU8, valid: Option<&[u8]>, field: Option<&PaperField>, grayscale: bool) {+fn render(+ image: &mut ImageU8,+ valid: Option<&[u8]>,+ field: Option<&PaperField>,+ adaptation: [f32; 3],+ grayscale: bool,+) {let width = image.width as usize;let height = image.height as usize;let tables = transfer_tables();@@ -560,7 +593,7 @@ fn render(image: &mut ImageU8, valid: Option<&[u8]>, field: Option<&PaperField>,pixel.fill(255);continue;}- if field.is_none() && !grayscale {+ if field.is_none() && adaptation == [1.0; 3] && !grayscale {continue;}let mut rgb = [@@ -575,12 +608,15 @@ fn render(image: &mut ImageU8, valid: Option<&[u8]>, field: Option<&PaperField>,let confidence = interpolate(&field.confidence, field.width, field.height, x, y);let gain =((field.target_log_luminance - paper).clamp(-0.08, 4.0f32.ln()) * confidence).exp();- for (value, adaptation) in rgb.iter_mut().zip(field.adaptation) {- *value *= gain * (1.0 + (adaptation - 1.0) * confidence);+ for value in &mut rgb {+ *value *= gain;}- let maximum = rgb.iter().copied().fold(1.0, f32::max);- rgb.iter_mut().for_each(|value| *value /= maximum);}+ for (value, adaptation) in rgb.iter_mut().zip(adaptation) {+ *value *= adaptation;+ }+ let maximum = rgb.iter().copied().fold(1.0, f32::max);+ rgb.iter_mut().for_each(|value| *value /= maximum);if grayscale {pixel.fill(encode(luminance(rgb), &tables.encode));} else {@@ -591,10 +627,20 @@ fn render(image: &mut ImageU8, valid: Option<&[u8]>, field: Option<&PaperField>,}}-fn meaningful_color(image: &ImageU8, valid: Option<&[u8]>) -> bool {+fn meaningful_color(image: &ImageU8, valid: Option<&[u8]>, adaptation: [f32; 3]) -> bool {let width = image.width as usize;let height = image.height as usize;- let mut coherent = 0;+ let decode = &transfer_tables().decode;+ let corrected = |pixel: &[u8]| {+ std::array::from_fn::<_, 3, _>(|channel| {+ decode[pixel[2 - channel] as usize] * adaptation[channel]+ })+ };+ let radius = (width.max(height) / 192).max(2);+ let mut saturated = 0.0;+ let mut colored_content = 0.0;+ let mut content = 0.0;+ let mut available = 0;for y in 0..height {for x in 0..width {let index = y * width + x;@@ -602,22 +648,37 @@ fn meaningful_color(image: &ImageU8, valid: Option<&[u8]>) -> bool {continue;}let pixel = &image.data[index * 3..index * 3 + 3];- let low = pixel[0].min(pixel[1]).min(pixel[2]) as i16;- let high = pixel[0].max(pixel[1]).max(pixel[2]) as i16;- if high - low < 13 {+ let rgb = corrected(pixel);+ let low = rgb.into_iter().fold(f32::INFINITY, f32::min);+ let high = rgb.into_iter().fold(0.0, f32::max);+ let brightness = |x: usize, y: usize| {+ let neighbor = y * width + x;+ if valid.is_some_and(|mask| mask[neighbor] == 0) {+ luminance(rgb)+ } else {+ luminance(corrected(&image.data[neighbor * 3..neighbor * 3 + 3]))+ }+ };+ let horizontal = brightness(x.saturating_sub(radius), y)+ .min(brightness((x + radius).min(width - 1), y));+ let vertical = brightness(x, y.saturating_sub(radius))+ .min(brightness(x, (y + radius).min(height - 1)));+ let local = horizontal.max(vertical);+ let contrast = (1.0 - luminance(rgb) / local.max(0.01)).max(0.0);+ let ink = contrast * transition(contrast, 0.08, 0.3);+ available += 1;+ content += ink;+ let strength = transition((high - low) / high.sqrt().max(0.01), 0.045, 0.12);+ if strength == 0.0 {continue;}- if width * height < 36 {- return true;- }- let differences = [- pixel[2] as i16 - pixel[1] as i16,- pixel[0] as i16 - pixel[1] as i16,- ];- let mut matched = false;+ let differences = [rgb[0] - rgb[1], rgb[2] - rgb[1]];+ let mut matched = width * height < 36;for neighbor in [(x + 1 < width).then_some(index + 1),(y + 1 < height).then_some(index + width),+ (x > 0 && y + 1 < height).then(|| index + width - 1),+ (x + 1 < width && y + 1 < height).then_some(index + width + 1),].into_iter().flatten()@@ -625,26 +686,21 @@ fn meaningful_color(image: &ImageU8, valid: Option<&[u8]>) -> bool {if valid.is_some_and(|mask| mask[neighbor] == 0) {continue;}- let other = &image.data[neighbor * 3..neighbor * 3 + 3];- let other_differences = [- other[2] as i16 - other[1] as i16,- other[0] as i16 - other[1] as i16,- ];- let dot = differences[0] as i32 * other_differences[0] as i32- + differences[1] as i32 * other_differences[1] as i32;- let length = differences[0] as i32 * differences[0] as i32- + differences[1] as i32 * differences[1] as i32;- matched |= dot > length / 2;+ let other = corrected(&image.data[neighbor * 3..neighbor * 3 + 3]);+ let other_differences = [other[0] - other[1], other[2] - other[1]];+ let dot =+ differences[0] * other_differences[0] + differences[1] * other_differences[1];+ let length = differences[0].powi(2) + differences[1].powi(2);+ matched |= dot > length * 0.5;}if matched {- coherent += 1;- if coherent >= 6 {- return true;- }+ saturated += strength * transition((high - low) / high.max(0.01), 0.12, 0.3);+ colored_content += ink * strength;}}}- false+ saturated / available.max(1) as f32 >= 0.02+ || (colored_content > 0.0 && colored_content >= content * 0.4)}pub(super) fn render_document(@@ -655,18 +711,21 @@ pub(super) fn render_document(validate(image, valid)?;let analysis = analyze(image, valid, ANALYSIS_LONG_EDGE);let field = estimate_field(&analysis);- let explicit_grayscale = matches!(requested_mode, Some(ColorMode::Grayscale));- render(image, valid, field.as_ref(), explicit_grayscale);+ let adaptation = paper_adaptation(&analysis);let mode = requested_mode.unwrap_or_else(|| {- if meaningful_color(image, valid) {+ if meaningful_color(image, valid, adaptation) {ColorMode::Color} else {ColorMode::Grayscale}});- if requested_mode.is_none() && matches!(mode, ColorMode::Grayscale) {- render(image, valid, None, true);- }+ render(+ image,+ valid,+ field.as_ref(),+ adaptation,+ matches!(mode, ColorMode::Grayscale),+ );Ok(mode)}@@ -696,6 +755,14 @@ mod tests {}}+ fn has_color(image: &ImageU8, valid: Option<&[u8]>) -> bool {+ meaningful_color(+ image,+ valid,+ paper_adaptation(&analyze(image, valid, ANALYSIS_LONG_EDGE)),+ )+ }+fn image(width: usize, height: usize, rgb: [u8; 3]) -> ImageU8 {ImageU8 {width: width as i32,@@ -874,18 +941,26 @@ mod tests {}#[test]- fn keeps_tiny_coherent_stamps_and_one_pixel_color_strokes() {- let mut stamped = image(1600, 2200, [244; 3]);- for y in 1200..1209 {- for x in 400..407 {- let offset = (y * 1600 + x) * 3;- stamped.data[offset..offset + 3].copy_from_slice(&[61, 32, 194]);+ fn small_stamp_on_monochrome_print_chooses_grayscale() {+ for stamp in [false, true] {+ let mut printed = document(160, 220, [244; 3], false).clean;+ if stamp {+ for y in 180..184 {+ for x in 40..44 {+ let offset = (y * 160 + x) * 3;+ printed.data[offset..offset + 3].copy_from_slice(&[194, 32, 61]);+ }+ }}+ assert!(matches!(+ process(&mut printed, None, None),+ ColorMode::Grayscale+ ));}- assert!(matches!(- process(&mut stamped, None, None),- ColorMode::Color- ));+ }++ #[test]+ fn sparse_colored_writing_as_main_content_stays_color() {let mut signature = image(700, 1000, [244; 3]);for x in 230..249 {let offset = (720 * 700 + x) * 3;@@ -897,6 +972,159 @@ mod tests {));}+ #[test]+ fn stamp_decisions_follow_content_proportions_across_resolutions() {+ for (width, height) in [(192, 264), (576, 792), (1152, 1584)] {+ for stamp_width in [width / 50, width / 20, width / 10] {+ let mut printed = document(width, height, [244; 3], false).clean;+ for y in height * 4 / 5..height * 4 / 5 + stamp_width {+ for x in width / 3..width / 3 + stamp_width {+ printed.data[(y * width + x) * 3..][..3].copy_from_slice(&[173, 93, 66]);+ }+ }+ let mut explicit = printed.clone();+ assert!(+ matches!(process(&mut printed, None, None), ColorMode::Grayscale),+ "{width}x{height}, stamp {stamp_width}"+ );+ assert!(+ printed+ .data+ .as_chunks::<3>()+ .0+ .iter()+ .all(|p| p[0] == p[1] && p[1] == p[2])+ );+ assert!(matches!(+ process(&mut explicit, None, Some(ColorMode::Color)),+ ColorMode::Color+ ));+ let pixel = &explicit.data[((height * 4 / 5) * width + width / 3) * 3..][..3];+ assert!(pixel[0] > pixel[2] + 60);+ }+ }+ }++ #[test]+ fn faint_colored_handwriting_is_distinct_from_an_accent_on_black_print() {+ for (width, height) in [(224, 320), (672, 960)] {+ for rgb in [[63, 82, 153], [191, 203, 227]] {+ let mut writing = image(width, height, [244; 3]);+ let mut printed = document(width, height, [244; 3], false).clean;+ let mut previous_y = (height * 4 / 5) as isize;+ for x in width / 4..width * 3 / 4 {+ let y = (height * 4 / 5) as isize+ + ((x as f32 / width as f32 * 80.0).sin() * 8.0) as isize;+ for stroke_y in previous_y.min(y)..=previous_y.max(y) {+ let index = stroke_y as usize * width + x;+ writing.data[index * 3..][..3].copy_from_slice(&[rgb[2], rgb[1], rgb[0]]);+ printed.data[index * 3..][..3].copy_from_slice(&[rgb[2], rgb[1], rgb[0]]);+ }+ previous_y = y;+ }+ for shadow in [false, true] {+ let mut writing = writing.clone();+ let mut printed = printed.clone();+ if shadow {+ for image in [&mut writing, &mut printed] {+ for (index, pixel) in+ image.data.as_chunks_mut::<3>().0.iter_mut().enumerate()+ {+ let illumination = if index % width < width / 2 {+ 0.42+ } else {+ 0.91+ };+ for value in pixel {+ *value = encode(+ transfer_tables().decode[*value as usize] * illumination,+ &transfer_tables().encode,+ );+ }+ }+ }+ }+ assert!(+ matches!(process(&mut writing, None, None), ColorMode::Color),+ "writing {width} {rgb:?} shadow {shadow}"+ );+ assert!(+ matches!(process(&mut printed, None, None), ColorMode::Grayscale),+ "accent {width} {rgb:?} shadow {shadow}"+ );+ }+ }+ }+ }++ #[test]+ fn highlighted_monochrome_print_stays_color() {+ let mut page = document(384, 544, [244; 3], false).clean;+ for y in 181..197 {+ for x in 35..323 {+ let pixel = &mut page.data[(y * 384 + x) * 3..][..3];+ if pixel[0] > 220 {+ pixel.copy_from_slice(&[103, 230, 242]);+ }+ }+ }+ assert!(matches!(process(&mut page, None, None), ColorMode::Color));+ let pixel = &page.data[(195 * 384 + 40) * 3..][..3];+ assert!(pixel[2] > pixel[0] + 100);+ }++ #[test]+ fn neutral_paper_cast_is_corrected_without_an_illumination_field() {+ let mut fixture = document(384, 544, [248; 3], true);+ let tables = transfer_tables();+ for (index, pixel) in fixture+ .shaded+ .data+ .as_chunks_mut::<3>()+ .0+ .iter_mut()+ .enumerate()+ {+ let illumination = if index % 384 < 192 { 0.45 } else { 0.9 };+ for (channel, value) in pixel.iter_mut().enumerate() {+ *value = encode(+ tables.decode[fixture.clean.data[index * 3 + channel] as usize]+ * illumination+ * [0.94, 1.01, 1.06][channel],+ &tables.encode,+ );+ }+ }+ assert!(estimate_field(&analyze(&fixture.shaded, None, ANALYSIS_LONG_EDGE)).is_none());+ let mut output = fixture.shaded;+ assert!(matches!(+ process(&mut output, None, Some(ColorMode::Color)),+ ColorMode::Color+ ));+ for x in [40, 340] {+ let paper = &output.data[(40 * 384 + x) * 3..][..3];+ assert!(+ paper.iter().max().expect("channel") - paper.iter().min().expect("channel") <= 2+ );+ }+ assert!(+ region_mean(&output, &fixture.regions, Region::Paper)+ - region_mean(&output, &fixture.regions, Region::Faint)+ > 10.0+ );+ for (actual, expected) in output+ .data+ .as_chunks::<3>()+ .0+ .iter()+ .zip(fixture.clean.data.as_chunks::<3>().0.iter())+ {+ if expected[0] as i16 - expected[2] as i16 > 100 {+ assert!(actual[0] as i16 - actual[2] as i16 > 75);+ }+ }+ }+#[test]fn photograph_without_broad_paper_support_is_unchanged() {let mut photo = image(384, 512, [0; 3]);@@ -1048,7 +1276,7 @@ mod tests {pixels.data[value * 3..value * 3 + 3].fill(value as u8);}let expected = pixels.data.clone();- render(&mut pixels, None, None, true);+ render(&mut pixels, None, None, [1.0; 3], true);assert_eq!(pixels.data, expected);}@@ -1176,7 +1404,7 @@ mod tests {- region_mean(&output, &fixture.regions, Region::Faint);assert!(faint_contrast > 10.0);assert!(region_mean(&output, &fixture.regions, Region::Ink) < 42.0);- assert!(meaningful_color(&output, None));+ assert!(has_color(&output, None));}#[test]@@ -1192,7 +1420,7 @@ mod tests {.iter().all(|pixel| pixel.iter().zip(original).all(|(&a, b)| a.abs_diff(b) <= 1)));- assert!(meaningful_color(&page, None));+ assert!(has_color(&page, None));}}@@ -1258,8 +1486,14 @@ mod tests {let analysis = analyze(&output, None, long_edge);samples = analysis.samples.len();let field = estimate_field(&analysis);- render(&mut output, None, field.as_ref(), false);- assert!(meaningful_color(&output, None));+ render(+ &mut output,+ None,+ field.as_ref(),+ paper_adaptation(&analysis),+ false,+ );+ assert!(has_color(&output, None));elapsed.push(start.elapsed().as_secs_f64() * 1000.0);}elapsed.sort_unstable_by(f64::total_cmp);