π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Fix ICC LUT profiles, JXL orientation, and EP self-test thresholds
laurenspriem522f15ba556f
diff --git a/rust/crates/image/src/color_management.rs b/rust/crates/image/src/color_management.rsindex b404eb1669..3434f33eba 100644--- a/rust/crates/image/src/color_management.rs+++ b/rust/crates/image/src/color_management.rs@@ -149,9 +149,22 @@ impl TransformSubpixel for u8 {encoded_profile: &[u8],layout: Layout,) -> Result<Option<Vec<Self>>, String> {- let transform = cached_u8_transform(encoded_profile, source_profile, layout)?;- transform.transform(pixels).map_err(|err| err.to_string())?;- Ok(None)+ // A2B LUTs take ICC precedence over matrix/TRC, and the in-place+ // executor only implements matrix math, so LUT-carrying profiles go+ // out-of-place; the matrix path is the fallback when the LUT does+ // not cover the default intent (moxcms has no cross-intent fallback).+ if profile_has_device_to_pcs_lut(source_profile)+ && let Ok(transformed) = transform_u8_out_of_place(pixels, source_profile, layout)+ {+ return Ok(Some(transformed));+ }++ if let Ok(transform) = cached_u8_transform(encoded_profile, source_profile, layout) {+ transform.transform(pixels).map_err(|err| err.to_string())?;+ return Ok(None);+ }++ transform_u8_out_of_place(pixels, source_profile, layout).map(Some)}}@@ -287,6 +300,29 @@ fn cache_u8_transform(encoded: &[u8], layout: Layout, transform: U8Transform) ->transform}+fn transform_u8_out_of_place(+ pixels: &[u8],+ source_profile: &ColorProfile,+ layout: Layout,+) -> Result<Vec<u8>, String> {+ let target_profile = target_profile_for_layout(layout);+ let transform = source_profile+ .create_transform_8bit(layout, target_profile, layout, transform_options())+ .map_err(|err| err.to_string())?;+ let mut transformed = vec![0; pixels.len()];+ transform+ .transform(pixels, &mut transformed)+ .map_err(|err| err.to_string())?;+ Ok(transformed)+}++/// Mirrors moxcms's crate-private `ColorProfile::has_device_to_pcs_lut`.+fn profile_has_device_to_pcs_lut(profile: &ColorProfile) -> bool {+ profile.lut_a_to_b_perceptual.is_some()+ || profile.lut_a_to_b_colorimetric.is_some()+ || profile.lut_a_to_b_saturation.is_some()+}+fn profile_is_cacheable(encoded: &[u8]) -> bool {!encoded.is_empty() && encoded.len() <= MAX_CACHEABLE_ICC_PROFILE_BYTES}@@ -450,10 +486,39 @@ fn tone_curve_matches_reference(#[cfg(test)]mod tests {use image::{DynamicImage, ImageBuffer};- use moxcms::{ColorProfile, Layout, TransformOptions};+ use moxcms::{+ ColorProfile, Layout, LutMultidimensionalType, LutWarehouse, Matrix3d, ToneReprCurve,+ TransformOptions, Vector3d,+ };use super::{apply_icc_profile_to_srgb, profile_is_effectively_srgb};+ fn identity_a_to_b_lut() -> LutWarehouse {+ let identity_curve = || ToneReprCurve::Lut(vec![0, u16::MAX]);+ LutWarehouse::Multidimensional(LutMultidimensionalType {+ num_input_channels: 3,+ num_output_channels: 3,+ grid_points: [0; 16],+ clut: None,+ a_curves: Vec::new(),+ b_curves: vec![identity_curve(), identity_curve(), identity_curve()],+ m_curves: Vec::new(),+ matrix: Matrix3d::IDENTITY,+ bias: Vector3d::default(),+ })+ }++ /// An RGB profile that converts only through an A2B LUT (no TRC tags),+ /// like calibrated display/scanner profiles.+ fn lut_based_rgb_profile() -> ColorProfile {+ let mut profile = ColorProfile::new_display_p3();+ profile.red_trc = None;+ profile.green_trc = None;+ profile.blue_trc = None;+ profile.lut_a_to_b_perceptual = Some(identity_a_to_b_lut());+ profile+ }+#[test]fn detects_generated_srgb_profile() {let srgb = ColorProfile::new_srgb();@@ -601,6 +666,133 @@ mod tests {assert_eq!(actual, expected);}+ /// A profile with matrix/TRC AND A2B LUT tags must be converted via the+ /// LUT (ICC precedence), even though the in-place constructor accepts it.+ #[test]+ fn profile_with_matrix_and_lut_tags_is_transformed_via_the_lut() {+ let mut profile = ColorProfile::new_display_p3();+ profile.lut_a_to_b_perceptual = Some(identity_a_to_b_lut());+ let icc = profile.encode().unwrap();+ let parsed = ColorProfile::new_from_slice(&icc).unwrap();+ let pixels = vec![0, 32, 64, 96, 128, 160, 192, 224, 255, 17, 91, 203];++ let in_place = parsed+ .create_in_place_transform_8bit(+ Layout::Rgb,+ &ColorProfile::new_srgb(),+ TransformOptions::default(),+ )+ .expect("the in-place constructor accepts matrix-shaper profiles with LUT tags");+ let mut matrix_result = pixels.clone();+ in_place.transform(&mut matrix_result).unwrap();++ let mut expected = vec![0; pixels.len()];+ parsed+ .create_transform_8bit(+ Layout::Rgb,+ &ColorProfile::new_srgb(),+ Layout::Rgb,+ TransformOptions::default(),+ )+ .unwrap()+ .transform(&pixels, &mut expected)+ .unwrap();+ assert_ne!(+ expected, matrix_result,+ "the LUT and matrix paths must disagree for this test to discriminate"+ );++ let image =+ DynamicImage::ImageRgb8(ImageBuffer::from_raw(4, 1, pixels).expect("valid RGB image"));++ let actual = apply_icc_profile_to_srgb(image, Some(&icc))+ .into_rgb8()+ .into_raw();++ assert_eq!(actual, expected);+ }++ /// A colorimetric-only A2B LUT (no A2B0) is unusable under moxcms's+ /// default Perceptual intent; the matrix path must still convert.+ #[test]+ fn profile_with_unusable_lut_intent_falls_back_to_the_matrix_transform() {+ let mut profile = ColorProfile::new_display_p3();+ profile.lut_a_to_b_colorimetric = Some(identity_a_to_b_lut());+ let icc = profile.encode().unwrap();+ let parsed = ColorProfile::new_from_slice(&icc).unwrap();+ let pixels = vec![0, 32, 64, 96, 128, 160, 192, 224, 255, 17, 91, 203];+ assert!(+ parsed+ .create_transform_8bit(+ Layout::Rgb,+ &ColorProfile::new_srgb(),+ Layout::Rgb,+ TransformOptions::default(),+ )+ .is_err(),+ "the out-of-place transform must fail for this test to cover the fallback"+ );+ let mut expected = pixels.clone();+ parsed+ .create_in_place_transform_8bit(+ Layout::Rgb,+ &ColorProfile::new_srgb(),+ TransformOptions::default(),+ )+ .unwrap()+ .transform(&mut expected)+ .unwrap();+ assert_ne!(expected, pixels);+ let image =+ DynamicImage::ImageRgb8(ImageBuffer::from_raw(4, 1, pixels).expect("valid RGB image"));++ let actual = apply_icc_profile_to_srgb(image, Some(&icc))+ .into_rgb8()+ .into_raw();++ assert_eq!(actual, expected);+ }++ #[test]+ fn lut_profile_rgb8_falls_back_to_out_of_place_transform() {+ let lut_icc = lut_based_rgb_profile().encode().unwrap();+ let parsed = ColorProfile::new_from_slice(&lut_icc).unwrap();+ assert!(+ parsed+ .create_in_place_transform_8bit(+ Layout::Rgb,+ &ColorProfile::new_srgb(),+ TransformOptions::default(),+ )+ .is_err(),+ "profile must be rejected by the in-place executor for this test to cover the fallback"+ );+ let pixels = vec![0, 32, 64, 96, 128, 160, 192, 224, 255, 17, 91, 203];+ let mut expected = vec![0; pixels.len()];+ parsed+ .create_transform_8bit(+ Layout::Rgb,+ &ColorProfile::new_srgb(),+ Layout::Rgb,+ TransformOptions::default(),+ )+ .unwrap()+ .transform(&pixels, &mut expected)+ .unwrap();+ assert_ne!(+ expected, pixels,+ "the LUT transform must actually change pixel values"+ );+ let image =+ DynamicImage::ImageRgb8(ImageBuffer::from_raw(4, 1, pixels).expect("valid RGB image"));++ let actual = apply_icc_profile_to_srgb(image, Some(&lut_icc))+ .into_rgb8()+ .into_raw();++ assert_eq!(actual, expected);+ }+#[test]fn rgba16_transform_matches_previous_out_of_place_result() {let display_p3 = ColorProfile::new_display_p3();diff --git a/rust/crates/image/src/decode.rs b/rust/crates/image/src/decode.rsindex aa49a0828c..17719a3568 100644--- a/rust/crates/image/src/decode.rs+++ b/rust/crates/image/src/decode.rs@@ -1,7 +1,7 @@use std::{ffi::OsStr,fs::File,- io::{BufRead, BufReader, Cursor, Seek},+ io::{BufRead, BufReader, Cursor, Read, Seek},path::Path,sync::Once,};@@ -362,6 +362,9 @@ fn orient_decoded_image(if path_extension_is_heif(path) {return apply_heif_exif_orientation_hint(image, path);}+ if file_looks_like_jxl(path) {+ return image;+ }match decoder_orientation {Some(orientation) => apply_exif_orientation_dynamic(image, orientation.to_exif()),@@ -377,6 +380,9 @@ fn orient_decoded_image_from_bytes(if bytes_look_like_heif(image_bytes) {return apply_heif_exif_orientation_hint_from_bytes(image, image_bytes);}+ if bytes_look_like_jxl(image_bytes) {+ return image;+ }match decoder_orientation {Some(orientation) => apply_exif_orientation_dynamic(image, orientation.to_exif()),@@ -455,6 +461,30 @@ fn read_exif_orientation_from_bytes(image_bytes: &[u8]) -> Option<u8> {.filter(|value| (1..=8).contains(value))}+/// JPEG XL must never have an EXIF-derived orientation applied: jxl-oxide+/// already applies the authoritative codestream orientation (ISO/IEC+/// 18181-2), and encoders keep a redundant EXIF tag in the container.+fn bytes_look_like_jxl(image_bytes: &[u8]) -> bool {+ const BARE_CODESTREAM_SIGNATURE: [u8; 2] = [0xFF, 0x0A];+ const CONTAINER_SIGNATURE: [u8; 12] = [+ 0x00, 0x00, 0x00, 0x0C, b'J', b'X', b'L', b' ', 0x0D, 0x0A, 0x87, 0x0A,+ ];++ image_bytes.starts_with(&BARE_CODESTREAM_SIGNATURE)+ || image_bytes.starts_with(&CONTAINER_SIGNATURE)+}++fn file_looks_like_jxl(path: &Path) -> bool {+ let Ok(file) = File::open(path) else {+ return false;+ };+ let mut signature = Vec::with_capacity(12);+ if file.take(12).read_to_end(&mut signature).is_err() {+ return false;+ }+ bytes_look_like_jxl(&signature)+}+fn bytes_look_like_heif(image_bytes: &[u8]) -> bool {if image_bytes.len() < 12 || &image_bytes[4..8] != b"ftyp" {return false;@@ -556,6 +586,15 @@ mod tests {assert_eq!(decoded.rgb, vec![128, 64, 32]);}+ #[test]+ fn detects_jxl_signatures() {+ assert!(super::bytes_look_like_jxl(&[0xFF, 0x0A, 0x00]));+ assert!(super::bytes_look_like_jxl(&[+ 0x00, 0x00, 0x00, 0x0C, b'J', b'X', b'L', b' ', 0x0D, 0x0A, 0x87, 0x0A, 0x00+ ]));+ assert!(!super::bytes_look_like_jxl(b"not an image"));+ }+#[test]fn decode_applies_orientation_reported_by_png_decoder() {let mut encoded = Vec::new();diff --git a/rust/crates/photos/src/ml/golden.rs b/rust/crates/photos/src/ml/golden.rsindex f32b9fe5d1..9aaf7d84cb 100644--- a/rust/crates/photos/src/ml/golden.rs+++ b/rust/crates/photos/src/ml/golden.rs@@ -20,22 +20,24 @@use crate::ml::golden_data::GOLDEN_ENTRIES;+// All thresholds are deliberately loose (a false rejection silently degrades+// the device to CPU) and unvalidated against real accelerated hardware;+// tighten them once field measurements exist. Corrupted output measures ~1.0+// on every metric.+/// Maximum cosine distance between an accelerated session's embedding output-/// and the CPU-generated golden output before the session is rejected.-/// Healthy fp16 provider divergence is around 1e-3; corrupted output is-/// orders of magnitude beyond this (random vectors sit near 1.0). The-/// threshold is deliberately loose β a false rejection silently degrades the-/// device to CPU, which is worse than letting marginal-but-healthy numerics-/// through.+/// and the CPU golden. Healthy fp16 divergence is around 1e-3.pub(crate) const COSINE_DISTANCE_THRESHOLD: f64 = 0.025;-/// Maximum relative L2 error (`|actual - expected| / |expected|`) for each-/// detector output channel group. Unlike cosine distance this also rejects-/// magnitude corruption, which matters for outputs whose absolute values are-/// consumed directly (box coordinates, scores). Healthy fp16 divergence is-/// around 1e-3 here too; the same loose-threshold rationale as for cosine-/// distance applies.-pub(crate) const RELATIVE_L2_THRESHOLD: f64 = 0.025;+/// Maximum relative L2 error for the detector's non-confidence outputs (box+/// coordinates, landmarks), whose absolute values are consumed directly.+/// Well-conditioned: healthy fp16 divergence is around 1e-3.+pub(crate) const RELATIVE_L2_THRESHOLD: f64 = 0.1;++/// Maximum relative L2 error for the detector's confidence group, which sits+/// at the fp16 noise floor on the no-face noise input (samples <= ~2e-3), so+/// healthy relative divergence can plausibly reach a few 1e-2.+pub(crate) const CONFIDENCE_RELATIVE_L2_THRESHOLD: f64 = 0.2;/// Reference outputs are compared on a deterministic strided sample capped at/// this many elements, so large detector outputs stay small in the committed@@ -73,11 +75,12 @@ impl GoldenMetric {}}- /// Maximum accepted distance from the committed CPU golden.+ /// The loosest accepted distance; separation checks must measure beyond+ /// this. [`compare_output`] applies the per-group thresholds.pub fn threshold(self) -> f64 {match self {Self::CosineDistance => COSINE_DISTANCE_THRESHOLD,- Self::DetectorRelativeL2 { .. } => RELATIVE_L2_THRESHOLD,+ Self::DetectorRelativeL2 { .. } => CONFIDENCE_RELATIVE_L2_THRESHOLD,}}}@@ -231,7 +234,7 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, Stringensure_within_threshold("detector confidence relative L2 error",confidence,- RELATIVE_L2_THRESHOLD,+ CONFIDENCE_RELATIVE_L2_THRESHOLD,)?;ensure_within_threshold("detector remaining-output relative L2 error",@@ -651,6 +654,49 @@ mod tests {assert!(error.contains("detector confidence"), "{error}");}+ /// Coordinate corruption must fail even where the looser confidence+ /// threshold would accept it.+ #[test]+ fn detector_metric_keeps_the_strict_threshold_for_coordinates() {+ static EXPECTED: [f32; 32] = [+ 100.0, 100.0, 20.0, 20.0, 0.001, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0,+ 19.0, 0.98, 200.0, 200.0, 40.0, 40.0, 0.002, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0,+ 27.0, 28.0, 29.0, 0.99,+ ];+ let entry = detector_entry(&EXPECTED);+ let mut output = EXPECTED;+ for (index, value) in output.iter_mut().enumerate() {+ if index % 16 != 4 {+ *value *= 1.25;+ }+ }++ let error = compare_output(&entry, &output).unwrap_err();+ assert!(error.contains("remaining"), "{error}");+ }++ /// A confidence-group deviation beyond the coordinate threshold must+ /// still be accepted (fp16 noise floor).+ #[test]+ fn detector_metric_tolerates_noise_floor_divergence_on_confidences() {+ static EXPECTED: [f32; 32] = [+ 100.0, 100.0, 20.0, 20.0, 0.001, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0,+ 19.0, 0.98, 200.0, 200.0, 40.0, 40.0, 0.002, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0,+ 27.0, 28.0, 29.0, 0.99,+ ];+ let entry = detector_entry(&EXPECTED);+ let mut output = EXPECTED;+ for confidence in output.iter_mut().skip(4).step_by(16) {+ *confidence *= 1.15;+ }++ let distance = compare_output(&entry, &output).unwrap();+ assert!(+ distance > super::RELATIVE_L2_THRESHOLD,+ "distance {distance} must exceed the coordinate threshold to discriminate"+ );+ }+#[test]fn samples_large_outputs_with_a_deterministic_stride() {let output: Vec<f32> = (0..10_000).map(|index| index as f32).collect();diff --git a/rust/crates/photos/src/ml/preprocess.rs b/rust/crates/photos/src/ml/preprocess.rsindex f75b6fdd76..c4c5919c85 100644--- a/rust/crates/photos/src/ml/preprocess.rs+++ b/rust/crates/photos/src/ml/preprocess.rs@@ -348,8 +348,7 @@ mod tests {for x in 0..CLIP_IMAGE_INPUT_SIZE {let source = ((start_y + y) * scaled_width as usize + start_x + x) * 3;let destination = y * CLIP_IMAGE_INPUT_SIZE + x;- output[destination] =- resized.buffer()[source] as f32 * super::NORMALIZATION_SCALE;+ output[destination] = resized.buffer()[source] as f32 * super::NORMALIZATION_SCALE;output[pixel_count + destination] =resized.buffer()[source + 1] as f32 * super::NORMALIZATION_SCALE;output[2 * pixel_count + destination] =