Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Fix ICC LUT profiles, JXL orientation, and EP self-test thresholds

laurenspriem522f15ba556f
diff --git a/rust/crates/image/src/color_management.rs b/rust/crates/image/src/color_management.rs
index b404eb1669..3434f33eba 100644
--- a/rust/crates/image/src/color_management.rs
+++ b/rust/crates/image/src/color_management.rs
@@ -149,9 +149,22 @@ impl TransformSubpixel for u8 {
encoded_profile: &[u8],
layout: Layout,
) -> Result<Option<Vec<Self>>, String> {
- let transform = cached_u8_transform(encoded_profile, source_profile, layout)?;
- transform.transform(pixels).map_err(|err| err.to_string())?;
- Ok(None)
+ // A2B LUTs take ICC precedence over matrix/TRC, and the in-place
+ // executor only implements matrix math, so LUT-carrying profiles go
+ // out-of-place; the matrix path is the fallback when the LUT does
+ // not cover the default intent (moxcms has no cross-intent fallback).
+ if profile_has_device_to_pcs_lut(source_profile)
+ && let Ok(transformed) = transform_u8_out_of_place(pixels, source_profile, layout)
+ {
+ return Ok(Some(transformed));
+ }
+
+ if let Ok(transform) = cached_u8_transform(encoded_profile, source_profile, layout) {
+ transform.transform(pixels).map_err(|err| err.to_string())?;
+ return Ok(None);
+ }
+
+ transform_u8_out_of_place(pixels, source_profile, layout).map(Some)
}
}
@@ -287,6 +300,29 @@ fn cache_u8_transform(encoded: &[u8], layout: Layout, transform: U8Transform) ->
transform
}
+fn transform_u8_out_of_place(
+ pixels: &[u8],
+ source_profile: &ColorProfile,
+ layout: Layout,
+) -> Result<Vec<u8>, String> {
+ let target_profile = target_profile_for_layout(layout);
+ let transform = source_profile
+ .create_transform_8bit(layout, target_profile, layout, transform_options())
+ .map_err(|err| err.to_string())?;
+ let mut transformed = vec![0; pixels.len()];
+ transform
+ .transform(pixels, &mut transformed)
+ .map_err(|err| err.to_string())?;
+ Ok(transformed)
+}
+
+/// Mirrors moxcms's crate-private `ColorProfile::has_device_to_pcs_lut`.
+fn profile_has_device_to_pcs_lut(profile: &ColorProfile) -> bool {
+ profile.lut_a_to_b_perceptual.is_some()
+ || profile.lut_a_to_b_colorimetric.is_some()
+ || profile.lut_a_to_b_saturation.is_some()
+}
+
fn profile_is_cacheable(encoded: &[u8]) -> bool {
!encoded.is_empty() && encoded.len() <= MAX_CACHEABLE_ICC_PROFILE_BYTES
}
@@ -450,10 +486,39 @@ fn tone_curve_matches_reference(
#[cfg(test)]
mod tests {
use image::{DynamicImage, ImageBuffer};
- use moxcms::{ColorProfile, Layout, TransformOptions};
+ use moxcms::{
+ ColorProfile, Layout, LutMultidimensionalType, LutWarehouse, Matrix3d, ToneReprCurve,
+ TransformOptions, Vector3d,
+ };
use super::{apply_icc_profile_to_srgb, profile_is_effectively_srgb};
+ fn identity_a_to_b_lut() -> LutWarehouse {
+ let identity_curve = || ToneReprCurve::Lut(vec![0, u16::MAX]);
+ LutWarehouse::Multidimensional(LutMultidimensionalType {
+ num_input_channels: 3,
+ num_output_channels: 3,
+ grid_points: [0; 16],
+ clut: None,
+ a_curves: Vec::new(),
+ b_curves: vec![identity_curve(), identity_curve(), identity_curve()],
+ m_curves: Vec::new(),
+ matrix: Matrix3d::IDENTITY,
+ bias: Vector3d::default(),
+ })
+ }
+
+ /// An RGB profile that converts only through an A2B LUT (no TRC tags),
+ /// like calibrated display/scanner profiles.
+ fn lut_based_rgb_profile() -> ColorProfile {
+ let mut profile = ColorProfile::new_display_p3();
+ profile.red_trc = None;
+ profile.green_trc = None;
+ profile.blue_trc = None;
+ profile.lut_a_to_b_perceptual = Some(identity_a_to_b_lut());
+ profile
+ }
+
#[test]
fn detects_generated_srgb_profile() {
let srgb = ColorProfile::new_srgb();
@@ -601,6 +666,133 @@ mod tests {
assert_eq!(actual, expected);
}
+ /// A profile with matrix/TRC AND A2B LUT tags must be converted via the
+ /// LUT (ICC precedence), even though the in-place constructor accepts it.
+ #[test]
+ fn profile_with_matrix_and_lut_tags_is_transformed_via_the_lut() {
+ let mut profile = ColorProfile::new_display_p3();
+ profile.lut_a_to_b_perceptual = Some(identity_a_to_b_lut());
+ let icc = profile.encode().unwrap();
+ let parsed = ColorProfile::new_from_slice(&icc).unwrap();
+ let pixels = vec![0, 32, 64, 96, 128, 160, 192, 224, 255, 17, 91, 203];
+
+ let in_place = parsed
+ .create_in_place_transform_8bit(
+ Layout::Rgb,
+ &ColorProfile::new_srgb(),
+ TransformOptions::default(),
+ )
+ .expect("the in-place constructor accepts matrix-shaper profiles with LUT tags");
+ let mut matrix_result = pixels.clone();
+ in_place.transform(&mut matrix_result).unwrap();
+
+ let mut expected = vec![0; pixels.len()];
+ parsed
+ .create_transform_8bit(
+ Layout::Rgb,
+ &ColorProfile::new_srgb(),
+ Layout::Rgb,
+ TransformOptions::default(),
+ )
+ .unwrap()
+ .transform(&pixels, &mut expected)
+ .unwrap();
+ assert_ne!(
+ expected, matrix_result,
+ "the LUT and matrix paths must disagree for this test to discriminate"
+ );
+
+ let image =
+ DynamicImage::ImageRgb8(ImageBuffer::from_raw(4, 1, pixels).expect("valid RGB image"));
+
+ let actual = apply_icc_profile_to_srgb(image, Some(&icc))
+ .into_rgb8()
+ .into_raw();
+
+ assert_eq!(actual, expected);
+ }
+
+ /// A colorimetric-only A2B LUT (no A2B0) is unusable under moxcms's
+ /// default Perceptual intent; the matrix path must still convert.
+ #[test]
+ fn profile_with_unusable_lut_intent_falls_back_to_the_matrix_transform() {
+ let mut profile = ColorProfile::new_display_p3();
+ profile.lut_a_to_b_colorimetric = Some(identity_a_to_b_lut());
+ let icc = profile.encode().unwrap();
+ let parsed = ColorProfile::new_from_slice(&icc).unwrap();
+ let pixels = vec![0, 32, 64, 96, 128, 160, 192, 224, 255, 17, 91, 203];
+ assert!(
+ parsed
+ .create_transform_8bit(
+ Layout::Rgb,
+ &ColorProfile::new_srgb(),
+ Layout::Rgb,
+ TransformOptions::default(),
+ )
+ .is_err(),
+ "the out-of-place transform must fail for this test to cover the fallback"
+ );
+ let mut expected = pixels.clone();
+ parsed
+ .create_in_place_transform_8bit(
+ Layout::Rgb,
+ &ColorProfile::new_srgb(),
+ TransformOptions::default(),
+ )
+ .unwrap()
+ .transform(&mut expected)
+ .unwrap();
+ assert_ne!(expected, pixels);
+ let image =
+ DynamicImage::ImageRgb8(ImageBuffer::from_raw(4, 1, pixels).expect("valid RGB image"));
+
+ let actual = apply_icc_profile_to_srgb(image, Some(&icc))
+ .into_rgb8()
+ .into_raw();
+
+ assert_eq!(actual, expected);
+ }
+
+ #[test]
+ fn lut_profile_rgb8_falls_back_to_out_of_place_transform() {
+ let lut_icc = lut_based_rgb_profile().encode().unwrap();
+ let parsed = ColorProfile::new_from_slice(&lut_icc).unwrap();
+ assert!(
+ parsed
+ .create_in_place_transform_8bit(
+ Layout::Rgb,
+ &ColorProfile::new_srgb(),
+ TransformOptions::default(),
+ )
+ .is_err(),
+ "profile must be rejected by the in-place executor for this test to cover the fallback"
+ );
+ let pixels = vec![0, 32, 64, 96, 128, 160, 192, 224, 255, 17, 91, 203];
+ let mut expected = vec![0; pixels.len()];
+ parsed
+ .create_transform_8bit(
+ Layout::Rgb,
+ &ColorProfile::new_srgb(),
+ Layout::Rgb,
+ TransformOptions::default(),
+ )
+ .unwrap()
+ .transform(&pixels, &mut expected)
+ .unwrap();
+ assert_ne!(
+ expected, pixels,
+ "the LUT transform must actually change pixel values"
+ );
+ let image =
+ DynamicImage::ImageRgb8(ImageBuffer::from_raw(4, 1, pixels).expect("valid RGB image"));
+
+ let actual = apply_icc_profile_to_srgb(image, Some(&lut_icc))
+ .into_rgb8()
+ .into_raw();
+
+ assert_eq!(actual, expected);
+ }
+
#[test]
fn rgba16_transform_matches_previous_out_of_place_result() {
let display_p3 = ColorProfile::new_display_p3();
diff --git a/rust/crates/image/src/decode.rs b/rust/crates/image/src/decode.rs
index aa49a0828c..17719a3568 100644
--- a/rust/crates/image/src/decode.rs
+++ b/rust/crates/image/src/decode.rs
@@ -1,7 +1,7 @@
use std::{
ffi::OsStr,
fs::File,
- io::{BufRead, BufReader, Cursor, Seek},
+ io::{BufRead, BufReader, Cursor, Read, Seek},
path::Path,
sync::Once,
};
@@ -362,6 +362,9 @@ fn orient_decoded_image(
if path_extension_is_heif(path) {
return apply_heif_exif_orientation_hint(image, path);
}
+ if file_looks_like_jxl(path) {
+ return image;
+ }
match decoder_orientation {
Some(orientation) => apply_exif_orientation_dynamic(image, orientation.to_exif()),
@@ -377,6 +380,9 @@ fn orient_decoded_image_from_bytes(
if bytes_look_like_heif(image_bytes) {
return apply_heif_exif_orientation_hint_from_bytes(image, image_bytes);
}
+ if bytes_look_like_jxl(image_bytes) {
+ return image;
+ }
match decoder_orientation {
Some(orientation) => apply_exif_orientation_dynamic(image, orientation.to_exif()),
@@ -455,6 +461,30 @@ fn read_exif_orientation_from_bytes(image_bytes: &[u8]) -> Option<u8> {
.filter(|value| (1..=8).contains(value))
}
+/// JPEG XL must never have an EXIF-derived orientation applied: jxl-oxide
+/// already applies the authoritative codestream orientation (ISO/IEC
+/// 18181-2), and encoders keep a redundant EXIF tag in the container.
+fn bytes_look_like_jxl(image_bytes: &[u8]) -> bool {
+ const BARE_CODESTREAM_SIGNATURE: [u8; 2] = [0xFF, 0x0A];
+ const CONTAINER_SIGNATURE: [u8; 12] = [
+ 0x00, 0x00, 0x00, 0x0C, b'J', b'X', b'L', b' ', 0x0D, 0x0A, 0x87, 0x0A,
+ ];
+
+ image_bytes.starts_with(&BARE_CODESTREAM_SIGNATURE)
+ || image_bytes.starts_with(&CONTAINER_SIGNATURE)
+}
+
+fn file_looks_like_jxl(path: &Path) -> bool {
+ let Ok(file) = File::open(path) else {
+ return false;
+ };
+ let mut signature = Vec::with_capacity(12);
+ if file.take(12).read_to_end(&mut signature).is_err() {
+ return false;
+ }
+ bytes_look_like_jxl(&signature)
+}
+
fn bytes_look_like_heif(image_bytes: &[u8]) -> bool {
if image_bytes.len() < 12 || &image_bytes[4..8] != b"ftyp" {
return false;
@@ -556,6 +586,15 @@ mod tests {
assert_eq!(decoded.rgb, vec![128, 64, 32]);
}
+ #[test]
+ fn detects_jxl_signatures() {
+ assert!(super::bytes_look_like_jxl(&[0xFF, 0x0A, 0x00]));
+ assert!(super::bytes_look_like_jxl(&[
+ 0x00, 0x00, 0x00, 0x0C, b'J', b'X', b'L', b' ', 0x0D, 0x0A, 0x87, 0x0A, 0x00
+ ]));
+ assert!(!super::bytes_look_like_jxl(b"not an image"));
+ }
+
#[test]
fn decode_applies_orientation_reported_by_png_decoder() {
let mut encoded = Vec::new();
diff --git a/rust/crates/photos/src/ml/golden.rs b/rust/crates/photos/src/ml/golden.rs
index f32b9fe5d1..9aaf7d84cb 100644
--- a/rust/crates/photos/src/ml/golden.rs
+++ b/rust/crates/photos/src/ml/golden.rs
@@ -20,22 +20,24 @@
use crate::ml::golden_data::GOLDEN_ENTRIES;
+// All thresholds are deliberately loose (a false rejection silently degrades
+// the device to CPU) and unvalidated against real accelerated hardware;
+// tighten them once field measurements exist. Corrupted output measures ~1.0
+// on every metric.
+
/// Maximum cosine distance between an accelerated session's embedding output
-/// and the CPU-generated golden output before the session is rejected.
-/// Healthy fp16 provider divergence is around 1e-3; corrupted output is
-/// orders of magnitude beyond this (random vectors sit near 1.0). The
-/// threshold is deliberately loose β€” a false rejection silently degrades the
-/// device to CPU, which is worse than letting marginal-but-healthy numerics
-/// through.
+/// and the CPU golden. Healthy fp16 divergence is around 1e-3.
pub(crate) const COSINE_DISTANCE_THRESHOLD: f64 = 0.025;
-/// Maximum relative L2 error (`|actual - expected| / |expected|`) for each
-/// detector output channel group. Unlike cosine distance this also rejects
-/// magnitude corruption, which matters for outputs whose absolute values are
-/// consumed directly (box coordinates, scores). Healthy fp16 divergence is
-/// around 1e-3 here too; the same loose-threshold rationale as for cosine
-/// distance applies.
-pub(crate) const RELATIVE_L2_THRESHOLD: f64 = 0.025;
+/// Maximum relative L2 error for the detector's non-confidence outputs (box
+/// coordinates, landmarks), whose absolute values are consumed directly.
+/// Well-conditioned: healthy fp16 divergence is around 1e-3.
+pub(crate) const RELATIVE_L2_THRESHOLD: f64 = 0.1;
+
+/// Maximum relative L2 error for the detector's confidence group, which sits
+/// at the fp16 noise floor on the no-face noise input (samples <= ~2e-3), so
+/// healthy relative divergence can plausibly reach a few 1e-2.
+pub(crate) const CONFIDENCE_RELATIVE_L2_THRESHOLD: f64 = 0.2;
/// Reference outputs are compared on a deterministic strided sample capped at
/// this many elements, so large detector outputs stay small in the committed
@@ -73,11 +75,12 @@ impl GoldenMetric {
}
}
- /// Maximum accepted distance from the committed CPU golden.
+ /// The loosest accepted distance; separation checks must measure beyond
+ /// this. [`compare_output`] applies the per-group thresholds.
pub fn threshold(self) -> f64 {
match self {
Self::CosineDistance => COSINE_DISTANCE_THRESHOLD,
- Self::DetectorRelativeL2 { .. } => RELATIVE_L2_THRESHOLD,
+ Self::DetectorRelativeL2 { .. } => CONFIDENCE_RELATIVE_L2_THRESHOLD,
}
}
}
@@ -231,7 +234,7 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String
ensure_within_threshold(
"detector confidence relative L2 error",
confidence,
- RELATIVE_L2_THRESHOLD,
+ CONFIDENCE_RELATIVE_L2_THRESHOLD,
)?;
ensure_within_threshold(
"detector remaining-output relative L2 error",
@@ -651,6 +654,49 @@ mod tests {
assert!(error.contains("detector confidence"), "{error}");
}
+ /// Coordinate corruption must fail even where the looser confidence
+ /// threshold would accept it.
+ #[test]
+ fn detector_metric_keeps_the_strict_threshold_for_coordinates() {
+ static EXPECTED: [f32; 32] = [
+ 100.0, 100.0, 20.0, 20.0, 0.001, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0,
+ 19.0, 0.98, 200.0, 200.0, 40.0, 40.0, 0.002, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0,
+ 27.0, 28.0, 29.0, 0.99,
+ ];
+ let entry = detector_entry(&EXPECTED);
+ let mut output = EXPECTED;
+ for (index, value) in output.iter_mut().enumerate() {
+ if index % 16 != 4 {
+ *value *= 1.25;
+ }
+ }
+
+ let error = compare_output(&entry, &output).unwrap_err();
+ assert!(error.contains("remaining"), "{error}");
+ }
+
+ /// A confidence-group deviation beyond the coordinate threshold must
+ /// still be accepted (fp16 noise floor).
+ #[test]
+ fn detector_metric_tolerates_noise_floor_divergence_on_confidences() {
+ static EXPECTED: [f32; 32] = [
+ 100.0, 100.0, 20.0, 20.0, 0.001, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0,
+ 19.0, 0.98, 200.0, 200.0, 40.0, 40.0, 0.002, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0,
+ 27.0, 28.0, 29.0, 0.99,
+ ];
+ let entry = detector_entry(&EXPECTED);
+ let mut output = EXPECTED;
+ for confidence in output.iter_mut().skip(4).step_by(16) {
+ *confidence *= 1.15;
+ }
+
+ let distance = compare_output(&entry, &output).unwrap();
+ assert!(
+ distance > super::RELATIVE_L2_THRESHOLD,
+ "distance {distance} must exceed the coordinate threshold to discriminate"
+ );
+ }
+
#[test]
fn samples_large_outputs_with_a_deterministic_stride() {
let output: Vec<f32> = (0..10_000).map(|index| index as f32).collect();
diff --git a/rust/crates/photos/src/ml/preprocess.rs b/rust/crates/photos/src/ml/preprocess.rs
index f75b6fdd76..c4c5919c85 100644
--- a/rust/crates/photos/src/ml/preprocess.rs
+++ b/rust/crates/photos/src/ml/preprocess.rs
@@ -348,8 +348,7 @@ mod tests {
for x in 0..CLIP_IMAGE_INPUT_SIZE {
let source = ((start_y + y) * scaled_width as usize + start_x + x) * 3;
let destination = y * CLIP_IMAGE_INPUT_SIZE + x;
- output[destination] =
- resized.buffer()[source] as f32 * super::NORMALIZATION_SCALE;
+ output[destination] = resized.buffer()[source] as f32 * super::NORMALIZATION_SCALE;
output[pixel_count + destination] =
resized.buffer()[source + 1] as f32 * super::NORMALIZATION_SCALE;
output[2 * pixel_count + destination] =