LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LR-PROF-CAMERACOLOR: fit the camera look without clipped sensor pixels
radialmonstercc7bf0e2fd9a
diff --git a/crates/engine/src/camera_preview.rs b/crates/engine/src/camera_preview.rsindex d6fdda14a..6bc2eeb91 100644--- a/crates/engine/src/camera_preview.rs+++ b/crates/engine/src/camera_preview.rs@@ -41,12 +41,12 @@ pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransfif !transform.matrix_is_fallback || !file_local_look(raw.format) {return None;}- let (sensor, reference) = proxies(raw, bytes, transform, PROXY)?;+ let (sensor, reference, clipped) = proxies(raw, bytes, transform, PROXY)?;// A camera profile pooled from many photos knows colours this photo shows too little of;// try its colour first and fit tone/chroma per photo (DRO and picture styles vary).let profile = raw.metadata.model.as_deref().and_then(crate::camera_profiles::get);let colour = profile.as_ref().and_then(|p| Some((p.matrix().mul(&transform.matrix.inverse()?), p.hue_sat.clone())));- let look = fit_pairs_with(&sensor, &reference, colour)?;+ let look = fit_look(&sensor, &reference, &clipped, colour)?;if lightcraft_pipeline::profiling() {eprintln!("[profile] {:?} camera look: {:?}, {:?}, hue/sat table {}, camera profile available {}",@@ -60,9 +60,73 @@ pub(crate) fn fit_preview(raw: &RawImage, bytes: &[u8], transform: &CameraTransfSome(look)}+/// Sensor level (normalised, white = 1) at or above which a raw channel counts as clipped, as in+/// the render's highlight reconstruction.+const SENSOR_CLIP: f32 = 0.99;++/// The photo's look, refitted without the proxy pixels whose sensor values are clipped.+///+/// A clipped raw pixel no longer records the scene's colour: with one channel held at the clip+/// level while the others keep rising, a white overcast sky comes out magenta or blue through the+/// camera matrix, while the camera JPEG shows it white. As correspondences such pixels teach the+/// colour matrix to pull colours toward grey, and the chroma curve to remove colour at their+/// display luminance (on a Z 6II forest under a clipped sky the curve cut colourfulness above+/// display luminance 0.4 to 13–19%, and every one of its samples there was clipped). The render+/// rebuilds clipped highlights separately (`lightcraft_raw::highlight`), so the look is fitted on+/// what the sensor measured.+///+/// The search on all pixels decides, as before, whether the photo gets a look and which attempt+/// of [`search_ordered`] gives it; only that attempt is refitted without the clipped pixels, and+/// when the refit misses the acceptance gates the look on all pixels is kept. So no photo loses+/// its look and none switches attempt (a refit that just clears the gate on all pixels must not+/// replace a look fitted away from edges). A photo that had no look gets one when the search+/// without the clipped pixels finds it.+fn fit_look(sensor: &Rgb32f, reference: &Rgb32f, clipped: &[bool], colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {+ let unclipped = without_clipped(sensor, clipped);+ let Some((look, attempt)) = search_ordered(sensor, reference, colour.clone(), TONE_FITS) else {+ return unclipped.and_then(|u| search_ordered(&u, reference, colour, TONE_FITS)).map(|(look, _)| look);+ };+ let Some(unclipped) = unclipped else { return Some(look) };+ let refit = fit_attempt(&unclipped, reference, &colour, attempt);+ if refit.is_none() && lightcraft_pipeline::profiling() {+ eprintln!("[profile] camera look: no fit without the clipped pixels on the accepted attempt, keeping the fit on all pixels");+ }+ Some(refit.unwrap_or(look))+}++/// `sensor` with its clipped pixels set to NaN (left out of every training pair), or `None` when+/// none is clipped.+fn without_clipped(sensor: &Rgb32f, clipped: &[bool]) -> Option<Rgb32f> {+ if clipped.len() != sensor.data.len() || !clipped.iter().any(|c| *c) {+ return None;+ }+ let mut out = sensor.clone();+ for (p, c) in out.data.iter_mut().zip(clipped) {+ if *c {+ *p = [f32::NAN; 3];+ }+ }+ if lightcraft_pipeline::profiling() {+ eprintln!(+ "[profile] camera look: {} of {} proxy pixels have a clipped sensor channel",+ clipped.iter().filter(|c| **c).count(),+ clipped.len()+ );+ }+ Some(out)+}++/// 1 where any channel of a camera-RGB pixel is at or above [`SENSOR_CLIP`], else 0.+fn clip_mask(camera: &Rgb32f) -> Rgb32f {+ let mut mask = camera.clone();+ mask.map_in_place(|p| if p.iter().any(|v| *v >= SENSOR_CLIP) { [1.0; 3] } else { [0.0; 3] });+ mask+}+/// Same-size proxies of the sensor (white-balanced, baseline exposure, through `transform`'s-/// matrix: the generic camera ≈ sRGB model) and of the file's embedded camera JPEG (linear Rec.2020).-fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usize) -> Option<(Rgb32f, Rgb32f)> {+/// matrix: the generic camera ≈ sRGB model) and of the file's embedded camera JPEG (linear Rec.2020),+/// and per proxy pixel whether any sensor sample it covers is clipped.+fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usize) -> Option<(Rgb32f, Rgb32f, Vec<bool>)> {let edge = (2 * size).max(384) as u32;let decoded = crate::files::decode_raw_preview(bytes, lightcraft_codecs::DecodeOptions { max_size: Some((edge, edge)), max_pixels: 64_000_000 })?;let mut reference = decoded.to_working();@@ -87,23 +151,28 @@ fn proxies(raw: &RawImage, bytes: &[u8], transform: &CameraTransform, size: usiz// Fixed, bounded proxy: the selected look cannot depend on thumbnail/export resolution.let k = (crop.width.max(crop.height).div_ceil(edge as usize).max(2)).div_ceil(2) * 2;let mut sensor = sensor_proxy(raw, k, edge as usize)?;+ let mut clipped = clip_mask(&sensor);let gain = 2f32.powf(transform.baseline_exposure as f32);let to_working = |p: [f32; 3]| transform.matrix.apply_f32(std::array::from_fn(|i| p[i] * transform.wb[i] * gain));if raw.format == RawFormat::Cr3 {// Orient the camera JPEG and the sensor identically before collecting pixel correspondences.sensor = sensor.into_oriented(raw.orientation);+ clipped = clipped.into_oriented(raw.orientation);reference = reference.into_oriented(raw.orientation);sensor.map_in_place(to_working);// Canon's JPEG can be cropped differently from the sensor. Estimate only that common// framing from edge directions, independently of the subsequent colour fit.- sensor = align_cr3_framing(sensor, &reference);+ (sensor, clipped) = align_cr3_framing(sensor, clipped, &reference);}let mut sensor = fit(&sensor, size, size, Filter::Box);let reference = fit(&reference, sensor.width, sensor.height, Filter::Box);+ // A proxy pixel is clipped when any sample it averages is (outside the CR3 framing the mask is+ // NaN: those pixels are left out of the pairs anyway).+ let clipped: Vec<bool> = fit(&clipped, sensor.width, sensor.height, Filter::Box).data.iter().map(|p| p[0] > 0.0).collect();if raw.format != RawFormat::Cr3 {sensor.map_in_place(to_working);}- Some((sensor, reference))+ Some((sensor, reference, clipped))}#[derive(Clone, Copy, Debug)]@@ -230,15 +299,19 @@ fn estimate_cr3_framing(sensor: &Rgb32f, reference: &Rgb32f) -> Option<Cr3FraminSome(best.1)}-fn align_cr3_framing(sensor: Rgb32f, reference: &Rgb32f) -> Rgb32f {- let Some(framing) = estimate_cr3_framing(&sensor, reference) else { return sensor };- let (w, h) = (sensor.width, sensor.height);- let mut aligned = Rgb32f::new(w, h);- for (i, pixel) in aligned.data.iter_mut().enumerate() {- let (x, y) = framing.source((i % w) as f32 + 0.5, (i / w) as f32 + 0.5, w, h);- *pixel = if x >= 0.5 && y >= 0.5 && x <= w as f32 - 0.5 && y <= h as f32 - 0.5 { sensor.sample_bilinear(x, y) } else { [f32::NAN; 3] };- }- aligned+/// The sensor proxy and its clip mask (same size), both moved to the camera JPEG's framing.+fn align_cr3_framing(sensor: Rgb32f, clipped: Rgb32f, reference: &Rgb32f) -> (Rgb32f, Rgb32f) {+ let Some(framing) = estimate_cr3_framing(&sensor, reference) else { return (sensor, clipped) };+ let apply = |image: &Rgb32f| {+ let (w, h) = (image.width, image.height);+ let mut aligned = Rgb32f::new(w, h);+ for (i, pixel) in aligned.data.iter_mut().enumerate() {+ let (x, y) = framing.source((i % w) as f32 + 0.5, (i / w) as f32 + 0.5, w, h);+ *pixel = if x >= 0.5 && y >= 0.5 && x <= w as f32 - 0.5 && y <= h as f32 - 0.5 { image.sample_bilinear(x, y) } else { [f32::NAN; 3] };+ }+ aligned+ };+ (apply(&sensor), apply(&clipped))}/// Colour training pairs of one raw for a camera profile: white-balanced camera RGB (with the@@ -249,8 +322,10 @@ pub(crate) fn profile_pairs(raw: &RawImage, bytes: &[u8]) -> Option<Vec<([f64; 3return None;}let transform = lightcraft_raw::color::camera_transform(raw, lightcraft_raw::color::as_shot_white_xy(raw));- let (sensor, reference) = proxies(raw, bytes, &transform, PROFILE_PROXY)?;+ let (sensor, reference, clipped) = proxies(raw, bytes, &transform, PROFILE_PROXY)?;let to_camera = transform.matrix.inverse()?;+ // clipped sensor pixels don't record the scene's colour (see `fit_look`)+ let sensor = without_clipped(&sensor, &clipped).unwrap_or(sensor);let (pairs, _) = collect_pairs(&sensor, &reference, 0.05, None)?;Some(pairs.into_iter().map(|(x, y)| (to_camera.apply(x), y)).collect())}@@ -263,7 +338,7 @@ pub(crate) fn fit_profile(pairs: &[([f64; 3], [f64; 3])]) -> Option<(Mat3, Optio}fn sensor_proxy(raw: &RawImage, k: usize, edge: usize) -> Option<Rgb32f> {- Some(match raw.develop_binned(k, 0.99).ok()? {+ Some(match raw.develop_binned(k, SENSOR_CLIP).ok()? {Some(sensor) => sensor,None if raw.cpp == 3 && raw.cfa.is_none() => fit(&raw.develop(lightcraft_raw::Method::Bilinear).ok()?, edge, edge, Filter::Box),None => return None,@@ -389,42 +464,66 @@ fn fit_matrix(pairs: &[([f64; 3], [f64; 3])]) -> Option<Mat3> {}/// Neighbourhood luminance ratio above which a pixel counts as an edge for the second attempt-/// of [`fit_pairs_with`]. Camera JPEGs are often lens-corrected (Sony "Distortion Comp.: Auto",+/// of [`search_ordered`]. Camera JPEGs are often lens-corrected (Sony "Distortion Comp.: Auto",/// compacts and kit zooms): on a 51 mm ILCE-7RM2 shot of a glass façade the JPEG is up to 15 px/// of 1440 (about one proxy pixel) off the raw, and the fit failed the gate on the mismatched/// window frames alone (held-out RMS 0.111; 0.059 for the same shot with distortion correction/// off). Away from edges: 0.049. Issue #232.const EDGE_CONTRAST: f32 = 3.0;-/// The photo's look: its own matrix (with and without a hue/saturation table), or the given-/// colour model (a camera profile's, in the sensor proxy's space), each completed with a tone-/// and chroma curve fitted to this photo. When the fit on all pixels fails the acceptance gates,-/// it is tried once more on the pixels away from edges (same gates): there colour pairs stay-/// valid when the camera JPEG's geometry differs slightly from the raw's.+/// The look on all pixels (the search of [`search_ordered`] with [`TONE_FITS`]), as before+/// clipped pixels were left out.+#[cfg(test)]fn fit_pairs_with(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {- fit_pairs_ordered(sensor, reference, colour, &[ToneFit::Quantile, ToneFit::ConditionalMedian])+ fit_pairs_ordered(sensor, reference, colour, TONE_FITS)}-/// [`fit_pairs_with`] with the tone fits to try, in order: the first whose look passes the-/// acceptance gates is used. Quantile matching comes first because it cannot produce flat steps;-/// the conditional-median fit follows because on a few photos (compacts, early Micro Four Thirds)-/// its slightly lower held-out error is what clears the gates, and a photo must not lose its look-/// to the change of fit. The whole search (all pixels, then away from edges, with and without the-/// camera profile's colour) runs with one tone fit before the next is tried, so the fallback-/// accepts exactly what the previous releases accepted.+/// The look from [`search_ordered`] without its attempt.+#[cfg(test)]fn fit_pairs_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, order: &[ToneFit]) -> Option<CameraLook> {- let has_profile = colour.is_some();+ search_ordered(sensor, reference, colour, order).map(|(look, _)| look)+}++/// The tone fits the search tries, in order: the first whose look passes the acceptance gates is+/// used. Quantile matching comes first because it cannot produce flat steps; the+/// conditional-median fit follows because on a few photos (compacts, early Micro Four Thirds) its+/// slightly lower held-out error is what clears the gates, and a photo must not lose its look to+/// the change of fit.+const TONE_FITS: &[ToneFit] = &[ToneFit::Quantile, ToneFit::ConditionalMedian];++/// One attempt of the search: tone fit, the camera profile's colour or the photo's own, all pixels+/// or only those away from edges.+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+struct Attempt {+ tone: ToneFit,+ profile: bool,+ away_from_edges: bool,+}++/// The photo's look and the attempt that produced it: its own matrix (with and without a+/// hue/saturation table), or the given colour model (a camera profile's, in the sensor proxy's+/// space), each completed with a tone and chroma curve fitted to this photo. Per tone fit in+/// `order`: the camera profile's colour (when there is one), then the photo's own, as a different+/// picture style can make a camera profile fail the gates; each fitted on all pixels and, when that+/// fails the gates, once more on the pixels away from edges (same gates): there colour pairs stay+/// valid when the camera JPEG's geometry differs slightly from the raw's (issue #232). The whole+/// search runs with one tone fit before the next is tried, so the conditional-median fallback+/// accepts exactly what the previous releases accepted.+fn search_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, order: &[ToneFit]) -> Option<(CameraLook, Attempt)> {+ let profiles: &[bool] = if colour.is_some() { &[true, false] } else { &[false] };order.iter().find_map(|&tone| {- // A different picture style can make a camera profile fail the gates. Preserve the- // photo's own colour fit before resorting to the neutral fallback.- fit_candidate(sensor, reference, colour.clone(), tone).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None, tone)).flatten())+ profiles.iter().find_map(|&profile| {+ [false, true].into_iter().find_map(|away_from_edges| {+ let attempt = Attempt { tone, profile, away_from_edges };+ fit_attempt(sensor, reference, &colour, attempt).map(|look| (look, attempt))+ })+ })})}-/// One candidate colour model, fitted on all pixels and, when that fails the gates, once more on-/// the pixels away from edges (issue #232).-fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, tone: ToneFit) -> Option<CameraLook> {- fit_pairs_on(sensor, reference, colour.clone(), None, tone).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST), tone))+fn fit_attempt(sensor: &Rgb32f, reference: &Rgb32f, colour: &Option<(Mat3, Option<HsvTable>)>, attempt: Attempt) -> Option<CameraLook> {+ let colour = if attempt.profile { colour.clone() } else { None };+ fit_pairs_on(sensor, reference, colour, attempt.away_from_edges.then_some(EDGE_CONTRAST), attempt.tone)}fn fit_pairs_on(@@ -1356,6 +1455,110 @@ mod tests {(sensor, reference)}+ /// A camera rendering with constant colourfulness (no highlight bleaching) of a colourful scene+ /// under a bright neutral sky, whose sensor pixels are clipped: one channel held at the clip+ /// level turns the white sky magenta in the raw while the camera JPEG keeps it white. Returns+ /// the sensor proxy, the JPEG proxy and the clip mask.+ fn clipped_sky_scene() -> (Rgb32f, Rgb32f, Vec<bool>) {+ let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]);+ let (w, h) = (96usize, 64usize);+ let mut sensor = Rgb32f::new(w, h);+ let mut reference = sensor.clone();+ let mut clipped = vec![false; w * h];+ let mut seed = 0x2545_f491_u32;+ let mut next = move || {+ seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);+ f32::from((seed >> 16) as u16) / 65536.0+ };+ let camera = |p: [f32; 3]| {+ let y = luminance_2020(p);+ let o = 1.0 - (-2.5 * y).exp();+ p.map(|v| (v * o / y).clamp(0.0, 0.97))+ };+ // the sky: a quarter of the frame, white in the scene and in the JPEG+ let neutral = known.inverse().unwrap().apply_f32([0.32; 3]);+ for (i, (src, dst)) in sensor.data.iter_mut().zip(&mut reference.data).enumerate() {+ let scene = if i < w * h / 4 {+ clipped[i] = true;+ let sky = neutral.map(|v| v * (0.8 + 0.4 * next()));+ *dst = camera(known.apply_f32(sky));+ // the green sample clips, red and blue (after white balance) go on rising+ *src = [sky[0] * 1.35, sky[1] * 0.85, sky[2] * 1.4];+ continue;+ } else {+ // colourful foliage, leaves, bark and bright flowers from dark to light+ let ev = 0.02 + next() * 0.4;+ [ev * (0.6 + next() * 0.8), ev, ev * (0.5 + next() * 0.8)]+ };+ *src = scene;+ *dst = camera(known.apply_f32(scene));+ }+ (sensor, reference, clipped)+ }++ /// Mean colourfulness (distance from neutral of luminance-normalised RGB) of the unclipped+ /// pixels rendered with `look`, over that of the camera JPEG, per display-luminance band.+ fn colourfulness_ratio(look: &CameraLook, sensor: &Rgb32f, reference: &Rgb32f, clipped: &[bool], band: std::ops::Range<f64>) -> f64 {+ let chroma = |p: [f64; 3]| {+ let y = luma(p);+ p.iter().map(|v| (v / y - 1.0).powi(2)).sum::<f64>().sqrt()+ };+ let tone = ToneMap::camera(&look.tone, 0.0, 0.0, 0.0);+ let (mut rendered, mut camera) = (0.0, 0.0);+ for ((x, y), c) in sensor.data.iter().zip(&reference.data).zip(clipped) {+ let y = y.map(f64::from);+ if *c || !band.contains(&luma(y)) {+ continue;+ }+ rendered += chroma(displayed(look.matrix.apply(x.map(f64::from)), &tone));+ camera += chroma(y);+ }+ rendered / camera+ }++ /// Clipped sensor pixels are left out of the look's training pairs: a clipped white sky no longer+ /// dulls the colour matrix or makes the chroma curve remove colour at the sky's brightness.+ #[test]+ fn clipped_sky_does_not_dull_the_look() {+ let (sensor, reference, clipped) = clipped_sky_scene();+ let old = fit_pairs(&sensor, &reference).expect("the earlier fit accepts the look");+ let new = fit_look(&sensor, &reference, &clipped, None).expect("look without the clipped pixels");+ let (mid, bright) = (0.1..0.35, 0.35..0.75);+ let old_ratio = (+ colourfulness_ratio(&old, &sensor, &reference, &clipped, mid.clone()),+ colourfulness_ratio(&old, &sensor, &reference, &clipped, bright.clone()),+ );+ let new_ratio =+ (colourfulness_ratio(&new, &sensor, &reference, &clipped, mid), colourfulness_ratio(&new, &sensor, &reference, &clipped, bright));+ // with the clipped sky in the pairs, the bright colours lose much of their colour+ assert!(old_ratio.1 < 0.8, "earlier fit: colourfulness vs camera {old_ratio:?}, chroma {:?}", old.tone.chroma());+ // without it, the render keeps the camera's colourfulness at every brightness+ for r in [new_ratio.0, new_ratio.1] {+ assert!((0.9..1.1).contains(&r), "colourfulness vs camera {new_ratio:?}, chroma {:?}", new.tone.chroma());+ }+ assert!(new.tone.chroma().iter().all(|k| *k > 0.85), "no colour cut: {:?}", new.tone.chroma());+ }++ /// When the accepted attempt misses the gates without the clipped pixels (here: too few pixels+ /// left), the look fitted on all pixels is kept unchanged.+ #[test]+ fn clipped_fit_falls_back_to_all_pixels() {+ let (sensor, reference, _) = clipped_sky_scene();+ let all = fit_pairs(&sensor, &reference).unwrap();+ // nearly everything clipped: too few pairs remain, the earlier look is used unchanged+ let mut clipped = vec![true; sensor.data.len()];+ clipped[..100].iter_mut().for_each(|c| *c = false);+ let fallback = fit_look(&sensor, &reference, &clipped, None).expect("the fallback keeps the look");+ assert_eq!((fallback.matrix.0, fallback.tone), (all.matrix.0, all.tone));+ // nothing clipped: the same look as before, exactly+ let none = fit_look(&sensor, &reference, &vec![false; sensor.data.len()], None).unwrap();+ assert_eq!((none.matrix.0, none.tone), (all.matrix.0, all.tone));+ // without clipped pixels the refit keeps the accepted attempt (here: all pixels, own colour, quantile tone)+ let (_, attempt) = search_ordered(&sensor, &reference, None, TONE_FITS).unwrap();+ assert_eq!(attempt, Attempt { tone: ToneFit::Quantile, profile: false, away_from_edges: false });+ assert!(without_clipped(&sensor, &[true; 3]).is_none(), "a mask of another size is ignored");+ }+/// Quantile matching is tried first; the conditional-median fit is the fallback and is used/// only where the quantile look misses the acceptance gates, so no photo loses its look.#[test]diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rsindex 1b8e8b15f..ce41bfaa2 100644--- a/crates/engine/src/media.rs+++ b/crates/engine/src/media.rs@@ -62,7 +62,7 @@ impl SettingsHashes {}/// Bump when the pipeline's output changes, to invalidate cached thumbnails.-pub const RENDER_CACHE_VERSION: u64 = 20;+pub const RENDER_CACHE_VERSION: u64 = 22;/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];