LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LR-IMP-FORMATS: decode table-less Nikon 34713 NEFs stored as packed 12/14-bit data
radialmonster749636fb2ec8
diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rsindex 22d76528d..e6c438ad7 100644--- a/crates/engine/src/media.rs+++ b/crates/engine/src/media.rs@@ -62,7 +62,7 @@ impl SettingsHashes {}/// Bump when the pipeline's output changes, to invalidate cached thumbnails.-pub const RENDER_CACHE_VERSION: u64 = 64;+pub const RENDER_CACHE_VERSION: u64 = 65;/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];diff --git a/crates/raw/src/vendor/nef.rs b/crates/raw/src/vendor/nef.rsindex 5076c6df9..5fbf83414 100644--- a/crates/raw/src/vendor/nef.rs+++ b/crates/raw/src/vendor/nef.rs@@ -7,6 +7,9 @@//! rule documented there; files it can't decode are reported as [`RawError::Unsupported`] and their embedded previews still work. So are High Efficiency NEFs//! (HE / HE★, Z 8, Z 9, Z 6III, Z f): they keep compression 34713 but carry a wavelet codestream that starts with//! the JPEG XS markers SOC + CAP (ISO/IEC 21122-1, `FF10 FF50`) and have no `0x0096` table (issue #193).+//! Strips labelled 34713 without a `0x0096`+//! linearization table are packed uncompressed data in several measured layouts ([`PackedLayout`]); Nikon small raw+//! (sRAW) and the D1X's genuinely Huffman-coded strip stay refused.use super::{nefc, white_from_data};use crate::tiffraw::{Packing, check_image, read_image};@@ -342,8 +345,83 @@ fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder, mn: Option<&maread_image(bytes, &top, order, packing)}-/// Nikon Huffman-compressed strip (see [`super::nefc`]); the decode table is maker note `0x0096`. Decodes the first-/// `rows` rows of the `info.height`.+/// Packed layouts of strips that are labelled compression 34713 but have no linearization table (maker note+/// `0x0096`): they hold plain packed samples, not Huffman codes. The layouts were measured from CC0 raw.pixls.us+/// files by a clean-room black-box analysis (strip byte-count identity, same-colour smoothness and correlation with+/// the camera JPEG against at least eight alternative packings). Every layout is a single strip with+/// `stride = StripByteCounts / height`, selected only when `stride * height` equals the strip size exactly.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum PackedLayout {+ /// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned.+ /// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes+ /// padded to 16 bytes.+ Lsb { stride: usize },+ /// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The+ /// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image.+ Chunked16 { stride: usize },+}++fn packed_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option<PackedLayout> {+ if w == 0 || h == 0 || !matches!(bits, 12 | 14) {+ return None;+ }+ let strip = usize::try_from(strip).ok()?;+ if !strip.is_multiple_of(h) {+ return None;+ }+ let stride = strip / h;+ let tight = w.checked_mul(bits as usize)?.div_ceil(8);+ let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None };+ let padded = tight.checked_next_multiple_of(16)?;+ let lsb_ok = match bits {+ 12 => stride == tight || stride == padded,+ _ => stride == padded,+ };+ if lsb_ok {+ return Some(PackedLayout::Lsb { stride });+ }+ None+}++fn d100_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option<PackedLayout> {+ let strip = usize::try_from(strip).ok()?;+ if bits != 12 || h == 0 || !strip.is_multiple_of(h) {+ return None;+ }+ let stride = strip / h;+ let per_row = stride / 16 * 10;+ (stride.is_multiple_of(16) && per_row >= w && per_row < w.checked_add(10)?).then_some(PackedLayout::Chunked16 { stride })+}++/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero.+fn unpack_packed(src: &[u8], w: usize, h: usize, bits: u32, layout: PackedLayout) -> Vec<u16> {+ use rayon::prelude::*;+ let mut out = vec![0u16; w * h];+ let stride = match layout {+ PackedLayout::Lsb { stride } | PackedLayout::Chunked16 { stride } => stride,+ };+ out.par_chunks_mut(w).enumerate().for_each(|(y, row)| {+ let Some(line) = src.get(y * stride..) else { return };+ let line = &line[..stride.min(line.len())];+ match layout {+ PackedLayout::Lsb { .. } => crate::unpack::unpack_lsb(line, bits, row),+ PackedLayout::Chunked16 { .. } => {+ let mut samples = vec![0u16; line.len() / 16 * 10];+ for (i, dst) in samples.chunks_mut(10).enumerate() {+ if let Some(chunk) = line.get(i * 16..i * 16 + 15) {+ crate::unpack::unpack_msb(chunk, 12, dst);+ }+ }+ let n = row.len().min(samples.len());+ row[..n].copy_from_slice(&samples[..n]);+ }+ }+ });+ out+}++/// Nikon Huffman-compressed strip (see [`super::nefc`]); the decode table is maker note `0x0096`. Without a table the+/// strip may be packed uncompressed data (see [`PackedLayout`]).fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, rows: usize) -> Result<RawData> {let starts_with = |magic: &[u8]| {let start = info.chunks(bytes.len() as u64).first().and_then(|c| usize::try_from(c.offset).ok());@@ -352,10 +430,9 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>,if starts_with(&JPEG_XS_START) {return Err(RawError::Unsupported(HIGH_EFFICIENCY.into()));}- // (some Z bodies label uncompressed, row-padded data 34713 without a table: not handled here yet)- let table = mn- .and_then(|m| Some((m.ifd.bytes(LINEARIZATION_TABLE)?, m.order)))- .ok_or_else(|| RawError::Unsupported("Nikon compressed NEF without a linearization table (maker note 0x96)".into()))?;+ let Some(table) = mn.and_then(|m| Some((m.ifd.bytes(LINEARIZATION_TABLE)?, m.order))) else {+ return packed(bytes, info, rows);+ };if info.samples_per_pixel != 1 {return Err(RawError::Unsupported(format!("Nikon compressed NEF with {} samples per pixel", info.samples_per_pixel)));}@@ -370,6 +447,32 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>,Ok(RawData::U16(nefc::decode_rows(src, info.width as usize, info.height as usize, rows, bits, &table)?))}+/// A table-less "compressed" strip: packed samples when the strip size matches a measured layout exactly.+fn packed(bytes: &[u8], info: &ImageInfo, rows: usize) -> Result<RawData> {+ let (w, h) = (info.width as usize, info.height as usize);+ let bits = info.bits() as u32;+ let chunks = info.chunks(bytes.len() as u64);+ let layout = match chunks.as_slice() {+ [c] if info.samples_per_pixel == 1 => packed_layout(w, h, bits, c.len).or_else(|| d100_layout(w, h, bits, c.len)),+ _ => None,+ };+ let (Some(layout), [c]) = (layout, chunks.as_slice()) else {+ let small = bits == 12 && chunks.len() == 1 && w.checked_mul(h).and_then(|n| n.checked_mul(3)).is_some_and(|n| u64::try_from(n).ok() == Some(chunks[0].len));+ return Err(RawError::Unsupported(if small {+ "Nikon small raw (sRAW) not decoded yet".into()+ } else {+ "Nikon compressed NEF without a linearization table (maker note 0x96)".into()+ }));+ };+ let end = c.offset.saturating_add(c.len).min(bytes.len() as u64);+ let src = usize::try_from(c.offset).ok().zip(usize::try_from(end).ok()).and_then(|(a, b)| bytes.get(a..b));+ let src = src.ok_or_else(|| RawError::Corrupt("NEF: image data outside the file".into()))?;+ if (w as u64).saturating_mul(h as u64) > crate::MAX_SAMPLES as u64 {+ return Err(RawError::Limit("image too large"));+ }+ Ok(RawData::U16(unpack_packed(src, w, rows.min(h), bits, layout)))+}+#[cfg(test)]mod tests {use super::*;@@ -819,4 +922,112 @@ mod tests {let short = nef(34713, 14, vec![vec![0xff, 0x10]], 1, 1, 1);assert!(matches!(crate::decode(&short), Err(RawError::Unsupported(w)) if w.contains("linearization table")));}++ fn samples(n: usize, bits: u32) -> Vec<u16> {+ (0..n).map(|i| (((i * 2654435761usize) >> 7) as u32 % (1 << bits)) as u16).collect()+ }++ fn pack_lsb(row: &[u16], bits: u32) -> Vec<u8> {+ let (mut out, mut acc, mut n) = (Vec::new(), 0u64, 0u32);+ for &v in row {+ acc |= (v as u64) << n;+ n += bits;+ while n >= 8 {+ out.push(acc as u8);+ acc >>= 8;+ n -= 8;+ }+ }+ if n > 0 {+ out.push(acc as u8);+ }+ out+ }++ fn pack_msb12(row: &[u16]) -> Vec<u8> {+ let mut out = Vec::new();+ for pair in row.chunks(2) {+ let (a, b) = (pair[0], pair.get(1).copied().unwrap_or(0));+ out.extend([(a >> 4) as u8, ((a & 15) << 4) as u8 | (b >> 8) as u8, b as u8]);+ }+ out+ }++ /// Rows of `samples` packed by `row`, each padded with `pad` bytes of 0xA5 (never decoded).+ fn strip(data: &[u16], w: usize, pad: usize, row: impl Fn(&[u16]) -> Vec<u8>) -> Vec<u8> {+ data.chunks(w).flat_map(|r| row(r).into_iter().chain(std::iter::repeat_n(0xA5, pad))).collect()+ }++ fn decoded(strip: Vec<u8>, bits: u16, w: u32, h: u32) -> Result<Vec<u16>> {+ let bytes = nef(34713, bits, vec![strip], w, h, h);+ match crate::decode(&bytes)?.data {+ RawData::U16(d) => Ok(d),+ _ => Err(RawError::Unsupported("float".into())),+ }+ }++ #[test]+ fn packed_34713_layouts_round_trip() {+ let (w, h) = (40usize, 6usize);+ let d12 = samples(w * h, 12);+ let d14 = samples(w * h, 14);+ // group A: 12-bit, 3w/2 bytes per row+ assert_eq!(decoded(strip(&d12, w, 0, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12);+ // group B: 12-bit, rows padded to 16 bytes (60 -> 64)+ assert_eq!(decoded(strip(&d12, w, 4, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12);+ // group C: 14-bit, 70 bytes -> 80+ assert_eq!(decoded(strip(&d14, w, 10, |r| pack_lsb(r, 14)), 14, 40, 6).unwrap(), d14);+ // group E: 16-byte chunks, 15 data bytes (ten samples) + a zero byte; 16 chunks = 160 samples (at 40 wide the+ // 64-byte stride would be group B's)+ let (w, h) = (160usize, 3usize);+ let d12 = samples(w * h, 12);+ let e = strip(&d12, w, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect());+ assert_eq!(e.len(), 256 * h);+ assert_eq!(decoded(e, 12, 160, 3).unwrap(), d12);+ }++ #[test]+ fn packed_d100_rows_carry_extra_columns() {+ // IFD width 154, rows hold 160 samples (16 chunks); the last 6 are dropped+ let (w, wide, h) = (154usize, 160usize, 3usize);+ let wide_data = samples(wide * h, 12);+ let e = strip(&wide_data, wide, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect());+ let got = decoded(e, 12, w as u32, h as u32).unwrap();+ let want: Vec<u16> = wide_data.chunks(wide).flat_map(|r| r[..w].iter().copied()).collect();+ assert_eq!(got, want);+ }++ #[test]+ fn packed_34713_wrong_sizes_stay_refused() {+ let d12 = samples(40 * 6, 12);+ let good = strip(&d12, 40, 0, |r| pack_lsb(r, 12));+ // one byte more or fewer, a stride that is neither tight nor 16-aligned, a 13-bit file+ for len in [good.len() - 1, good.len() + 1, good.len() + 6] {+ let mut s = good.clone();+ s.resize(len, 0);+ assert!(matches!(decoded(s, 12, 40, 6), Err(RawError::Unsupported(_))), "len {len}");+ }+ assert!(matches!(decoded(good.clone(), 13, 40, 6), Err(RawError::Unsupported(_))));+ // 14-bit rows must be padded to 16 bytes+ let d14 = samples(40 * 6, 14);+ assert!(matches!(decoded(strip(&d14, 40, 0, |r| pack_lsb(r, 14)), 14, 40, 6), Err(RawError::Unsupported(_))));+ // 3 bytes per IFD-width pixel is a Nikon small raw: still refused, with its own reason+ let Err(RawError::Unsupported(why)) = decoded(vec![0; 3 * 40 * 6], 12, 40, 6) else { panic!("small raw decoded") };+ assert!(why.contains("sRAW"), "{why}");+ }++ #[test]+ fn packed_34713_short_input_does_not_panic() {+ let (w, h) = (40usize, 6usize);+ for layout in [PackedLayout::Lsb { stride: 64 }, PackedLayout::Lsb { stride: 80 }, PackedLayout::Chunked16 { stride: 64 }] {+ for len in [0, 1, 15, 16, 17, 63, 64, 65, 200] {+ let bits = if matches!(layout, PackedLayout::Lsb { stride: 80 }) { 14 } else { 12 };+ assert_eq!(unpack_packed(&vec![0xFF; len], w, h, bits, layout).len(), w * h);+ }+ }+ // a strip that claims more bytes than the file has+ let mut bytes = nef(34713, 12, vec![vec![0x55; 60 * 6]], 40, 6, 6);+ bytes.truncate(bytes.len() - 100);+ let _ = crate::decode(&bytes);+ }}