Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

M3.7: Nikon NEF uncompressed decoding

Brandon Thomas5560415d567e
diff --git a/crates/raw/examples/colstats.rs b/crates/raw/examples/colstats.rs
new file mode 100644
index 000000000..d536274cd
--- /dev/null
+++ b/crates/raw/examples/colstats.rs
@@ -0,0 +1,17 @@
+//! Debug helper: per-column / per-row statistics of the decoded raw (outliers above the bit range).
+fn main() {
+ let a: Vec<String> = std::env::args().skip(1).collect();
+ let r = lightcraft_raw::decode(&std::fs::read(&a[0]).unwrap()).unwrap();
+ let lightcraft_raw::RawData::U16(d) = &r.data else { return };
+ let lim = (1u32 << r.bits) as u16;
+ let (w, h) = (r.width, r.height);
+ let bad_cols: Vec<(usize, usize)> = (0..w).map(|x| (x, (0..h).filter(|&y| d[y * w + x] >= lim).count())).filter(|c| c.1 > 0).collect();
+ let bad_rows: Vec<(usize, usize)> = (0..h).map(|y| (y, (0..w).filter(|&x| d[y * w + x] >= lim).count())).filter(|c| c.1 > 0).collect();
+ println!("cols with >= {lim}: {} {:?}", bad_cols.len(), &bad_cols[..bad_cols.len().min(20)]);
+ println!("rows with >= {lim}: {} {:?}", bad_rows.len(), &bad_rows[..bad_rows.len().min(10)]);
+ let mx = d.iter().filter(|&&v| v < lim).max();
+ println!(
+ "max in range: {mx:?}; right-edge col means: {:?}",
+ (w - 8..w).map(|x| (0..h).map(|y| d[y * w + x] as u64).sum::<u64>() / h as u64).collect::<Vec<_>>()
+ );
+}
diff --git a/crates/raw/src/vendor/mod.rs b/crates/raw/src/vendor/mod.rs
index f15e64159..cce984549 100644
--- a/crates/raw/src/vendor/mod.rs
+++ b/crates/raw/src/vendor/mod.rs
@@ -30,13 +30,17 @@ pub(crate) fn black_from_columns(data: &[u16], width: usize, cols: Range<usize>,
/// otherwise the full `bits` range.
pub(crate) fn white_from_data(data: &[u16], bits: u32) -> f32 {
let full = ((1u32 << bits.clamp(1, 16)) - 1) as f32;
- let Some(&mx) = data.iter().max() else { return full };
+ // samples above the nominal range are padding / invalid (e.g. 0xFFFF fill), not saturation
+ let Some(&mx) = data.iter().filter(|&&v| v as f32 <= full).max() else { return full };
let band = (mx / 256).max(1);
let near = mx.saturating_sub(band);
let below = near.saturating_sub(band);
let step = (data.len() / 2_000_000).max(1);
let (mut top, mut under) = (0usize, 0usize);
for &v in data.iter().step_by(step) {
+ if v as f32 > full {
+ continue;
+ }
if v >= near {
top += 1;
} else if v >= below {
@@ -48,7 +52,7 @@ pub(crate) fn white_from_data(data: &[u16], bits: u32) -> f32 {
// plateau: use its lower edge so everything at saturation maps to ≥ 1.0
near as f32
} else {
- full.max(mx as f32)
+ full
}
}
diff --git a/crates/raw/src/vendor/nef.rs b/crates/raw/src/vendor/nef.rs
index b82f3453a..3f2055d49 100644
--- a/crates/raw/src/vendor/nef.rs
+++ b/crates/raw/src/vendor/nef.rs
@@ -1,5 +1,164 @@
-use crate::{RawError, RawImage, Result};
+//! Nikon NEF / NRW — uncompressed variants only.
+//!
+//! Sources: TIFF 6.0 (the raw image is a standard CFA SubIFD), Laurent Clévy's NEF structure notes (prose: IFD
+//! layout, SubIFDs, maker note header) and the ExifTool Nikon tag-name documentation (`0x000c` WB_RBLevels,
+//! `0x003d` BlackLevel). Nikon's Huffman-compressed variants (compression 34713: lossless, lossy type 1/2) are
+//! **not** implemented: the only descriptions we found are derived from GPL code, which our clean-room rules
+//! forbid. They are reported as [`RawError::Unsupported`]; their embedded previews still work.
-pub(crate) fn decode(_bytes: &[u8]) -> Result<RawImage> {
- Err(RawError::Unsupported("nef: not yet".into()))
+use super::white_from_data;
+use crate::tiffraw::{Packing, read_image};
+use crate::{BlackLevel, Cfa, ColorData, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
+use lightcraft_geom::Orientation;
+use lightcraft_tiff::tags::{self as t, photometric};
+use lightcraft_tiff::{Ifd, Tiff, makernote};
+
+const WB_RB_LEVELS: u16 = 0x000c;
+const BLACK_LEVEL: u16 = 0x003d;
+
+fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
+ tiff.all_ifds()
+ .into_iter()
+ .filter(|i| i.u16(t::PHOTOMETRIC) == Some(photometric::CFA))
+ .max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0) * i.u64(t::IMAGE_LENGTH).unwrap_or(0))
+}
+
+/// Width without the optically masked columns some bodies append on the right: trailing columns (at most 64)
+/// whose mean is below 1% of the white level while the image interior is brighter. Kept even for CFA phase.
+pub(crate) fn trailing_masked_columns(d: &[u16], w: usize, h: usize, white: f32) -> usize {
+ if w < 128 || h == 0 {
+ return w;
+ }
+ let step = (h / 256).max(1);
+ let col_mean = |x: usize| (0..h).step_by(step).map(|y| d[y * w + x] as f64).sum::<f64>() / h.div_ceil(step) as f64;
+ let interior = (w / 4..w * 3 / 4).step_by(w / 64).map(col_mean).sum::<f64>() / 32.0;
+ let dark = (white as f64 * 0.01).min(interior * 0.1);
+ let mut aw = w;
+ while aw > w - 64 && col_mean(aw - 1) < dark {
+ aw -= 1;
+ }
+ aw & !1
+}
+
+pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
+ let tiff = Tiff::parse(bytes)?;
+ let ifd0 = &tiff.ifds[0];
+ let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Unsupported("NEF without a CFA image IFD".into()))?;
+ let info = raw.image()?;
+ if info.compression == t::compression::NIKON {
+ return Err(RawError::Unsupported("Nikon Huffman-compressed NEF (no clean-room description available)".into()));
+ }
+ let (w, h) = (info.width as usize, info.height as usize);
+ let bits = info.bits() as u32;
+ let row_samples = w * info.samples_per_pixel as usize;
+ let chunks = info.chunks(bytes.len() as u64);
+ let rows_in_first = chunks.first().map(|c| c.height as usize).unwrap_or(h).max(1);
+ let bytes_per_row = chunks.first().map(|c| c.len as usize / rows_in_first).unwrap_or(0);
+ let packing = if bytes_per_row >= row_samples * 2 {
+ Packing::Word16
+ } else if bits == 12 && bytes_per_row * 8 >= row_samples * 12 && bytes_per_row * 8 < row_samples * 13 {
+ Packing::Msb
+ } else if info.compression == 1 && bits != 8 && bits != 16 {
+ return Err(RawError::Unsupported(format!("NEF uncompressed packing ({bytes_per_row} bytes per {row_samples}-sample row)")));
+ } else {
+ Packing::Msb
+ };
+ let data = read_image(bytes, &info, tiff.order, packing)?;
+ let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float NEF".into())) };
+
+ let make = ifd0.string(t::MAKE).unwrap_or_default();
+ let mn =
+ tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make));
+ let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
+ Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => {
+ BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![*a as f32, *b as f32, *c as f32, *d as f32], ..Default::default() }
+ }
+ _ => BlackLevel::uniform(0.0),
+ };
+ let wb = mn
+ .as_ref()
+ .and_then(|m| m.ifd.f64s(WB_RB_LEVELS))
+ .filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0)
+ .map(|v| [v[0] as f32, 1.0, v[1] as f32]);
+ let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
+ (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },
+ _ => Cfa::bayer("RGGB").expect("static"),
+ };
+ let white = white_from_data(samples, bits);
+ let active_w = trailing_masked_columns(samples, w, h, white);
+ let mut metadata = lightcraft_meta::from_tiff(&tiff);
+ metadata.width = Some(active_w as u32);
+ metadata.height = Some(h as u32);
+ let img = RawImage {
+ format: RawFormat::Nef,
+ width: w,
+ height: h,
+ cpp: 1,
+ data,
+ cfa: Some(cfa),
+ bits,
+ black,
+ white: vec![white],
+ active_area: Rect::new(0, 0, active_w, h),
+ crop: Rect::new(0, 0, active_w, h),
+ orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
+ color: ColorData::default(),
+ wb_multipliers: wb,
+ linearized: false,
+ opcodes: OpcodeLists::default(),
+ metadata,
+ };
+ img.validate()?;
+ Ok(img)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value};
+
+ fn nef(compression: u16, bits: u16, strips: Vec<Vec<u8>>, w: u32, h: u32, rps: u32) -> Vec<u8> {
+ let mut raw = IfdBuilder::new();
+ raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
+ raw.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
+ raw.set(t::IMAGE_LENGTH, Value::Long(vec![h]));
+ raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits]));
+ raw.set(t::COMPRESSION, Value::Short(vec![compression]));
+ raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]));
+ raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
+ raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![2, 1, 1, 0]));
+ raw.set_image(ImageData::Strips { rows_per_strip: rps, strips });
+ let mut ifd0 = IfdBuilder::new();
+ ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into()));
+ ifd0.set(t::MODEL, Value::Ascii("NIKON TEST".into()));
+ ifd0.add_sub_ifd(raw);
+ TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap()
+ }
+
+ #[test]
+ fn uncompressed_word16_and_packed12() {
+ let (w, h) = (16usize, 6usize);
+ let px: Vec<u16> = (0..w * h).map(|i| (i * 131 % 4096) as u16).collect();
+ let words: Vec<u8> = px.iter().flat_map(|v| v.to_be_bytes()).collect();
+ let bytes = nef(1, 12, words.chunks(w * 2 * 3).map(|c| c.to_vec()).collect(), w as u32, h as u32, 3);
+ assert_eq!(crate::probe(&bytes), Some(RawFormat::Nef));
+ let r = crate::decode(&bytes).unwrap();
+ assert_eq!(r.data, RawData::U16(px.clone()));
+ assert_eq!(r.cfa.as_ref().unwrap().name(), "BGGR");
+ // 12-bit MSB packed
+ let mut packed = Vec::new();
+ for pair in px.chunks(2) {
+ let (a, b) = (pair[0] as u32, pair[1] as u32);
+ let v = (a << 12) | b;
+ packed.extend_from_slice(&[(v >> 16) as u8, (v >> 8) as u8, v as u8]);
+ }
+ let bytes = nef(1, 12, vec![packed], w as u32, h as u32, h as u32);
+ assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px));
+ }
+
+ #[test]
+ fn compressed_is_unsupported() {
+ let bytes = nef(34713, 14, vec![vec![0; 64]], 8, 8, 8);
+ assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))));
+ }
}