LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
M11.3: Pentax PEF decoding (uncompressed and Huffman)
Brandon Thomas3f86346c5dcf
diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rsindex f4f5e4120..57b6df6f5 100644--- a/apps/lightcraft/src/main.rs+++ b/apps/lightcraft/src/main.rs@@ -71,7 +71,10 @@ fn services() -> Services {rfd::FileDialog::new().add_filter("Photos",- &["jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "nef", "nrw", "arw", "raf", "rw2", "psd", "jxl", "gif", "bmp"],+ &[+ "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "nef", "nrw", "arw", "raf", "rw2", "pef", "psd", "jxl", "gif",+ "bmp",+ ],).pick_files().unwrap_or_default()@@ -99,7 +102,24 @@ fn collect(path: &std::path::Path, out: &mut Vec<String>) {} else if path.extension().is_some_and(|e| {matches!(e.to_string_lossy().to_lowercase().as_str(),- "jpg" | "jpeg" | "png" | "tif" | "tiff" | "webp" | "dng" | "cr2" | "nef" | "nrw" | "arw" | "raf" | "rw2" | "psd" | "jxl" | "gif" | "bmp"+ "jpg"+ | "jpeg"+ | "png"+ | "tif"+ | "tiff"+ | "webp"+ | "dng"+ | "cr2"+ | "nef"+ | "nrw"+ | "arw"+ | "raf"+ | "rw2"+ | "pef"+ | "psd"+ | "jxl"+ | "gif"+ | "bmp")}) {out.push(path.to_string_lossy().to_string());diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rsindex 62efc05ac..fc20fe134 100644--- a/crates/raw/src/lib.rs+++ b/crates/raw/src/lib.rs@@ -11,7 +11,8 @@//!//! Formats: DNG (uncompressed, lossless JPEG, Deflate incl. floating point, tiled/stripped, CFA and LinearRaw),//! Canon CR2, Nikon NEF (uncompressed), Sony ARW (uncompressed, lossless), Fujifilm RAF (uncompressed Bayer and-//! X-Trans), Panasonic RW2 (packed 12/14-bit). See the crate README for sources and+//! X-Trans), Panasonic RW2 (packed 12/14-bit), Pentax PEF+//! (uncompressed, Huffman). See the crate README for sources and//! gaps. The decoders never panic on malformed input.#![forbid(unsafe_code)]@@ -81,7 +82,10 @@ pub enum RawFormat {impl RawFormat {/// Whether [`decode`] supports this container (possibly not every compression inside it).pub fn is_supported(self) -> bool {- matches!(self, RawFormat::Dng | RawFormat::Cr2 | RawFormat::Nef | RawFormat::Nrw | RawFormat::Arw | RawFormat::Raf | RawFormat::Rw2)+ matches!(+ self,+ RawFormat::Dng | RawFormat::Cr2 | RawFormat::Nef | RawFormat::Nrw | RawFormat::Arw | RawFormat::Raf | RawFormat::Rw2 | RawFormat::Pef+ )}}@@ -144,6 +148,7 @@ pub fn decode(bytes: &[u8]) -> Result<RawImage> {RawFormat::Arw => vendor::arw::decode(bytes),RawFormat::Raf => vendor::raf::decode(bytes),RawFormat::Rw2 => vendor::rw2::decode(bytes),+ RawFormat::Pef => vendor::pef::decode(bytes),other => Err(RawError::Unsupported(format!("{other:?} files are not decoded yet"))),}}diff --git a/crates/raw/src/vendor/mod.rs b/crates/raw/src/vendor/mod.rsindex 9417a74fc..642741201 100644--- a/crates/raw/src/vendor/mod.rs+++ b/crates/raw/src/vendor/mod.rs@@ -3,6 +3,7 @@pub mod arw;pub mod cr2;pub mod nef;+pub mod pef;pub mod raf;pub mod rw2;diff --git a/crates/raw/src/vendor/pef.rs b/crates/raw/src/vendor/pef.rsnew file mode 100644index 000000000..57034d772--- /dev/null+++ b/crates/raw/src/vendor/pef.rs@@ -0,0 +1,336 @@+//! Pentax PEF — uncompressed and Pentax Huffman-compressed (compression 65535).+//!+//! Sources: TIFF 6.0 (container), ITU-T T.81 (JPEG) Annex H / F.1.2.1 for the lossless "difference category ++//! additional bits" coding, the ExifTool Pentax tag-name documentation (maker-note `0x0200` BlackPoint, `0x0201`+//! WhitePoint i.e. WB levels, `0x0220` HuffmanTable) and our own black-box analysis of CC0 samples from+//! raw.pixls.us (K10D, K-5 II s, K-3):+//!+//! - The Huffman table is stored in the file (maker note `0x0220`, in the maker note's byte order): a `u16` `d`,+//! 12 further bytes, then `n = d + 12` `u16` codes left-aligned in 12 bits, then `n` code lengths (bytes).+//! Symbol `i` is the difference category `i` (T.81 Table H.2: the number of additional bits).+//! - The bit stream (MSB-first, no byte stuffing, continuous across rows) codes one difference per pixel; the+//! additional bits follow T.81 F.1.2.1 (a leading 0 means a negative value).+//! - Prediction: each pixel is predicted from the previous same-colour pixel in its row (two to the left); the+//! first two pixels of a row are predicted from the first two pixels of the previous row of the same parity+//! (starting at 0). Found by trying candidate predictors and checking the decoded rows are continuous.+//! - Maker note `0x0038`/`0x0039`: the image area's left/top and width/height (verified against the data).++use super::{black_from_columns, white_from_data};++use crate::tiffraw::{Packing, read_image};+use crate::{BlackLevel, Cfa, ColorData, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};+use lightcraft_geom::Orientation;+use lightcraft_tiff::image::chunk_bytes;+use lightcraft_tiff::{ByteOrder, Tiff, makernote, tags as t};++const CROP_ORIGIN: u16 = 0x0038;+const CROP_SIZE: u16 = 0x0039;+const BLACK_POINT: u16 = 0x0200;+const WB_LEVELS: u16 = 0x0201;+const HUFFMAN: u16 = 0x0220;++/// A canonical-by-table Huffman decoder with a 12-bit lookup.+pub(crate) struct Huffman {+ /// Indexed by the next 12 bits: (symbol, length); length 0 = invalid.+ lut: Vec<(u8, u8)>,+}++impl Huffman {+ /// From `(left-aligned 12-bit code, length)` per symbol.+ pub fn new(codes: &[(u16, u8)]) -> Result<Huffman> {+ let mut lut = vec![(0u8, 0u8); 4096];+ for (sym, &(code, len)) in codes.iter().enumerate() {+ if len == 0 {+ continue;+ }+ if len > 12 || sym > 16 {+ return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}")));+ }+ let first = (code >> (12 - len) << (12 - len)) as usize;+ for e in &mut lut[first..first + (1 << (12 - len))] {+ if e.1 == 0 {+ *e = (sym as u8, len);+ }+ }+ }+ Ok(Huffman { lut })+ }+}++/// MSB-first bit reader that yields zeros past the end.+pub(crate) struct Bits<'a> {+ src: &'a [u8],+ pos: usize,+ acc: u64,+ n: u32,+}++impl<'a> Bits<'a> {+ pub fn new(src: &'a [u8]) -> Self {+ Bits { src, pos: 0, acc: 0, n: 0 }+ }+ #[inline]+ fn fill(&mut self) {+ while self.n <= 56 {+ let b = self.src.get(self.pos).copied().unwrap_or(0);+ self.pos += 1;+ self.acc |= (b as u64) << (56 - self.n);+ self.n += 8;+ }+ }+ #[inline]+ pub fn peek(&mut self, k: u32) -> u32 {+ if self.n < k {+ self.fill();+ }+ (self.acc >> (64 - k)) as u32+ }+ #[inline]+ pub fn skip(&mut self, k: u32) {+ self.acc <<= k;+ self.n -= k;+ }+ #[inline]+ pub fn get(&mut self, k: u32) -> u32 {+ if k == 0 {+ return 0;+ }+ let v = self.peek(k);+ self.skip(k);+ v+ }+ pub fn overrun(&self) -> bool {+ self.pos > self.src.len() + 8+ }+}++/// Decode one difference: Huffman category then T.81 additional bits.+#[inline]+pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option<i32> {+ let (sym, len) = h.lut[bits.peek(12) as usize];+ if len == 0 {+ return None;+ }+ bits.skip(len as u32);+ let s = sym as u32;+ if s == 0 {+ return Some(0);+ }+ if s > 16 {+ return None;+ }+ let v = bits.get(s) as i32;+ Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v })+}++/// Parse the maker-note Huffman table.+pub(crate) fn table(b: &[u8], order: ByteOrder) -> Result<Huffman> {+ let u16_at = |i: usize| b.get(i..i + 2).map(|s| order.u16([s[0], s[1]]));+ let d = u16_at(0).ok_or_else(|| RawError::Corrupt("PEF Huffman table too short".into()))? as usize;+ let n = d + 12;+ if n > 17 || b.len() < 14 + 3 * n {+ return Err(RawError::Corrupt(format!("PEF Huffman table: {n} symbols in {} bytes", b.len())));+ }+ let codes: Vec<(u16, u8)> = (0..n).map(|i| (u16_at(14 + 2 * i).unwrap_or(0) & 0x0fff, b[14 + 2 * n + i])).collect();+ Huffman::new(&codes)+}++/// Decode a `w × h` Huffman-coded image.+pub(crate) fn decode_huffman(src: &[u8], h: &Huffman, w: usize, hgt: usize, bits_per: u32) -> Result<Vec<u16>> {+ let mut out = vec![0u16; w * hgt];+ let mut bits = Bits::new(src);+ let mut vpred = [[0i32; 2]; 2];+ let max = (1i32 << bits_per.min(16)) - 1;+ for y in 0..hgt {+ let mut hpred = [0i32; 2];+ let row = &mut out[y * w..(y + 1) * w];+ for (x, o) in row.iter_mut().enumerate() {+ let d = diff(&mut bits, h).ok_or_else(|| RawError::Corrupt(format!("PEF: invalid Huffman code at row {y}")))?;+ let p = if x < 2 {+ vpred[y & 1][x] += d;+ hpred[x] = vpred[y & 1][x];+ hpred[x]+ } else {+ hpred[x & 1] += d;+ hpred[x & 1]+ };+ *o = p.clamp(0, max) as u16;+ }+ if bits.overrun() {+ return Err(RawError::Corrupt(format!("PEF: data ends at row {y} of {hgt}")));+ }+ }+ Ok(out)+}++/// The image area without all-dark leading/trailing columns (masked or empty), kept at even offsets.+fn dark_trimmed(d: &[u16], w: usize, h: usize) -> Rect {+ if w < 256 || h == 0 {+ return Rect::new(0, 0, w, h);+ }+ let step = (h / 256).max(1);+ let col_mean = |x: usize| (0..h).step_by(step).map(|y| d[y * w + x] as f64).sum::<f64>() / h.div_ceil(step) as f64;+ let interior = (w / 4..w * 3 / 4).step_by(w / 64).map(col_mean).sum::<f64>() / 32.0;+ let dark = interior * 0.05;+ let mut left = 0;+ while left < 64 && col_mean(left) < dark {+ left += 1;+ }+ let mut right = w;+ while right > w - 64 && col_mean(right - 1) < dark {+ right -= 1;+ }+ let left = left.next_multiple_of(2);+ Rect::new(left, 0, (right - left) & !1, h)+}++pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {+ let tiff = Tiff::parse(bytes)?;+ let ifd0 = &tiff.ifds[0];+ let info = ifd0.image()?;+ let (w, hgt) = (info.width as usize, info.height as usize);+ let bits = info.bits() as u32;+ if w == 0 || hgt == 0 || w.saturating_mul(hgt) > crate::MAX_SAMPLES {+ return Err(RawError::Limit("image too large"));+ }+ let make = ifd0.string(t::MAKE).unwrap_or_default();+ let mn =+ tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make));+ let data = match info.compression {+ 65535 => {+ let mn = mn.as_ref().ok_or_else(|| RawError::Corrupt("compressed PEF without maker note".into()))?;+ let tb = mn.ifd.bytes(HUFFMAN).ok_or_else(|| RawError::Unsupported("compressed PEF without a Huffman table".into()))?;+ let huff = table(tb, mn.order)?;+ let chunks = info.chunks(bytes.len() as u64);+ let first = chunks.first().ok_or_else(|| RawError::Corrupt("PEF without image data".into()))?;+ let mut c = *first;+ c.len = chunks.iter().map(|c| c.len).sum::<u64>().max(first.len);+ let src = chunk_bytes(bytes, &c)+ .or_else(|| bytes.get(first.offset as usize..))+ .ok_or_else(|| RawError::Corrupt("PEF data outside file".into()))?;+ RawData::U16(decode_huffman(src, &huff, w, hgt, bits)?)+ }+ 1 => {+ let strip: u64 = info.chunks(bytes.len() as u64).iter().map(|c| c.len).sum();+ let packing = if strip >= (w * hgt * 2) as u64 { Packing::Word16 } else { Packing::Msb };+ read_image(bytes, &info, tiff.order, packing)?+ }+ c => return Err(RawError::Unsupported(format!("PEF compression {c}"))),+ };+ let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float PEF".into())) };++ let pair = |tag: u16| mn.as_ref().and_then(|m| m.ifd.u64s(tag)).filter(|v| v.len() == 2).map(|v| (v[0] as usize, v[1] as usize));+ let active = match (pair(CROP_ORIGIN), pair(CROP_SIZE)) {+ (Some((x, y)), Some((cw, ch))) if cw > 0 && ch > 0 && x + cw <= w && y + ch <= hgt => Rect::new(x, y, cw, ch),+ _ => dark_trimmed(samples, w, hgt),+ };+ let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) {+ (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },+ _ => Cfa::bayer("BGGR").expect("static"),+ };+ let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_POINT)).as_deref() {+ Some(v @ [_, _, _, _]) => {+ // RGGB-ordered levels → per CFA position at the active area origin+ let a = cfa.shifted(active.x, active.y);+ let values = a.pattern.iter().map(|&c| [v[0], (v[1] + v[2]) / 2.0, v[3]][c as usize] as f32).collect();+ BlackLevel { repeat_rows: 2, repeat_cols: 2, values, ..Default::default() }+ }+ _ if active.x >= 4 => black_from_columns(samples, w, 0..active.x.saturating_sub(2), 0..hgt, active),+ _ => BlackLevel::uniform(0.0),+ };+ let wb = mn.as_ref().and_then(|m| m.ifd.f64s(WB_LEVELS)).filter(|v| v.len() == 4 && v.iter().all(|x| *x > 0.0)).map(|v| {+ let g = (v[1] + v[2]) / 2.0;+ [(v[0] / g) as f32, 1.0, (v[3] / g) as f32]+ });+ let white = white_from_data(samples, bits);+ let mut metadata = lightcraft_meta::from_tiff(&tiff);+ metadata.width = Some(active.width as u32);+ metadata.height = Some(active.height as u32);+ let img = RawImage {+ format: RawFormat::Pef,+ width: w,+ height: hgt,+ cpp: 1,+ data,+ cfa: Some(cfa),+ bits,+ black,+ white: vec![white],+ active_area: active,+ crop: Rect::new(0, 0, active.width, active.height),+ orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),+ color: ColorData::default(),+ wb_multipliers: wb,+ linearized: false,+ opcodes: OpcodeLists::default(),+ metadata,+ };+ img.validate()?;+ Ok(img)+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Our own prefix code for categories 0..=14: lengths 2,2,3,3,4,5,..., codes assigned canonically.+ fn test_table() -> (Vec<u8>, Vec<(u16, u8)>) {+ let lens: [u8; 15] = [3, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12];+ let mut order: Vec<usize> = (0..15).collect();+ order.sort_by_key(|&i| (lens[i], i));+ let mut codes = vec![(0u16, 0u8); 15];+ let (mut code, mut prev) = (0u32, lens[order[0]]);+ for (k, &i) in order.iter().enumerate() {+ if k > 0 {+ code = (code + 1) << (lens[i] - prev);+ }+ prev = lens[i];+ codes[i] = ((code << (12 - lens[i])) as u16, lens[i]);+ }+ let mut b = vec![0u8, 3];+ b.extend_from_slice(&[0; 12]);+ for c in &codes {+ b.extend_from_slice(&c.0.to_be_bytes());+ }+ b.extend(codes.iter().map(|c| c.1));+ (b, codes)+ }++ fn encode(px: &[u16], w: usize, codes: &[(u16, u8)]) -> Vec<u8> {+ let mut bits: Vec<bool> = Vec::new();+ let put = |v: u32, n: u8, bits: &mut Vec<bool>| (0..n).rev().for_each(|i| bits.push(v >> i & 1 == 1));+ let mut vpred = [[0i32; 2]; 2];+ for (y, row) in px.chunks(w).enumerate() {+ let mut hpred = [0i32; 2];+ for (x, &v) in row.iter().enumerate() {+ let pred = if x < 2 { vpred[y & 1][x] } else { hpred[x & 1] };+ let d = v as i32 - pred;+ if x < 2 {+ vpred[y & 1][x] = v as i32;+ }+ hpred[x & 1] = v as i32;+ let s = if d == 0 { 0 } else { 32 - d.unsigned_abs().leading_zeros() } as u8;+ let (c, l) = codes[s as usize];+ put((c >> (12 - l)) as u32, l, &mut bits);+ if s > 0 {+ let extra = if d < 0 { (d - 1) as u32 & ((1 << s) - 1) } else { d as u32 };+ put(extra, s, &mut bits);+ }+ }+ }+ bits.chunks(8).map(|c| c.iter().enumerate().fold(0u8, |a, (i, &b)| a | (b as u8) << (7 - i))).collect()+ }++ #[test]+ fn huffman_round_trip() {+ let (tb, codes) = test_table();+ let h = table(&tb, ByteOrder::Big).unwrap();+ let (w, hgt) = (37usize, 9usize);+ let px: Vec<u16> = (0..w * hgt).map(|i| ((i * 7919) % 16384) as u16).collect();+ let src = encode(&px, w, &codes);+ assert_eq!(decode_huffman(&src, &h, w, hgt, 14).unwrap(), px);+ // truncated data is reported, not a panic+ assert!(decode_huffman(&src[..src.len() / 3], &h, w, hgt, 14).is_err());+ assert!(table(&tb[..20], ByteOrder::Big).is_err());+ }+}