Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

LRC-LIB-CATALOG-IMPORT: describe raws from their headers without reading pixel data

ingolfteetzbbd973f32106
diff --git a/crates/meta/src/cr3.rs b/crates/meta/src/cr3.rs
index 5fa6437bc..55b3ae6a6 100644
--- a/crates/meta/src/cr3.rs
+++ b/crates/meta/src/cr3.rs
@@ -226,7 +226,8 @@ impl Cr3<'_> {
}
const CANON_UUID: [u8; 16] = [0x85, 0xc0, 0xb6, 0x87, 0x82, 0x0f, 0x11, 0xe0, 0x81, 0x11, 0xf4, 0xce, 0x46, 0x2b, 0x6a, 0x48];
-const XMP_UUID: [u8; 16] = [0xbe, 0x7a, 0xcf, 0xcb, 0x97, 0xa9, 0x42, 0xe8, 0x9c, 0x71, 0x99, 0x94, 0x91, 0xe3, 0xaf, 0xac];
+/// The `uuid` of the top-level box holding the XMP packet.
+pub const XMP_UUID: [u8; 16] = [0xbe, 0x7a, 0xcf, 0xcb, 0x97, 0xa9, 0x42, 0xe8, 0x9c, 0x71, 0x99, 0x94, 0x91, 0xe3, 0xaf, 0xac];
const MAX_DEPTH: usize = 12;
const MAX_BOXES: usize = 4096;
/// A `VisualSampleEntry` (78 bytes after the box header) plus 4 bytes Canon adds before the child boxes.
@@ -237,7 +238,8 @@ pub fn is_cr3(bytes: &[u8]) -> bool {
bytes.get(4..12) == Some(b"ftypcrx ".as_slice())
}
-/// Parse a CR3 file's boxes; `None` when it is not a CR3 file.
+/// Parse a CR3 file's boxes; `None` when it is not a CR3 file. Of the top-level boxes only `moov`
+/// and the XMP `uuid` are read (`lightcraft_raw::sparse` relies on it to skip the others).
pub fn parse_cr3(bytes: &[u8]) -> Option<Cr3<'_>> {
if !is_cr3(bytes) {
return None;
diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs
index f0331d610..2c2114110 100644
--- a/crates/raw/src/lib.rs
+++ b/crates/raw/src/lib.rs
@@ -40,6 +40,7 @@ mod preview;
pub mod profile;
mod saturation;
pub mod semantic;
+pub mod sparse;
pub mod spectral;
mod tiffraw;
mod unpack;
diff --git a/crates/raw/src/sparse.rs b/crates/raw/src/sparse.rs
new file mode 100644
index 000000000..14fa3c1f8
--- /dev/null
+++ b/crates/raw/src/sparse.rs
@@ -0,0 +1,553 @@
+//! Describing a raw file without reading its pixel data.
+//!
+//! [`probe_info`](crate::probe_info) and `lightcraft_meta::extract` read only a raw file's
+//! headers, but those are a few hundred kilobytes of a file of tens of megabytes: an import from a
+//! slow drive spends nearly all its time reading samples nothing looks at. [`header_plan`] says
+//! which byte ranges of a file those readers need. It is asked about a buffer of the file's full
+//! length in which only the ranges in `loaded` hold the file's bytes (the rest is zeros); the caller
+//! loads the ranges it names and asks again. Once it says [`HeaderPlan::Ready`], probing the buffer
+//! gives what probing the whole file gives.
+//!
+//! Only bulk sample data is left out: the strips and tiles of each large TIFF image (keeping the
+//! start of the first one, where the coding headers a header probe validates sit), a CR3's image
+//! tracks (keeping the raw track's coding headers and quantization maps, and the timed-metadata
+//! track where Canon puts the as-shot white balance), a RAF's CFA strip (keeping its compression
+//! header) and a Panasonic RW2's raw data. Formats whose header probe reads samples (NEF: masked
+//! columns; ORF: bit depth and CFA phase; PEF: dark borders) and anything else get
+//! [`HeaderPlan::Whole`], as does a structure that is incomplete, inconsistent or keeps metadata
+//! inside its sample data: the plan never guesses.
+
+use std::collections::HashSet;
+
+use lightcraft_meta::cr3::{Cr3TrackKind, XMP_UUID, is_cr3, parse_cr3};
+use lightcraft_tiff::{ByteOrder, FieldType, Tiff, tags as t};
+
+use crate::RawFormat;
+
+/// Bytes kept at the start of a skipped block: the coding headers a header probe validates there
+/// (lossless-JPEG frame headers, a RAF's compression header and stripe table) fit with room to spare.
+const HEAD: u64 = 64 * 1024;
+/// An image whose chunks hold fewer bytes than this is read (thumbnails, small previews).
+const BULK: u64 = 256 * 1024;
+/// The same bounds as `lightcraft_tiff`'s reader.
+const MAX_IFDS: usize = 1024;
+const MAX_ENTRIES: u64 = 8192;
+const MAX_BOXES: usize = 4096;
+/// Child IFDs listed by one `SubIFDs` entry.
+const MAX_CHILDREN: u64 = 4096;
+/// The RAF raw container's pointer to its raw IFD.
+const RAF_RAW_IFD: u16 = 0xf000;
+
+/// Byte ranges `[start, end)` of a file: sorted, disjoint and not touching.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Ranges(Vec<(u64, u64)>);
+
+impl Ranges {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Add `[start, end)` (nothing when it is empty).
+ pub fn insert(&mut self, start: u64, end: u64) {
+ if start >= end {
+ return;
+ }
+ let (mut start, mut end) = (start, end);
+ let mut out = Vec::with_capacity(self.0.len() + 1);
+ let mut placed = false;
+ for &(a, b) in &self.0 {
+ if b < start {
+ out.push((a, b));
+ } else if a > end {
+ if !placed {
+ out.push((start, end));
+ placed = true;
+ }
+ out.push((a, b));
+ } else {
+ start = start.min(a);
+ end = end.max(b);
+ }
+ }
+ if !placed {
+ out.push((start, end));
+ }
+ self.0 = out;
+ }
+
+ /// Whether all of `[start, end)` is covered (an empty range always is).
+ pub fn contains(&self, start: u64, end: u64) -> bool {
+ start >= end || self.0.iter().any(|&(a, b)| a <= start && end <= b)
+ }
+
+ /// Whether some of `[start, end)` is covered.
+ pub fn overlaps(&self, start: u64, end: u64) -> bool {
+ start < end && self.0.iter().any(|&(a, b)| a < end && start < b)
+ }
+
+ pub fn spans(&self) -> &[(u64, u64)] {
+ &self.0
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
+
+ /// Bytes covered.
+ pub fn total(&self) -> u64 {
+ self.0.iter().fold(0u64, |n, &(a, b)| n.saturating_add(b - a))
+ }
+
+ /// The parts of `[start, end)` not covered.
+ pub fn gaps(&self, start: u64, end: u64) -> Ranges {
+ let mut out = Ranges::new();
+ let mut at = start;
+ for &(a, b) in &self.0 {
+ if b <= at {
+ continue;
+ }
+ if a >= end {
+ break;
+ }
+ if a > at {
+ out.insert(at, a);
+ }
+ at = at.max(b);
+ }
+ if at < end {
+ out.insert(at, end);
+ }
+ out
+ }
+
+ /// These ranges without the ones in `other`.
+ pub fn minus(&self, other: &Ranges) -> Ranges {
+ let mut out = Ranges::new();
+ for &(a, b) in &self.0 {
+ for &(x, y) in other.gaps(a, b).spans() {
+ out.insert(x, y);
+ }
+ }
+ out
+ }
+}
+
+/// What [`header_plan`] says about a partly loaded file.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum HeaderPlan {
+ /// Load these ranges (none of them loaded yet) and ask again.
+ Need(Ranges),
+ /// Everything the header readers look at is loaded: probing the buffer gives the whole file's
+ /// result.
+ Ready,
+ /// The file can't be described from part of it (why): read it whole.
+ Whole(&'static str),
+}
+
+/// What of the raw file in `bytes` a header probe reads. `bytes` has the file's full length;
+/// only the ranges in `loaded` hold its bytes (see the module docs).
+pub fn header_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
+ plan(bytes, loaded, false)
+}
+
+/// [`header_plan`] for a raw shown from its embedded preview (a coding not decoded yet): also what
+/// [`crate::embedded_preview`] reads — a CR3's JPEG track and preview boxes, the strips of a TIFF
+/// raw's preview images.
+pub fn preview_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
+ plan(bytes, loaded, true)
+}
+
+fn plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
+ let mut w = Want::new(bytes, loaded);
+ if !w.want(0, 16) {
+ return w.into_plan().unwrap_or(HeaderPlan::Whole("file too short"));
+ }
+ if is_cr3(bytes) {
+ cr3_plan(bytes, loaded, preview)
+ } else if bytes.starts_with(b"FUJIFILMCCD-RAW") {
+ // (its preview, the embedded JPEG, is read anyway: its metadata are there)
+ raf_plan(bytes, loaded)
+ } else if bytes.get(..4).is_some_and(|m| matches!(m, b"IIRO" | b"IIRS" | b"MMOR")) {
+ HeaderPlan::Whole("an ORF header probe reads samples")
+ } else if Tiff::sniff(bytes).is_some() {
+ tiff_plan(bytes, loaded, preview)
+ } else {
+ HeaderPlan::Whole("not a raw layout read in part")
+ }
+}
+
+/// Collects the ranges a plan still needs.
+struct Want<'a> {
+ len: u64,
+ loaded: &'a Ranges,
+ need: Ranges,
+}
+
+impl<'a> Want<'a> {
+ fn new(bytes: &[u8], loaded: &'a Ranges) -> Self {
+ Self { len: bytes.len() as u64, loaded, need: Ranges::new() }
+ }
+
+ /// Whether `[start, end)` (clipped to the file) is loaded; asks for what isn't.
+ fn want(&mut self, start: u64, end: u64) -> bool {
+ let end = end.min(self.len);
+ if self.loaded.contains(start, end) {
+ return true;
+ }
+ for &(a, b) in self.loaded.gaps(start, end).spans() {
+ self.need.insert(a, b);
+ }
+ false
+ }
+
+ /// `Need` when something is missing.
+ fn into_plan(self) -> Option<HeaderPlan> {
+ (!self.need.is_empty()).then_some(HeaderPlan::Need(self.need))
+ }
+}
+
+/// Everything but `holes` is needed. `meta` (IFD tables and out-of-line values) must not lie in a
+/// hole; once all is loaded, `ready` decides.
+fn finish(len: u64, loaded: &Ranges, holes: &Ranges, meta: &Ranges, ready: impl FnOnce() -> HeaderPlan) -> HeaderPlan {
+ if holes.spans().iter().any(|&(a, b)| meta.overlaps(a, b)) {
+ return HeaderPlan::Whole("metadata inside sample data");
+ }
+ let missing = holes.gaps(0, len).minus(loaded);
+ if missing.is_empty() { ready() } else { HeaderPlan::Need(missing) }
+}
+
+fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
+ let len = bytes.len() as u64;
+ let mut w = Want::new(bytes, loaded);
+ let meta = match walk_tiff(&mut w, 0, bytes, &[]) {
+ Ok(meta) => meta,
+ Err(why) => return HeaderPlan::Whole(why),
+ };
+ if let Some(plan) = w.into_plan() {
+ return plan;
+ }
+ // (the Make is usually in the first kilobytes: a NEF, PEF or ORF is known before the rest is read)
+ if matches!(crate::probe(bytes), Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef)) {
+ return HeaderPlan::Whole("its header probe reads samples");
+ }
+ let Ok(tiff) = Tiff::parse(bytes) else { return HeaderPlan::Whole("TIFF structure") };
+ let mut holes = Ranges::new();
+ for ifd in tiff.all_ifds() {
+ let chunks: Vec<(u64, u64)> = match ifd.image() {
+ // a preview's strips are read when the preview is wanted: only the raw image is skipped
+ Ok(info) if preview && !matches!(info.photometric, t::photometric::CFA | t::photometric::LINEAR_RAW) => continue,
+ Ok(info) => info.chunks(len).iter().map(|c| (c.offset, c.len)).collect(),
+ // a raw IFD without the tags a TIFF image needs (an older CR2's) still lists its strips
+ Err(_) => match (ifd.u64s(t::STRIP_OFFSETS), ifd.u64s(t::STRIP_BYTE_COUNTS)) {
+ (Some(offsets), Some(counts)) if offsets.len() == counts.len() => offsets.into_iter().zip(counts).collect(),
+ _ => continue,
+ },
+ };
+ let total = chunks.iter().fold(0u64, |n, c| n.saturating_add(c.1));
+ if total < BULK {
+ continue;
+ }
+ for (i, &(offset, size)) in chunks.iter().enumerate() {
+ let start = if i == 0 { offset.saturating_add(HEAD) } else { offset };
+ holes.insert(start, offset.saturating_add(size).min(len));
+ }
+ }
+ // A Panasonic RW2 keeps its raw data from `RawDataOffset` to the end, outside any TIFF image.
+ if holes.is_empty()
+ && bytes.starts_with(b"IIU\0")
+ && let Some(off) = tiff.ifds.first().and_then(|i| i.u64(crate::vendor::rw2::RAW_OFFSET))
+ {
+ holes.insert(off.saturating_add(HEAD), len);
+ }
+ finish(len, loaded, &holes, &meta, || match crate::probe(bytes) {
+ Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef) => HeaderPlan::Whole("its header probe reads samples"),
+ Some(f) if f.is_supported() => HeaderPlan::Ready,
+ _ => HeaderPlan::Whole("not a supported raw"),
+ })
+}
+
+/// Walk the IFDs of the TIFF in `data` (which starts at `origin` in the file) as `lightcraft_tiff`'s
+/// reader does — the chain, `SubIFDs`, Exif, GPS and Interop, plus the `extra` pointer tags —
+/// asking for every IFD table and for the values that locate IFDs and image chunks. Returns the IFD
+/// tables and out-of-line values found (absolute ranges); incomplete while `w` needs more.
+fn walk_tiff(w: &mut Want, origin: u64, data: &[u8], extra: &[u16]) -> Result<Ranges, &'static str> {
+ let len = data.len() as u64;
+ let at = |rel: u64| origin.saturating_add(rel);
+ let mut meta = Ranges::new();
+ if !w.want(origin, at(len.min(16))) {
+ return Ok(meta);
+ }
+ let (order, big) = Tiff::sniff(data).ok_or("not a TIFF")?;
+ let first = if big { order.read_u64(data, 8) } else { order.read_u32(data, 4).map(u64::from) }.ok_or("truncated TIFF header")?;
+ meta.insert(origin, at(len.min(if big { 16 } else { 8 })));
+ let (head, entry, inline) = if big { (8u64, 20u64, 8u64) } else { (2, 12, 4) };
+ let offset_at = |pos: u64| -> Option<u64> {
+ let pos = usize::try_from(pos).ok()?;
+ if big { order.read_u64(data, pos) } else { order.read_u32(data, pos).map(u64::from) }
+ };
+ let mut stack = vec![first];
+ let mut seen = HashSet::new();
+ while let Some(off) = stack.pop() {
+ if off == 0 || off >= len || seen.len() >= MAX_IFDS || !seen.insert(off) {
+ continue;
+ }
+ if !w.want(at(off), at(off.saturating_add(head))) {
+ continue;
+ }
+ let Ok(pos) = usize::try_from(off) else { continue };
+ let n = if big { order.read_u64(data, pos) } else { order.read_u16(data, pos).map(u64::from) }.unwrap_or(0);
+ let table = off.saturating_add(head);
+ let n_read = n.min(len.saturating_sub(table) / entry).min(MAX_ENTRIES);
+ if n_read == 0 {
+ continue;
+ }
+ let table_end = table.saturating_add(n_read * entry);
+ // the reader reads the next-IFD pointer only when every entry fits
+ let complete = n_read == n;
+ let end = if complete { table_end.saturating_add(inline).min(len) } else { table_end };
+ if !w.want(at(off), at(end)) {
+ continue;
+ }
+ meta.insert(at(off), at(end));
+ for i in 0..n_read {
+ let p = table + i * entry;
+ let Ok(pu) = usize::try_from(p) else { continue };
+ let (Some(tag), Some(typ)) = (order.read_u16(data, pu), order.read_u16(data, pu + 2).and_then(FieldType::from_u16)) else { continue };
+ let count = if big { order.read_u64(data, pu + 4) } else { order.read_u32(data, pu + 4).map(u64::from) };
+ let Some(count) = count else { continue };
+ let field = p + if big { 12 } else { 8 };
+ let Some(total) = count.checked_mul(typ.size() as u64) else { continue };
+ let start = if total <= inline {
+ field
+ } else {
+ match offset_at(field) {
+ Some(v) => v,
+ None => continue,
+ }
+ };
+ let Some(value_end) = start.checked_add(total).filter(|&e| e <= len) else { continue };
+ if total > inline {
+ meta.insert(at(start), at(value_end));
+ }
+ let child = matches!(tag, t::SUB_IFDS | t::EXIF_IFD | t::GPS_IFD | t::INTEROP_IFD) || extra.contains(&tag);
+ let locates = child || matches!(tag, t::STRIP_OFFSETS | t::STRIP_BYTE_COUNTS | t::TILE_OFFSETS | t::TILE_BYTE_COUNTS);
+ if !locates || !w.want(at(start), at(value_end)) || !child {
+ continue;
+ }
+ // the reader follows every `SubIFDs` value and the first value of the other pointers
+ let take = if tag == t::SUB_IFDS { count.min(MAX_CHILDREN) } else { count.min(1) };
+ for k in 0..take {
+ if let Some(v) = unsigned(data, order, typ, start + k * typ.size() as u64) {
+ stack.push(v);
+ }
+ }
+ }
+ if complete && let Some(next) = offset_at(table_end) {
+ stack.push(next);
+ }
+ }
+ Ok(meta)
+}
+
+/// One unsigned value of type `typ` at `pos`.
+fn unsigned(data: &[u8], order: ByteOrder, typ: FieldType, pos: u64) -> Option<u64> {
+ let pos = usize::try_from(pos).ok()?;
+ match typ.size() {
+ 1 => data.get(pos).map(|&b| u64::from(b)),
+ 2 => order.read_u16(data, pos).map(u64::from),
+ 4 => order.read_u32(data, pos).map(u64::from),
+ _ => order.read_u64(data, pos),
+ }
+}
+
+fn be32(bytes: &[u8], at: u64) -> Option<u64> {
+ let at = usize::try_from(at).ok()?;
+ bytes.get(at..at.checked_add(4)?).map(|s| u64::from(u32::from_be_bytes([s[0], s[1], s[2], s[3]])))
+}
+
+fn be64(bytes: &[u8], at: u64) -> Option<u64> {
+ let at = usize::try_from(at).ok()?;
+ bytes.get(at..at.checked_add(8)?).map(|s| u64::from_be_bytes([s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]]))
+}
+
+/// A CR3 is read where `lightcraft_meta::cr3::parse_cr3` and the CR3 decoder look: the file type,
+/// `moov` and the XMP `uuid` box, the first sample of each timed-metadata (`CTMD`) track, and the
+/// main raw track's coding headers and quantization maps. Previews and image tracks are skipped,
+/// unless `preview`: then every top-level box but `mdat` and the JPEG tracks are read too.
+fn cr3_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
+ let len = bytes.len() as u64;
+ let mut w = Want::new(bytes, loaded);
+ // top-level boxes, walked as `parse_cr3` walks them (it stops at the first malformed header)
+ let mut at = 0u64;
+ let mut boxes = 0;
+ while at.saturating_add(8) <= len && boxes < MAX_BOXES {
+ boxes += 1;
+ // size, type, a 64-bit size and a uuid
+ if !w.want(at, at.saturating_add(32)) {
+ break;
+ }
+ let Some(size) = be32(bytes, at) else { break };
+ let (box_len, hdr) = match size {
+ 0 => (len - at, 8),
+ 1 => match be64(bytes, at + 8) {
+ Some(l) => (l, 16),
+ None => break,
+ },
+ n => (n, 8),
+ };
+ let Some(end) = at.checked_add(box_len).filter(|&e| e <= len && box_len >= hdr) else { break };
+ let kind = bytes.get(at as usize + 4..at as usize + 8);
+ let needed = match kind {
+ Some(b"mdat") => false,
+ Some(b"moov") => true,
+ Some(b"uuid") => preview || bytes.get((at + hdr) as usize..(at + hdr + 16) as usize) == Some(XMP_UUID.as_slice()),
+ _ => preview,
+ };
+ if needed {
+ w.want(at, end);
+ }
+ at = end;
+ }
+ if let Some(plan) = w.into_plan() {
+ return plan;
+ }
+ let mut w = Want::new(bytes, loaded);
+ let Some(cr3) = parse_cr3(bytes) else { return HeaderPlan::Whole("CR3 structure") };
+ for track in &cr3.tracks {
+ if (track.kind == Cr3TrackKind::Other(*b"CTMD") || preview && track.kind == Cr3TrackKind::Jpeg)
+ && let Some((offset, size)) = track.data
+ {
+ w.want(offset as u64, offset.saturating_add(size) as u64);
+ }
+ }
+ if let Some(track) = crate::vendor::cr3::main_track(&cr3.tracks)
+ && let Some((offset, size)) = track.data
+ && let Some(coding) = track.compression(bytes)
+ {
+ let o = offset as u64;
+ if w.want(o, o.saturating_add(u64::from(coding.header_size)))
+ && let Some(sample) = bytes.get(offset..offset.saturating_add(size))
+ && let Ok(ranges) = crate::vendor::crx::header_ranges(sample, &coding)
+ {
+ for r in ranges {
+ w.want(o.saturating_add(r.start as u64), o.saturating_add(r.end as u64));
+ }
+ }
+ }
+ w.into_plan().unwrap_or(HeaderPlan::Ready)
+}
+
+/// A RAF is read whole but for its CFA strip: the header, the embedded JPEG (its metadata), the
+/// record directory and the raw container's IFDs and compression header.
+fn raf_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
+ let len = bytes.len() as u64;
+ let mut w = Want::new(bytes, loaded);
+ if !w.want(0, 160) {
+ return w.into_plan().unwrap_or(HeaderPlan::Whole("RAF header"));
+ }
+ let (Some(start), Some(size)) = (be32(bytes, 100), be32(bytes, 104)) else { return HeaderPlan::Whole("RAF header") };
+ let Some(end) = start.checked_add(size).filter(|&e| start > 0 && size > 0 && e <= len) else { return HeaderPlan::Whole("RAF raw container") };
+ let Some(container) = bytes.get(start as usize..end as usize) else { return HeaderPlan::Whole("RAF raw container") };
+ let meta = match walk_tiff(&mut w, start, container, &[RAF_RAW_IFD]) {
+ Ok(meta) => meta,
+ Err(why) => return HeaderPlan::Whole(why),
+ };
+ if let Some(plan) = w.into_plan() {
+ return plan;
+ }
+ let Ok(ifd) = crate::vendor::raf::raw_ifd(container) else { return HeaderPlan::Whole("RAF raw IFD") };
+ let (Some(off), Some(strip)) = (ifd.u64(0xf007), ifd.u64(0xf008)) else { return HeaderPlan::Whole("RAF raw strip") };
+ let mut holes = Ranges::new();
+ if strip >= BULK {
+ let s = start.saturating_add(off);
+ holes.insert(s.saturating_add(HEAD), s.saturating_add(strip).min(end));
+ }
+ finish(len, loaded, &holes, &meta, || HeaderPlan::Ready)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Run a plan to its end, loading from `file` what it asks for; the buffer's unloaded bytes are `fill`.
+ pub(crate) fn run(file: &[u8], prefix: usize, fill: u8) -> (HeaderPlan, Vec<u8>, Ranges) {
+ let mut buf = vec![fill; file.len()];
+ let mut loaded = Ranges::new();
+ let load = |buf: &mut Vec<u8>, loaded: &mut Ranges, a: u64, b: u64| {
+ buf[a as usize..b as usize].copy_from_slice(&file[a as usize..b as usize]);
+ loaded.insert(a, b);
+ };
+ load(&mut buf, &mut loaded, 0, prefix.min(file.len()) as u64);
+ for _ in 0..32 {
+ match header_plan(&buf, &loaded) {
+ HeaderPlan::Need(r) => {
+ for &(a, b) in r.spans() {
+ assert!(!loaded.overlaps(a, b), "asked again for {a}..{b}");
+ load(&mut buf, &mut loaded, a, b);
+ }
+ }
+ done => return (done, buf, loaded),
+ }
+ }
+ panic!("no answer after 32 rounds");
+ }
+
+ #[test]
+ fn ranges_merge_and_subtract() {
+ let mut r = Ranges::new();
+ r.insert(10, 20);
+ r.insert(30, 40);
+ r.insert(20, 25); // touching: merged
+ r.insert(5, 5); // empty
+ assert_eq!(r.spans(), &[(10, 25), (30, 40)]);
+ r.insert(0, 100);
+ assert_eq!(r.spans(), &[(0, 100)]);
+ let mut r = Ranges::new();
+ r.insert(10, 20);
+ r.insert(30, 40);
+ assert!(r.contains(12, 18) && r.contains(5, 5) && !r.contains(15, 35));
+ assert!(r.overlaps(15, 35) && !r.overlaps(20, 30));
+ assert_eq!(r.gaps(0, 50).spans(), &[(0, 10), (20, 30), (40, 50)]);
+ assert_eq!(r.gaps(12, 18).spans(), &[] as &[(u64, u64)]);
+ let mut cut = Ranges::new();
+ cut.insert(15, 32);
+ assert_eq!(r.minus(&cut).spans(), &[(10, 15), (32, 40)]);
+ assert_eq!(r.total(), 20);
+ }
+
+ #[test]
+ fn hostile_input_never_panics_and_ends() {
+ let mut loaded = Ranges::new();
+ for len in [0usize, 1, 8, 15, 16, 200] {
+ let bytes = vec![0u8; len];
+ loaded.insert(0, len as u64);
+ assert!(matches!(header_plan(&bytes, &loaded), HeaderPlan::Whole(_)));
+ }
+ // an IFD that points at itself, an IFD count past the end, a huge SubIFDs count
+ let mut tiff = b"II*\0\x08\0\0\0".to_vec();
+ tiff.extend_from_slice(&2u16.to_le_bytes());
+ tiff.extend_from_slice(&[0x4a, 0x01, 4, 0, 0xff, 0xff, 0xff, 0x0f, 8, 0, 0, 0]); // SubIFDs, 0x0fffffff values at 8
+ tiff.extend_from_slice(&[0x69, 0x87, 4, 0, 1, 0, 0, 0, 8, 0, 0, 0]); // Exif → itself
+ tiff.extend_from_slice(&8u32.to_le_bytes()); // next → itself
+ let (plan, _, _) = run(&tiff, 4, 0);
+ assert!(matches!(plan, HeaderPlan::Whole(_)), "{plan:?}");
+ // a CR3 whose `moov` claims more than the file holds: walked as the parser walks it (it
+ // stops there), so the plan ends with the little there is, and the probe then fails as
+ // it does on the whole file
+ let mut cr3 = 24u32.to_be_bytes().to_vec();
+ cr3.extend_from_slice(b"ftypcrx \0\0\0\x01crx isom");
+ cr3.extend_from_slice(&u32::MAX.to_be_bytes());
+ cr3.extend_from_slice(b"moov");
+ cr3.resize(4096, 0);
+ let (plan, buf, loaded) = run(&cr3, 16, 0);
+ assert_eq!(plan, HeaderPlan::Ready);
+ assert_eq!(loaded.total(), 24 + 32, "only the two box headers are read");
+ assert_eq!(format!("{:?}", crate::probe_info(&buf)), format!("{:?}", crate::probe_info(&cr3)));
+ // a RAF whose container lies outside the file
+ let mut raf = b"FUJIFILMCCD-RAW 0201FF383501".to_vec();
+ raf.resize(160, 0);
+ raf[100..104].copy_from_slice(&1000u32.to_be_bytes());
+ raf[104..108].copy_from_slice(&1000u32.to_be_bytes());
+ assert_eq!(run(&raf, 160, 0).0, HeaderPlan::Whole("RAF raw container"));
+ }
+}
diff --git a/crates/raw/src/vendor/cr3.rs b/crates/raw/src/vendor/cr3.rs
index b5442790c..4ea8761aa 100644
--- a/crates/raw/src/vendor/cr3.rs
+++ b/crates/raw/src/vendor/cr3.rs
@@ -31,7 +31,7 @@ const ASPECT_INFO: u16 = 0x009a;
/// Pick the first largest raw track. Equal-sized later tracks can be the Dual Pixel delta, which
/// must never replace the main sensor image. An invalid full-size descriptor is reported, not
/// silently replaced with the reduced raw preview.
-fn main_track(tracks: &[Cr3Track]) -> Option<&Cr3Track> {
+pub(crate) fn main_track(tracks: &[Cr3Track]) -> Option<&Cr3Track> {
let mut best = None;
let mut best_area = 0u64;
for track in tracks {
diff --git a/crates/raw/src/vendor/crx.rs b/crates/raw/src/vendor/crx.rs
index 2d15627fa..d9cc5cd14 100644
--- a/crates/raw/src/vendor/crx.rs
+++ b/crates/raw/src/vendor/crx.rs
@@ -68,6 +68,15 @@ struct Tile {
qp: Option<Range<usize>>,
}
+/// The byte ranges of a CRX sample (relative to its start) that [`validate`] reads: the tile, plane
+/// and band headers, and each tile's quantization map. `sample` must hold its first
+/// `config.header_size` bytes; the rest is only measured.
+pub(crate) fn header_ranges(sample: &[u8], config: &Cr3Compression) -> Result<Vec<Range<usize>>> {
+ let tiles = layout(sample, config)?;
+ let header = 0..config.header_size as usize;
+ Ok(std::iter::once(header).chain(tiles.into_iter().filter_map(|t| t.qp)).collect())
+}
+
/// Decode to full sensor-order CFA samples, including masked borders.
pub(crate) fn decode(sample: &[u8], config: &Cr3Compression) -> Result<Vec<u16>> {
supports(config)?;
diff --git a/crates/raw/src/vendor/raf.rs b/crates/raw/src/vendor/raf.rs
index 61205a3f7..00406d87f 100644
--- a/crates/raw/src/vendor/raf.rs
+++ b/crates/raw/src/vendor/raf.rs
@@ -85,7 +85,7 @@ fn pair(h: &Header, tag: u16) -> Option<(usize, usize)> {
Some((u16::from_be_bytes([d[0], d[1]]) as usize, u16::from_be_bytes([d[2], d[3]]) as usize))
}
-fn raw_ifd(raw: &[u8]) -> Result<Ifd> {
+pub(crate) fn raw_ifd(raw: &[u8]) -> Result<Ifd> {
let tiff = Tiff::parse(raw).map_err(|_| RawError::Unsupported("RAF without a raw IFD (older FinePix layout)".into()))?;
let off = tiff.ifds.first().and_then(|i| i.u64(0xf000)).ok_or_else(|| RawError::Unsupported("RAF without a raw IFD".into()))?;
let (ifd, _) = lightcraft_tiff::parse_ifd_at(raw, off, ByteOrder::Little, 0, false, &Default::default())?;
diff --git a/crates/raw/src/vendor/rw2.rs b/crates/raw/src/vendor/rw2.rs
index 7fe6d0e42..0a64c9e44 100644
--- a/crates/raw/src/vendor/rw2.rs
+++ b/crates/raw/src/vendor/rw2.rs
@@ -122,7 +122,7 @@ const RAW_FORMAT: u16 = 0x002d;
/// The in-camera aspect-ratio crop: top, left, bottom, right in sensor coordinates.
const CROP: [u16; 4] = [0x002f, 0x0030, 0x0031, 0x0032];
const ISO_32: u16 = 0x0037;
-const RAW_OFFSET: u16 = 0x0118;
+pub(crate) const RAW_OFFSET: u16 = 0x0118;
const HUFFMAN_CODES: u16 = 0x0040;
const STRIP_OFFSETS: u16 = 0x0044;
const STRIP_LEFT: u16 = 0x0045;
diff --git a/crates/raw/tests/sparse_corpus.rs b/crates/raw/tests/sparse_corpus.rs
new file mode 100644
index 000000000..3ab735299
--- /dev/null
+++ b/crates/raw/tests/sparse_corpus.rs
@@ -0,0 +1,108 @@
+//! [`lightcraft_raw::sparse::header_plan`] over `corpus/raw/**` (git-ignored CC0 samples from
+//! raw.pixls.us, fetched with `cargo xtask corpus --download`; `LIGHTCRAFT_CORPUS` overrides the
+//! corpus root). Skips cleanly when absent.
+//!
+//! For every file the plan reaches `Ready` on, probing the partly loaded buffer must give exactly
+//! what probing the whole file gives — with the skipped bytes zero and again with them 0xFF, so a
+//! reader that looked at skipped data would be caught. CR3 and DNG files must be described from a
+//! small part of the file (`cargo test -p lightcraft-raw --release --test sparse_corpus -- --nocapture`
+//! prints what each format reads).
+
+use lightcraft_raw::sparse::{HeaderPlan, Ranges, header_plan, preview_plan};
+use lightcraft_raw::{embedded_preview, probe, probe_info};
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+fn corpus_root() -> PathBuf {
+ std::env::var_os("LIGHTCRAFT_CORPUS").map(PathBuf::from).unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
+}
+
+/// Run the plan (with the embedded preview when `preview`), loading from `file` what it asks for
+/// into a buffer whose other bytes are `fill`.
+fn run(file: &[u8], fill: u8, preview: bool) -> (HeaderPlan, Vec<u8>, Ranges) {
+ let mut buf = vec![fill; file.len()];
+ let mut loaded = Ranges::new();
+ let prefix = file.len().min(512 * 1024);
+ buf[..prefix].copy_from_slice(&file[..prefix]);
+ loaded.insert(0, prefix as u64);
+ for _ in 0..32 {
+ match if preview { preview_plan(&buf, &loaded) } else { header_plan(&buf, &loaded) } {
+ HeaderPlan::Need(r) => {
+ for &(a, b) in r.spans() {
+ assert!(!loaded.overlaps(a, b), "asked again for {a}..{b}");
+ buf[a as usize..b as usize].copy_from_slice(&file[a as usize..b as usize]);
+ loaded.insert(a, b);
+ }
+ }
+ done => return (done, buf, loaded),
+ }
+ }
+ panic!("no answer after 32 rounds");
+}
+
+/// What an import learns from a file: the raw description and the metadata.
+fn describe(bytes: &[u8]) -> String {
+ let e = lightcraft_meta::embedded(bytes);
+ format!("{:?}\n{:?}\n{:?}", probe_info(bytes), lightcraft_meta::extract(bytes), e.xmp)
+}
+
+#[test]
+fn partial_reads_describe_raws_as_whole_files_do() {
+ let dir = corpus_root().join("raw");
+ let mut paths: Vec<PathBuf> = std::fs::read_dir(corpus_root().join("leica"))
+ .into_iter()
+ .chain(std::fs::read_dir(&dir))
+ .flatten()
+ .flatten()
+ .map(|e| e.path())
+ .filter(|p| p.is_file() && !p.extension().is_some_and(|e| e == "part" || e == "txt" || e == "md"))
+ .collect();
+ if paths.is_empty() {
+ eprintln!("skip: {} absent", dir.display());
+ return;
+ }
+ paths.sort();
+ // format → (files ready, bytes read, bytes total, files read whole)
+ let mut stats: BTreeMap<String, (usize, u64, u64, usize)> = BTreeMap::new();
+ for p in paths {
+ let name = p.file_name().unwrap().to_string_lossy().to_string();
+ let file = std::fs::read(&p).unwrap();
+ let format = probe(&file).map_or("?".to_string(), |f| format!("{f:?}"));
+ let (plan, zeros, loaded) = run(&file, 0, false);
+ let entry = stats.entry(format.clone()).or_default();
+ match plan {
+ HeaderPlan::Ready => {
+ let (plan_ff, ones, loaded_ff) = run(&file, 0xff, false);
+ assert_eq!((plan_ff, &loaded_ff), (HeaderPlan::Ready, &loaded), "{name}: the plan read skipped bytes");
+ let whole = describe(&file);
+ assert_eq!(describe(&zeros), whole, "{name}: differs from the whole file (skipped bytes 0)");
+ assert_eq!(describe(&ones), whole, "{name}: differs from the whole file (skipped bytes 0xFF)");
+ entry.0 += 1;
+ entry.1 += loaded.total();
+ entry.2 += file.len() as u64;
+ // with the preview (for a raw shown from it), the preview is what the whole file gives too
+ let preview = embedded_preview(&file);
+ for fill in [0, 0xff] {
+ let (plan, buf, _) = run(&file, fill, true);
+ assert_eq!(plan, HeaderPlan::Ready, "{name}: preview plan");
+ assert_eq!(describe(&buf), whole, "{name}: preview plan, skipped bytes {fill}");
+ assert!(embedded_preview(&buf) == preview, "{name}: a different embedded preview (skipped bytes {fill})");
+ }
+ if loaded.total() * 4 > file.len() as u64 {
+ eprintln!("{name}: read {:.0} %", loaded.total() as f64 * 100.0 / file.len() as f64);
+ }
+ if matches!(format.as_str(), "Cr3" | "Dng") && file.len() > 8 << 20 {
+ assert!(loaded.total() * 4 < file.len() as u64, "{name}: read {} of {} bytes", loaded.total(), file.len());
+ }
+ }
+ HeaderPlan::Whole(why) => {
+ entry.3 += 1;
+ eprintln!("{name}: whole ({why})");
+ }
+ HeaderPlan::Need(_) => unreachable!(),
+ }
+ }
+ for (format, (ready, read, total, whole)) in stats {
+ eprintln!("{format:>10}: {ready} described from {:.1} % of their bytes, {whole} read whole", read as f64 * 100.0 / total.max(1) as f64);
+ }
+}