LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LRC-LIB-CATALOG-IMPORT: describe raws from their headers without reading pixel data
ingolfteetzbbd973f32106
diff --git a/crates/meta/src/cr3.rs b/crates/meta/src/cr3.rsindex 5fa6437bc..55b3ae6a6 100644--- a/crates/meta/src/cr3.rs+++ b/crates/meta/src/cr3.rs@@ -226,7 +226,8 @@ impl Cr3<'_> {}const CANON_UUID: [u8; 16] = [0x85, 0xc0, 0xb6, 0x87, 0x82, 0x0f, 0x11, 0xe0, 0x81, 0x11, 0xf4, 0xce, 0x46, 0x2b, 0x6a, 0x48];-const XMP_UUID: [u8; 16] = [0xbe, 0x7a, 0xcf, 0xcb, 0x97, 0xa9, 0x42, 0xe8, 0x9c, 0x71, 0x99, 0x94, 0x91, 0xe3, 0xaf, 0xac];+/// The `uuid` of the top-level box holding the XMP packet.+pub const XMP_UUID: [u8; 16] = [0xbe, 0x7a, 0xcf, 0xcb, 0x97, 0xa9, 0x42, 0xe8, 0x9c, 0x71, 0x99, 0x94, 0x91, 0xe3, 0xaf, 0xac];const MAX_DEPTH: usize = 12;const MAX_BOXES: usize = 4096;/// A `VisualSampleEntry` (78 bytes after the box header) plus 4 bytes Canon adds before the child boxes.@@ -237,7 +238,8 @@ pub fn is_cr3(bytes: &[u8]) -> bool {bytes.get(4..12) == Some(b"ftypcrx ".as_slice())}-/// Parse a CR3 file's boxes; `None` when it is not a CR3 file.+/// Parse a CR3 file's boxes; `None` when it is not a CR3 file. Of the top-level boxes only `moov`+/// and the XMP `uuid` are read (`lightcraft_raw::sparse` relies on it to skip the others).pub fn parse_cr3(bytes: &[u8]) -> Option<Cr3<'_>> {if !is_cr3(bytes) {return None;diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rsindex f0331d610..2c2114110 100644--- a/crates/raw/src/lib.rs+++ b/crates/raw/src/lib.rs@@ -40,6 +40,7 @@ mod preview;pub mod profile;mod saturation;pub mod semantic;+pub mod sparse;pub mod spectral;mod tiffraw;mod unpack;diff --git a/crates/raw/src/sparse.rs b/crates/raw/src/sparse.rsnew file mode 100644index 000000000..14fa3c1f8--- /dev/null+++ b/crates/raw/src/sparse.rs@@ -0,0 +1,553 @@+//! Describing a raw file without reading its pixel data.+//!+//! [`probe_info`](crate::probe_info) and `lightcraft_meta::extract` read only a raw file's+//! headers, but those are a few hundred kilobytes of a file of tens of megabytes: an import from a+//! slow drive spends nearly all its time reading samples nothing looks at. [`header_plan`] says+//! which byte ranges of a file those readers need. It is asked about a buffer of the file's full+//! length in which only the ranges in `loaded` hold the file's bytes (the rest is zeros); the caller+//! loads the ranges it names and asks again. Once it says [`HeaderPlan::Ready`], probing the buffer+//! gives what probing the whole file gives.+//!+//! Only bulk sample data is left out: the strips and tiles of each large TIFF image (keeping the+//! start of the first one, where the coding headers a header probe validates sit), a CR3's image+//! tracks (keeping the raw track's coding headers and quantization maps, and the timed-metadata+//! track where Canon puts the as-shot white balance), a RAF's CFA strip (keeping its compression+//! header) and a Panasonic RW2's raw data. Formats whose header probe reads samples (NEF: masked+//! columns; ORF: bit depth and CFA phase; PEF: dark borders) and anything else get+//! [`HeaderPlan::Whole`], as does a structure that is incomplete, inconsistent or keeps metadata+//! inside its sample data: the plan never guesses.++use std::collections::HashSet;++use lightcraft_meta::cr3::{Cr3TrackKind, XMP_UUID, is_cr3, parse_cr3};+use lightcraft_tiff::{ByteOrder, FieldType, Tiff, tags as t};++use crate::RawFormat;++/// Bytes kept at the start of a skipped block: the coding headers a header probe validates there+/// (lossless-JPEG frame headers, a RAF's compression header and stripe table) fit with room to spare.+const HEAD: u64 = 64 * 1024;+/// An image whose chunks hold fewer bytes than this is read (thumbnails, small previews).+const BULK: u64 = 256 * 1024;+/// The same bounds as `lightcraft_tiff`'s reader.+const MAX_IFDS: usize = 1024;+const MAX_ENTRIES: u64 = 8192;+const MAX_BOXES: usize = 4096;+/// Child IFDs listed by one `SubIFDs` entry.+const MAX_CHILDREN: u64 = 4096;+/// The RAF raw container's pointer to its raw IFD.+const RAF_RAW_IFD: u16 = 0xf000;++/// Byte ranges `[start, end)` of a file: sorted, disjoint and not touching.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct Ranges(Vec<(u64, u64)>);++impl Ranges {+ pub fn new() -> Self {+ Self::default()+ }++ /// Add `[start, end)` (nothing when it is empty).+ pub fn insert(&mut self, start: u64, end: u64) {+ if start >= end {+ return;+ }+ let (mut start, mut end) = (start, end);+ let mut out = Vec::with_capacity(self.0.len() + 1);+ let mut placed = false;+ for &(a, b) in &self.0 {+ if b < start {+ out.push((a, b));+ } else if a > end {+ if !placed {+ out.push((start, end));+ placed = true;+ }+ out.push((a, b));+ } else {+ start = start.min(a);+ end = end.max(b);+ }+ }+ if !placed {+ out.push((start, end));+ }+ self.0 = out;+ }++ /// Whether all of `[start, end)` is covered (an empty range always is).+ pub fn contains(&self, start: u64, end: u64) -> bool {+ start >= end || self.0.iter().any(|&(a, b)| a <= start && end <= b)+ }++ /// Whether some of `[start, end)` is covered.+ pub fn overlaps(&self, start: u64, end: u64) -> bool {+ start < end && self.0.iter().any(|&(a, b)| a < end && start < b)+ }++ pub fn spans(&self) -> &[(u64, u64)] {+ &self.0+ }++ pub fn is_empty(&self) -> bool {+ self.0.is_empty()+ }++ /// Bytes covered.+ pub fn total(&self) -> u64 {+ self.0.iter().fold(0u64, |n, &(a, b)| n.saturating_add(b - a))+ }++ /// The parts of `[start, end)` not covered.+ pub fn gaps(&self, start: u64, end: u64) -> Ranges {+ let mut out = Ranges::new();+ let mut at = start;+ for &(a, b) in &self.0 {+ if b <= at {+ continue;+ }+ if a >= end {+ break;+ }+ if a > at {+ out.insert(at, a);+ }+ at = at.max(b);+ }+ if at < end {+ out.insert(at, end);+ }+ out+ }++ /// These ranges without the ones in `other`.+ pub fn minus(&self, other: &Ranges) -> Ranges {+ let mut out = Ranges::new();+ for &(a, b) in &self.0 {+ for &(x, y) in other.gaps(a, b).spans() {+ out.insert(x, y);+ }+ }+ out+ }+}++/// What [`header_plan`] says about a partly loaded file.+#[derive(Clone, Debug, PartialEq, Eq)]+pub enum HeaderPlan {+ /// Load these ranges (none of them loaded yet) and ask again.+ Need(Ranges),+ /// Everything the header readers look at is loaded: probing the buffer gives the whole file's+ /// result.+ Ready,+ /// The file can't be described from part of it (why): read it whole.+ Whole(&'static str),+}++/// What of the raw file in `bytes` a header probe reads. `bytes` has the file's full length;+/// only the ranges in `loaded` hold its bytes (see the module docs).+pub fn header_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {+ plan(bytes, loaded, false)+}++/// [`header_plan`] for a raw shown from its embedded preview (a coding not decoded yet): also what+/// [`crate::embedded_preview`] reads — a CR3's JPEG track and preview boxes, the strips of a TIFF+/// raw's preview images.+pub fn preview_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {+ plan(bytes, loaded, true)+}++fn plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {+ let mut w = Want::new(bytes, loaded);+ if !w.want(0, 16) {+ return w.into_plan().unwrap_or(HeaderPlan::Whole("file too short"));+ }+ if is_cr3(bytes) {+ cr3_plan(bytes, loaded, preview)+ } else if bytes.starts_with(b"FUJIFILMCCD-RAW") {+ // (its preview, the embedded JPEG, is read anyway: its metadata are there)+ raf_plan(bytes, loaded)+ } else if bytes.get(..4).is_some_and(|m| matches!(m, b"IIRO" | b"IIRS" | b"MMOR")) {+ HeaderPlan::Whole("an ORF header probe reads samples")+ } else if Tiff::sniff(bytes).is_some() {+ tiff_plan(bytes, loaded, preview)+ } else {+ HeaderPlan::Whole("not a raw layout read in part")+ }+}++/// Collects the ranges a plan still needs.+struct Want<'a> {+ len: u64,+ loaded: &'a Ranges,+ need: Ranges,+}++impl<'a> Want<'a> {+ fn new(bytes: &[u8], loaded: &'a Ranges) -> Self {+ Self { len: bytes.len() as u64, loaded, need: Ranges::new() }+ }++ /// Whether `[start, end)` (clipped to the file) is loaded; asks for what isn't.+ fn want(&mut self, start: u64, end: u64) -> bool {+ let end = end.min(self.len);+ if self.loaded.contains(start, end) {+ return true;+ }+ for &(a, b) in self.loaded.gaps(start, end).spans() {+ self.need.insert(a, b);+ }+ false+ }++ /// `Need` when something is missing.+ fn into_plan(self) -> Option<HeaderPlan> {+ (!self.need.is_empty()).then_some(HeaderPlan::Need(self.need))+ }+}++/// Everything but `holes` is needed. `meta` (IFD tables and out-of-line values) must not lie in a+/// hole; once all is loaded, `ready` decides.+fn finish(len: u64, loaded: &Ranges, holes: &Ranges, meta: &Ranges, ready: impl FnOnce() -> HeaderPlan) -> HeaderPlan {+ if holes.spans().iter().any(|&(a, b)| meta.overlaps(a, b)) {+ return HeaderPlan::Whole("metadata inside sample data");+ }+ let missing = holes.gaps(0, len).minus(loaded);+ if missing.is_empty() { ready() } else { HeaderPlan::Need(missing) }+}++fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {+ let len = bytes.len() as u64;+ let mut w = Want::new(bytes, loaded);+ let meta = match walk_tiff(&mut w, 0, bytes, &[]) {+ Ok(meta) => meta,+ Err(why) => return HeaderPlan::Whole(why),+ };+ if let Some(plan) = w.into_plan() {+ return plan;+ }+ // (the Make is usually in the first kilobytes: a NEF, PEF or ORF is known before the rest is read)+ if matches!(crate::probe(bytes), Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef)) {+ return HeaderPlan::Whole("its header probe reads samples");+ }+ let Ok(tiff) = Tiff::parse(bytes) else { return HeaderPlan::Whole("TIFF structure") };+ let mut holes = Ranges::new();+ for ifd in tiff.all_ifds() {+ let chunks: Vec<(u64, u64)> = match ifd.image() {+ // a preview's strips are read when the preview is wanted: only the raw image is skipped+ Ok(info) if preview && !matches!(info.photometric, t::photometric::CFA | t::photometric::LINEAR_RAW) => continue,+ Ok(info) => info.chunks(len).iter().map(|c| (c.offset, c.len)).collect(),+ // a raw IFD without the tags a TIFF image needs (an older CR2's) still lists its strips+ Err(_) => match (ifd.u64s(t::STRIP_OFFSETS), ifd.u64s(t::STRIP_BYTE_COUNTS)) {+ (Some(offsets), Some(counts)) if offsets.len() == counts.len() => offsets.into_iter().zip(counts).collect(),+ _ => continue,+ },+ };+ let total = chunks.iter().fold(0u64, |n, c| n.saturating_add(c.1));+ if total < BULK {+ continue;+ }+ for (i, &(offset, size)) in chunks.iter().enumerate() {+ let start = if i == 0 { offset.saturating_add(HEAD) } else { offset };+ holes.insert(start, offset.saturating_add(size).min(len));+ }+ }+ // A Panasonic RW2 keeps its raw data from `RawDataOffset` to the end, outside any TIFF image.+ if holes.is_empty()+ && bytes.starts_with(b"IIU\0")+ && let Some(off) = tiff.ifds.first().and_then(|i| i.u64(crate::vendor::rw2::RAW_OFFSET))+ {+ holes.insert(off.saturating_add(HEAD), len);+ }+ finish(len, loaded, &holes, &meta, || match crate::probe(bytes) {+ Some(RawFormat::Nef | RawFormat::Nrw | RawFormat::Orf | RawFormat::Pef) => HeaderPlan::Whole("its header probe reads samples"),+ Some(f) if f.is_supported() => HeaderPlan::Ready,+ _ => HeaderPlan::Whole("not a supported raw"),+ })+}++/// Walk the IFDs of the TIFF in `data` (which starts at `origin` in the file) as `lightcraft_tiff`'s+/// reader does — the chain, `SubIFDs`, Exif, GPS and Interop, plus the `extra` pointer tags —+/// asking for every IFD table and for the values that locate IFDs and image chunks. Returns the IFD+/// tables and out-of-line values found (absolute ranges); incomplete while `w` needs more.+fn walk_tiff(w: &mut Want, origin: u64, data: &[u8], extra: &[u16]) -> Result<Ranges, &'static str> {+ let len = data.len() as u64;+ let at = |rel: u64| origin.saturating_add(rel);+ let mut meta = Ranges::new();+ if !w.want(origin, at(len.min(16))) {+ return Ok(meta);+ }+ let (order, big) = Tiff::sniff(data).ok_or("not a TIFF")?;+ let first = if big { order.read_u64(data, 8) } else { order.read_u32(data, 4).map(u64::from) }.ok_or("truncated TIFF header")?;+ meta.insert(origin, at(len.min(if big { 16 } else { 8 })));+ let (head, entry, inline) = if big { (8u64, 20u64, 8u64) } else { (2, 12, 4) };+ let offset_at = |pos: u64| -> Option<u64> {+ let pos = usize::try_from(pos).ok()?;+ if big { order.read_u64(data, pos) } else { order.read_u32(data, pos).map(u64::from) }+ };+ let mut stack = vec![first];+ let mut seen = HashSet::new();+ while let Some(off) = stack.pop() {+ if off == 0 || off >= len || seen.len() >= MAX_IFDS || !seen.insert(off) {+ continue;+ }+ if !w.want(at(off), at(off.saturating_add(head))) {+ continue;+ }+ let Ok(pos) = usize::try_from(off) else { continue };+ let n = if big { order.read_u64(data, pos) } else { order.read_u16(data, pos).map(u64::from) }.unwrap_or(0);+ let table = off.saturating_add(head);+ let n_read = n.min(len.saturating_sub(table) / entry).min(MAX_ENTRIES);+ if n_read == 0 {+ continue;+ }+ let table_end = table.saturating_add(n_read * entry);+ // the reader reads the next-IFD pointer only when every entry fits+ let complete = n_read == n;+ let end = if complete { table_end.saturating_add(inline).min(len) } else { table_end };+ if !w.want(at(off), at(end)) {+ continue;+ }+ meta.insert(at(off), at(end));+ for i in 0..n_read {+ let p = table + i * entry;+ let Ok(pu) = usize::try_from(p) else { continue };+ let (Some(tag), Some(typ)) = (order.read_u16(data, pu), order.read_u16(data, pu + 2).and_then(FieldType::from_u16)) else { continue };+ let count = if big { order.read_u64(data, pu + 4) } else { order.read_u32(data, pu + 4).map(u64::from) };+ let Some(count) = count else { continue };+ let field = p + if big { 12 } else { 8 };+ let Some(total) = count.checked_mul(typ.size() as u64) else { continue };+ let start = if total <= inline {+ field+ } else {+ match offset_at(field) {+ Some(v) => v,+ None => continue,+ }+ };+ let Some(value_end) = start.checked_add(total).filter(|&e| e <= len) else { continue };+ if total > inline {+ meta.insert(at(start), at(value_end));+ }+ let child = matches!(tag, t::SUB_IFDS | t::EXIF_IFD | t::GPS_IFD | t::INTEROP_IFD) || extra.contains(&tag);+ let locates = child || matches!(tag, t::STRIP_OFFSETS | t::STRIP_BYTE_COUNTS | t::TILE_OFFSETS | t::TILE_BYTE_COUNTS);+ if !locates || !w.want(at(start), at(value_end)) || !child {+ continue;+ }+ // the reader follows every `SubIFDs` value and the first value of the other pointers+ let take = if tag == t::SUB_IFDS { count.min(MAX_CHILDREN) } else { count.min(1) };+ for k in 0..take {+ if let Some(v) = unsigned(data, order, typ, start + k * typ.size() as u64) {+ stack.push(v);+ }+ }+ }+ if complete && let Some(next) = offset_at(table_end) {+ stack.push(next);+ }+ }+ Ok(meta)+}++/// One unsigned value of type `typ` at `pos`.+fn unsigned(data: &[u8], order: ByteOrder, typ: FieldType, pos: u64) -> Option<u64> {+ let pos = usize::try_from(pos).ok()?;+ match typ.size() {+ 1 => data.get(pos).map(|&b| u64::from(b)),+ 2 => order.read_u16(data, pos).map(u64::from),+ 4 => order.read_u32(data, pos).map(u64::from),+ _ => order.read_u64(data, pos),+ }+}++fn be32(bytes: &[u8], at: u64) -> Option<u64> {+ let at = usize::try_from(at).ok()?;+ bytes.get(at..at.checked_add(4)?).map(|s| u64::from(u32::from_be_bytes([s[0], s[1], s[2], s[3]])))+}++fn be64(bytes: &[u8], at: u64) -> Option<u64> {+ let at = usize::try_from(at).ok()?;+ bytes.get(at..at.checked_add(8)?).map(|s| u64::from_be_bytes([s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]]))+}++/// A CR3 is read where `lightcraft_meta::cr3::parse_cr3` and the CR3 decoder look: the file type,+/// `moov` and the XMP `uuid` box, the first sample of each timed-metadata (`CTMD`) track, and the+/// main raw track's coding headers and quantization maps. Previews and image tracks are skipped,+/// unless `preview`: then every top-level box but `mdat` and the JPEG tracks are read too.+fn cr3_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {+ let len = bytes.len() as u64;+ let mut w = Want::new(bytes, loaded);+ // top-level boxes, walked as `parse_cr3` walks them (it stops at the first malformed header)+ let mut at = 0u64;+ let mut boxes = 0;+ while at.saturating_add(8) <= len && boxes < MAX_BOXES {+ boxes += 1;+ // size, type, a 64-bit size and a uuid+ if !w.want(at, at.saturating_add(32)) {+ break;+ }+ let Some(size) = be32(bytes, at) else { break };+ let (box_len, hdr) = match size {+ 0 => (len - at, 8),+ 1 => match be64(bytes, at + 8) {+ Some(l) => (l, 16),+ None => break,+ },+ n => (n, 8),+ };+ let Some(end) = at.checked_add(box_len).filter(|&e| e <= len && box_len >= hdr) else { break };+ let kind = bytes.get(at as usize + 4..at as usize + 8);+ let needed = match kind {+ Some(b"mdat") => false,+ Some(b"moov") => true,+ Some(b"uuid") => preview || bytes.get((at + hdr) as usize..(at + hdr + 16) as usize) == Some(XMP_UUID.as_slice()),+ _ => preview,+ };+ if needed {+ w.want(at, end);+ }+ at = end;+ }+ if let Some(plan) = w.into_plan() {+ return plan;+ }+ let mut w = Want::new(bytes, loaded);+ let Some(cr3) = parse_cr3(bytes) else { return HeaderPlan::Whole("CR3 structure") };+ for track in &cr3.tracks {+ if (track.kind == Cr3TrackKind::Other(*b"CTMD") || preview && track.kind == Cr3TrackKind::Jpeg)+ && let Some((offset, size)) = track.data+ {+ w.want(offset as u64, offset.saturating_add(size) as u64);+ }+ }+ if let Some(track) = crate::vendor::cr3::main_track(&cr3.tracks)+ && let Some((offset, size)) = track.data+ && let Some(coding) = track.compression(bytes)+ {+ let o = offset as u64;+ if w.want(o, o.saturating_add(u64::from(coding.header_size)))+ && let Some(sample) = bytes.get(offset..offset.saturating_add(size))+ && let Ok(ranges) = crate::vendor::crx::header_ranges(sample, &coding)+ {+ for r in ranges {+ w.want(o.saturating_add(r.start as u64), o.saturating_add(r.end as u64));+ }+ }+ }+ w.into_plan().unwrap_or(HeaderPlan::Ready)+}++/// A RAF is read whole but for its CFA strip: the header, the embedded JPEG (its metadata), the+/// record directory and the raw container's IFDs and compression header.+fn raf_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {+ let len = bytes.len() as u64;+ let mut w = Want::new(bytes, loaded);+ if !w.want(0, 160) {+ return w.into_plan().unwrap_or(HeaderPlan::Whole("RAF header"));+ }+ let (Some(start), Some(size)) = (be32(bytes, 100), be32(bytes, 104)) else { return HeaderPlan::Whole("RAF header") };+ let Some(end) = start.checked_add(size).filter(|&e| start > 0 && size > 0 && e <= len) else { return HeaderPlan::Whole("RAF raw container") };+ let Some(container) = bytes.get(start as usize..end as usize) else { return HeaderPlan::Whole("RAF raw container") };+ let meta = match walk_tiff(&mut w, start, container, &[RAF_RAW_IFD]) {+ Ok(meta) => meta,+ Err(why) => return HeaderPlan::Whole(why),+ };+ if let Some(plan) = w.into_plan() {+ return plan;+ }+ let Ok(ifd) = crate::vendor::raf::raw_ifd(container) else { return HeaderPlan::Whole("RAF raw IFD") };+ let (Some(off), Some(strip)) = (ifd.u64(0xf007), ifd.u64(0xf008)) else { return HeaderPlan::Whole("RAF raw strip") };+ let mut holes = Ranges::new();+ if strip >= BULK {+ let s = start.saturating_add(off);+ holes.insert(s.saturating_add(HEAD), s.saturating_add(strip).min(end));+ }+ finish(len, loaded, &holes, &meta, || HeaderPlan::Ready)+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Run a plan to its end, loading from `file` what it asks for; the buffer's unloaded bytes are `fill`.+ pub(crate) fn run(file: &[u8], prefix: usize, fill: u8) -> (HeaderPlan, Vec<u8>, Ranges) {+ let mut buf = vec![fill; file.len()];+ let mut loaded = Ranges::new();+ let load = |buf: &mut Vec<u8>, loaded: &mut Ranges, a: u64, b: u64| {+ buf[a as usize..b as usize].copy_from_slice(&file[a as usize..b as usize]);+ loaded.insert(a, b);+ };+ load(&mut buf, &mut loaded, 0, prefix.min(file.len()) as u64);+ for _ in 0..32 {+ match header_plan(&buf, &loaded) {+ HeaderPlan::Need(r) => {+ for &(a, b) in r.spans() {+ assert!(!loaded.overlaps(a, b), "asked again for {a}..{b}");+ load(&mut buf, &mut loaded, a, b);+ }+ }+ done => return (done, buf, loaded),+ }+ }+ panic!("no answer after 32 rounds");+ }++ #[test]+ fn ranges_merge_and_subtract() {+ let mut r = Ranges::new();+ r.insert(10, 20);+ r.insert(30, 40);+ r.insert(20, 25); // touching: merged+ r.insert(5, 5); // empty+ assert_eq!(r.spans(), &[(10, 25), (30, 40)]);+ r.insert(0, 100);+ assert_eq!(r.spans(), &[(0, 100)]);+ let mut r = Ranges::new();+ r.insert(10, 20);+ r.insert(30, 40);+ assert!(r.contains(12, 18) && r.contains(5, 5) && !r.contains(15, 35));+ assert!(r.overlaps(15, 35) && !r.overlaps(20, 30));+ assert_eq!(r.gaps(0, 50).spans(), &[(0, 10), (20, 30), (40, 50)]);+ assert_eq!(r.gaps(12, 18).spans(), &[] as &[(u64, u64)]);+ let mut cut = Ranges::new();+ cut.insert(15, 32);+ assert_eq!(r.minus(&cut).spans(), &[(10, 15), (32, 40)]);+ assert_eq!(r.total(), 20);+ }++ #[test]+ fn hostile_input_never_panics_and_ends() {+ let mut loaded = Ranges::new();+ for len in [0usize, 1, 8, 15, 16, 200] {+ let bytes = vec![0u8; len];+ loaded.insert(0, len as u64);+ assert!(matches!(header_plan(&bytes, &loaded), HeaderPlan::Whole(_)));+ }+ // an IFD that points at itself, an IFD count past the end, a huge SubIFDs count+ let mut tiff = b"II*\0\x08\0\0\0".to_vec();+ tiff.extend_from_slice(&2u16.to_le_bytes());+ tiff.extend_from_slice(&[0x4a, 0x01, 4, 0, 0xff, 0xff, 0xff, 0x0f, 8, 0, 0, 0]); // SubIFDs, 0x0fffffff values at 8+ tiff.extend_from_slice(&[0x69, 0x87, 4, 0, 1, 0, 0, 0, 8, 0, 0, 0]); // Exif → itself+ tiff.extend_from_slice(&8u32.to_le_bytes()); // next → itself+ let (plan, _, _) = run(&tiff, 4, 0);+ assert!(matches!(plan, HeaderPlan::Whole(_)), "{plan:?}");+ // a CR3 whose `moov` claims more than the file holds: walked as the parser walks it (it+ // stops there), so the plan ends with the little there is, and the probe then fails as+ // it does on the whole file+ let mut cr3 = 24u32.to_be_bytes().to_vec();+ cr3.extend_from_slice(b"ftypcrx \0\0\0\x01crx isom");+ cr3.extend_from_slice(&u32::MAX.to_be_bytes());+ cr3.extend_from_slice(b"moov");+ cr3.resize(4096, 0);+ let (plan, buf, loaded) = run(&cr3, 16, 0);+ assert_eq!(plan, HeaderPlan::Ready);+ assert_eq!(loaded.total(), 24 + 32, "only the two box headers are read");+ assert_eq!(format!("{:?}", crate::probe_info(&buf)), format!("{:?}", crate::probe_info(&cr3)));+ // a RAF whose container lies outside the file+ let mut raf = b"FUJIFILMCCD-RAW 0201FF383501".to_vec();+ raf.resize(160, 0);+ raf[100..104].copy_from_slice(&1000u32.to_be_bytes());+ raf[104..108].copy_from_slice(&1000u32.to_be_bytes());+ assert_eq!(run(&raf, 160, 0).0, HeaderPlan::Whole("RAF raw container"));+ }+}diff --git a/crates/raw/src/vendor/cr3.rs b/crates/raw/src/vendor/cr3.rsindex b5442790c..4ea8761aa 100644--- a/crates/raw/src/vendor/cr3.rs+++ b/crates/raw/src/vendor/cr3.rs@@ -31,7 +31,7 @@ const ASPECT_INFO: u16 = 0x009a;/// Pick the first largest raw track. Equal-sized later tracks can be the Dual Pixel delta, which/// must never replace the main sensor image. An invalid full-size descriptor is reported, not/// silently replaced with the reduced raw preview.-fn main_track(tracks: &[Cr3Track]) -> Option<&Cr3Track> {+pub(crate) fn main_track(tracks: &[Cr3Track]) -> Option<&Cr3Track> {let mut best = None;let mut best_area = 0u64;for track in tracks {diff --git a/crates/raw/src/vendor/crx.rs b/crates/raw/src/vendor/crx.rsindex 2d15627fa..d9cc5cd14 100644--- a/crates/raw/src/vendor/crx.rs+++ b/crates/raw/src/vendor/crx.rs@@ -68,6 +68,15 @@ struct Tile {qp: Option<Range<usize>>,}+/// The byte ranges of a CRX sample (relative to its start) that [`validate`] reads: the tile, plane+/// and band headers, and each tile's quantization map. `sample` must hold its first+/// `config.header_size` bytes; the rest is only measured.+pub(crate) fn header_ranges(sample: &[u8], config: &Cr3Compression) -> Result<Vec<Range<usize>>> {+ let tiles = layout(sample, config)?;+ let header = 0..config.header_size as usize;+ Ok(std::iter::once(header).chain(tiles.into_iter().filter_map(|t| t.qp)).collect())+}+/// Decode to full sensor-order CFA samples, including masked borders.pub(crate) fn decode(sample: &[u8], config: &Cr3Compression) -> Result<Vec<u16>> {supports(config)?;diff --git a/crates/raw/src/vendor/raf.rs b/crates/raw/src/vendor/raf.rsindex 61205a3f7..00406d87f 100644--- a/crates/raw/src/vendor/raf.rs+++ b/crates/raw/src/vendor/raf.rs@@ -85,7 +85,7 @@ fn pair(h: &Header, tag: u16) -> Option<(usize, usize)> {Some((u16::from_be_bytes([d[0], d[1]]) as usize, u16::from_be_bytes([d[2], d[3]]) as usize))}-fn raw_ifd(raw: &[u8]) -> Result<Ifd> {+pub(crate) fn raw_ifd(raw: &[u8]) -> Result<Ifd> {let tiff = Tiff::parse(raw).map_err(|_| RawError::Unsupported("RAF without a raw IFD (older FinePix layout)".into()))?;let off = tiff.ifds.first().and_then(|i| i.u64(0xf000)).ok_or_else(|| RawError::Unsupported("RAF without a raw IFD".into()))?;let (ifd, _) = lightcraft_tiff::parse_ifd_at(raw, off, ByteOrder::Little, 0, false, &Default::default())?;diff --git a/crates/raw/src/vendor/rw2.rs b/crates/raw/src/vendor/rw2.rsindex 7fe6d0e42..0a64c9e44 100644--- a/crates/raw/src/vendor/rw2.rs+++ b/crates/raw/src/vendor/rw2.rs@@ -122,7 +122,7 @@ const RAW_FORMAT: u16 = 0x002d;/// The in-camera aspect-ratio crop: top, left, bottom, right in sensor coordinates.const CROP: [u16; 4] = [0x002f, 0x0030, 0x0031, 0x0032];const ISO_32: u16 = 0x0037;-const RAW_OFFSET: u16 = 0x0118;+pub(crate) const RAW_OFFSET: u16 = 0x0118;const HUFFMAN_CODES: u16 = 0x0040;const STRIP_OFFSETS: u16 = 0x0044;const STRIP_LEFT: u16 = 0x0045;diff --git a/crates/raw/tests/sparse_corpus.rs b/crates/raw/tests/sparse_corpus.rsnew file mode 100644index 000000000..3ab735299--- /dev/null+++ b/crates/raw/tests/sparse_corpus.rs@@ -0,0 +1,108 @@+//! [`lightcraft_raw::sparse::header_plan`] over `corpus/raw/**` (git-ignored CC0 samples from+//! raw.pixls.us, fetched with `cargo xtask corpus --download`; `LIGHTCRAFT_CORPUS` overrides the+//! corpus root). Skips cleanly when absent.+//!+//! For every file the plan reaches `Ready` on, probing the partly loaded buffer must give exactly+//! what probing the whole file gives — with the skipped bytes zero and again with them 0xFF, so a+//! reader that looked at skipped data would be caught. CR3 and DNG files must be described from a+//! small part of the file (`cargo test -p lightcraft-raw --release --test sparse_corpus -- --nocapture`+//! prints what each format reads).++use lightcraft_raw::sparse::{HeaderPlan, Ranges, header_plan, preview_plan};+use lightcraft_raw::{embedded_preview, probe, probe_info};+use std::collections::BTreeMap;+use std::path::{Path, PathBuf};++fn corpus_root() -> PathBuf {+ std::env::var_os("LIGHTCRAFT_CORPUS").map(PathBuf::from).unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))+}++/// Run the plan (with the embedded preview when `preview`), loading from `file` what it asks for+/// into a buffer whose other bytes are `fill`.+fn run(file: &[u8], fill: u8, preview: bool) -> (HeaderPlan, Vec<u8>, Ranges) {+ let mut buf = vec![fill; file.len()];+ let mut loaded = Ranges::new();+ let prefix = file.len().min(512 * 1024);+ buf[..prefix].copy_from_slice(&file[..prefix]);+ loaded.insert(0, prefix as u64);+ for _ in 0..32 {+ match if preview { preview_plan(&buf, &loaded) } else { header_plan(&buf, &loaded) } {+ HeaderPlan::Need(r) => {+ for &(a, b) in r.spans() {+ assert!(!loaded.overlaps(a, b), "asked again for {a}..{b}");+ buf[a as usize..b as usize].copy_from_slice(&file[a as usize..b as usize]);+ loaded.insert(a, b);+ }+ }+ done => return (done, buf, loaded),+ }+ }+ panic!("no answer after 32 rounds");+}++/// What an import learns from a file: the raw description and the metadata.+fn describe(bytes: &[u8]) -> String {+ let e = lightcraft_meta::embedded(bytes);+ format!("{:?}\n{:?}\n{:?}", probe_info(bytes), lightcraft_meta::extract(bytes), e.xmp)+}++#[test]+fn partial_reads_describe_raws_as_whole_files_do() {+ let dir = corpus_root().join("raw");+ let mut paths: Vec<PathBuf> = std::fs::read_dir(corpus_root().join("leica"))+ .into_iter()+ .chain(std::fs::read_dir(&dir))+ .flatten()+ .flatten()+ .map(|e| e.path())+ .filter(|p| p.is_file() && !p.extension().is_some_and(|e| e == "part" || e == "txt" || e == "md"))+ .collect();+ if paths.is_empty() {+ eprintln!("skip: {} absent", dir.display());+ return;+ }+ paths.sort();+ // format → (files ready, bytes read, bytes total, files read whole)+ let mut stats: BTreeMap<String, (usize, u64, u64, usize)> = BTreeMap::new();+ for p in paths {+ let name = p.file_name().unwrap().to_string_lossy().to_string();+ let file = std::fs::read(&p).unwrap();+ let format = probe(&file).map_or("?".to_string(), |f| format!("{f:?}"));+ let (plan, zeros, loaded) = run(&file, 0, false);+ let entry = stats.entry(format.clone()).or_default();+ match plan {+ HeaderPlan::Ready => {+ let (plan_ff, ones, loaded_ff) = run(&file, 0xff, false);+ assert_eq!((plan_ff, &loaded_ff), (HeaderPlan::Ready, &loaded), "{name}: the plan read skipped bytes");+ let whole = describe(&file);+ assert_eq!(describe(&zeros), whole, "{name}: differs from the whole file (skipped bytes 0)");+ assert_eq!(describe(&ones), whole, "{name}: differs from the whole file (skipped bytes 0xFF)");+ entry.0 += 1;+ entry.1 += loaded.total();+ entry.2 += file.len() as u64;+ // with the preview (for a raw shown from it), the preview is what the whole file gives too+ let preview = embedded_preview(&file);+ for fill in [0, 0xff] {+ let (plan, buf, _) = run(&file, fill, true);+ assert_eq!(plan, HeaderPlan::Ready, "{name}: preview plan");+ assert_eq!(describe(&buf), whole, "{name}: preview plan, skipped bytes {fill}");+ assert!(embedded_preview(&buf) == preview, "{name}: a different embedded preview (skipped bytes {fill})");+ }+ if loaded.total() * 4 > file.len() as u64 {+ eprintln!("{name}: read {:.0} %", loaded.total() as f64 * 100.0 / file.len() as f64);+ }+ if matches!(format.as_str(), "Cr3" | "Dng") && file.len() > 8 << 20 {+ assert!(loaded.total() * 4 < file.len() as u64, "{name}: read {} of {} bytes", loaded.total(), file.len());+ }+ }+ HeaderPlan::Whole(why) => {+ entry.3 += 1;+ eprintln!("{name}: whole ({why})");+ }+ HeaderPlan::Need(_) => unreachable!(),+ }+ }+ for (format, (ready, read, total, whole)) in stats {+ eprintln!("{format:>10}: {ready} described from {:.1} % of their bytes, {whole} read whole", read as f64 * 100.0 / total.max(1) as f64);+ }+}