LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LR-IMP-FORMATS: decode compressed RAF 13-17 % faster with a buffered bit reader
Christian Bram42e54b5bb083
diff --git a/crates/raw/proptest-regressions/vendor/rafc.txt b/crates/raw/proptest-regressions/vendor/rafc.txtnew file mode 100644index 000000000..7633abe0f--- /dev/null+++ b/crates/raw/proptest-regressions/vendor/rafc.txt@@ -0,0 +1,7 @@+# Seeds for failure cases proptest has generated in the past. It is+# automatically read and these particular cases re-run before any+# novel cases are generated.+#+# It is recommended to check this file in to source control so that+# everyone who runs the test benefits from these saved cases.+cc 8cb6b5999a8279e655a148a18b220d75b02c519f644583f4593166286297a3f4 # shrinks to data = [0, 0, 0, 8], reads = [(false, 13, 0), (true, 0, 15)]diff --git a/crates/raw/src/vendor/rafc.rs b/crates/raw/src/vendor/rafc.rsindex f3b925b51..fd08f9869 100644--- a/crates/raw/src/vendor/rafc.rs+++ b/crates/raw/src/vendor/rafc.rs@@ -114,26 +114,57 @@ impl<'a> Bits<'a> {Self { src, pos: 0, acc: 0, available: 0 }}+ /// Tops the accumulator up to at least 57 bits, or to every remaining byte near the end of the stream.#[inline]- fn get(&mut self, n: u32) -> Result<u32> {- while self.available < n {- let b = self.src.get(self.pos).copied().ok_or_else(|| corrupt("truncated entropy stream"))?;+ fn refill(&mut self) {+ while self.available <= 56 {+ let Some(&b) = self.src.get(self.pos) else { return };self.pos += 1;self.acc = (self.acc << 8) | u64::from(b);self.available += 8;}+ }++ #[inline]+ fn get(&mut self, n: u32) -> Result<u32> {+ if self.available < n {+ self.refill();+ if self.available < n {+ return Err(corrupt("truncated entropy stream"));+ }+ }self.available -= n;Ok(((self.acc >> self.available) & ((1u64 << n) - 1)) as u32)}- fn residual(&mut self, k: u32, bits: u32, range: u32, limit: u32) -> Result<i32> {+ /// Reads the unary prefix: the zeros before the next one bit, which is consumed. Errors exactly where reading+ /// bit by bit would: more than `limit` zeros, or a stream that ends first.+ #[inline]+ fn zeros(&mut self, limit: u32) -> Result<u32> {let mut zeros = 0;- while self.get(1)? == 0 {- zeros += 1;- if zeros > limit {+ loop {+ if self.available == 0 {+ self.refill();+ if self.available == 0 {+ return Err(corrupt("truncated entropy stream"));+ }+ }+ // the unread bits, left-aligned; the zeros shifted in below them are cut off by `min`+ let run = (self.acc << (64 - self.available)).leading_zeros().min(self.available);+ if zeros + run > limit {return Err(corrupt("invalid residual escape"));}+ zeros += run;+ if run < self.available {+ self.available -= run + 1;+ return Ok(zeros);+ }+ self.available = 0;}+ }++ fn residual(&mut self, k: u32, bits: u32, range: u32, limit: u32) -> Result<i32> {+ let zeros = self.zeros(limit)?;let code = if zeros == limit { self.get(bits)? + 1 } else { (zeros << k) | self.get(k)? };if code > range {return Err(corrupt(&format!(@@ -501,5 +532,48 @@ mod tests {src.extend_from_slice(&data);let _ = decode(&src, 780, 6, bits, xtrans, Mode::Full);}++ /// The word-at-a-time reader returns what reading bit by bit returns, and fails where it fails: long zero+ /// runs, the escape limit and the end of the stream included.+ #[test]+ fn bit_reader_matches_reading_bit_by_bit(+ data in prop::collection::vec(prop_oneof![3 => Just(0u8), 1 => any::<u8>()], 0..64),+ reads in prop::collection::vec((any::<bool>(), 0u32..=24, 0u32..=63), 1..200),+ ) {+ let bit = |i: usize| data.get(i / 8).map(|b| (b >> (7 - i % 8)) & 1 == 1);+ let mut at = 0usize;+ let mut reader = Bits::new(&data);+ for (prefix, n, limit) in reads {+ let expected = if prefix {+ let mut zeros = 0;+ loop {+ let Some(b) = bit(at) else { break Err("truncated") };+ at += 1;+ if b {+ break Ok(zeros);+ }+ zeros += 1;+ if zeros > limit {+ break Err("escape");+ }+ }+ } else if at + n as usize > data.len() * 8 {+ Err("truncated")+ } else {+ let value = (0..n as usize).fold(0, |v, i| (v << 1) | u32::from(bit(at + i) == Some(true)));+ at += n as usize;+ Ok(value)+ };+ let actual = if prefix { reader.zeros(limit) } else { reader.get(n) };+ match (expected, actual) {+ (Ok(want), Ok(got)) => prop_assert_eq!(want, got),+ (Err(want), Err(got)) => {+ prop_assert!(got.to_string().contains(want), "{got}");+ break;+ }+ (want, got) => prop_assert!(false, "expected {want:?}, got {got:?}"),+ }+ }+ }}}diff --git a/docs/raf-compression.md b/docs/raf-compression.mdindex 0141d893e..29056502e 100644--- a/docs/raf-compression.md+++ b/docs/raf-compression.md@@ -17,7 +17,7 @@ Fujifilm now uses a guarded colour/tone estimate from each file’s own embedded- Signed prediction errors use adaptive Rice coding: a unary prefix followed by a context-dependent number of bits, with a bounded escape for large errors. Three pair phases have separate even/odd statistics. The 81 signed gradient contexts fold into 41 states.- Lossy quantizers change the error range and reconstruction step. Low-gradient sites retain finer precision (0, 1 or 2), with five folded contexts per precision and statistics that survive main-quantizer changes. The main context statistics reset when its quantizer changes. Their initial sum is `max(2, (range + 32) / 64)`; using 16 as the minimum breaks more heavily quantized files.-Stripes decode in parallel using the existing Rayon dependency. Offsets, sizes, dimensions and entropy reads are checked; truncated data, impossible codes and inconsistent headers return errors. Unknown container compression modes, disagreements between container and stream modes, and compressed X-Trans layouts other than the supported CFA return errors. Header-only probing validates the tables and stripe ranges without decompressing the image.+Stripes decode in parallel using the existing Rayon dependency. The entropy reader keeps up to 64 bits buffered and counts a unary prefix's zeros in one step instead of bit by bit; a property test checks it against reading bit by bit, errors included. Offsets, sizes, dimensions and entropy reads are checked; truncated data, impossible codes and inconsistent headers return errors. Unknown container compression modes, disagreements between container and stream modes, and compressed X-Trans layouts other than the supported CFA return errors. Header-only probing validates the tables and stripe ranges without decompressing the image.## Verification@@ -36,7 +36,7 @@ The 17 CC0 files live in the gitignored corpus. Their published SHA-256 identitiUnconditional procedural tests cover Bayer/X-Trans, 12/14/16 bits, partial final stripes, quantizer changes, residual signs and escapes, malformed headers and truncated/random entropy streams. Regressions also check overflowing uncompressed row strides, unsupported modes and CFA layouts, and container/stream mode mismatches.-Local release decode measurements: about 0.16–0.24 seconds for 24–40 MP X-Trans files, 0.29 seconds for 50 MP Bayer, and 0.5–0.6 seconds for 100 MP Bayer. These measure sensor decoding, not the full develop/export pipeline.+Local release decode measurements: about 0.16–0.24 seconds for 24–40 MP X-Trans files, 0.29 seconds for 50 MP Bayer, and 0.5–0.6 seconds for 100 MP Bayer. These measure sensor decoding, not the full develop/export pipeline. The buffered prefix count took 13–17 % off the decode time on an Apple M2 (8 cores, minimum of 5 runs, old and new build alternated): X-T2 178 → 152 ms, X-S10 lossless 265 → 226 ms, X-E5 442 → 385 ms, X-H2 448 → 378 ms, GFX 50S 522 → 435 ms, GFX 100 762 → 644 ms, with sensor arrays unchanged bit for bit.App verification imported all four supplied compressed files, confirmed `previewOnly: null`, applied +2 EV exposure edits and inspected headless UI screenshots. CLI JPEG exports at a 1600-pixel long edge succeeded for all four files (0.58–0.76 seconds each, including process startup).