Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

LR-IMP-FORMATS: decode compressed RAF 13-17 % faster with a buffered bit reader

Christian Bram42e54b5bb083
diff --git a/crates/raw/proptest-regressions/vendor/rafc.txt b/crates/raw/proptest-regressions/vendor/rafc.txt
new file mode 100644
index 000000000..7633abe0f
--- /dev/null
+++ b/crates/raw/proptest-regressions/vendor/rafc.txt
@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 8cb6b5999a8279e655a148a18b220d75b02c519f644583f4593166286297a3f4 # shrinks to data = [0, 0, 0, 8], reads = [(false, 13, 0), (true, 0, 15)]
diff --git a/crates/raw/src/vendor/rafc.rs b/crates/raw/src/vendor/rafc.rs
index f3b925b51..fd08f9869 100644
--- a/crates/raw/src/vendor/rafc.rs
+++ b/crates/raw/src/vendor/rafc.rs
@@ -114,26 +114,57 @@ impl<'a> Bits<'a> {
Self { src, pos: 0, acc: 0, available: 0 }
}
+ /// Tops the accumulator up to at least 57 bits, or to every remaining byte near the end of the stream.
#[inline]
- fn get(&mut self, n: u32) -> Result<u32> {
- while self.available < n {
- let b = self.src.get(self.pos).copied().ok_or_else(|| corrupt("truncated entropy stream"))?;
+ fn refill(&mut self) {
+ while self.available <= 56 {
+ let Some(&b) = self.src.get(self.pos) else { return };
self.pos += 1;
self.acc = (self.acc << 8) | u64::from(b);
self.available += 8;
}
+ }
+
+ #[inline]
+ fn get(&mut self, n: u32) -> Result<u32> {
+ if self.available < n {
+ self.refill();
+ if self.available < n {
+ return Err(corrupt("truncated entropy stream"));
+ }
+ }
self.available -= n;
Ok(((self.acc >> self.available) & ((1u64 << n) - 1)) as u32)
}
- fn residual(&mut self, k: u32, bits: u32, range: u32, limit: u32) -> Result<i32> {
+ /// Reads the unary prefix: the zeros before the next one bit, which is consumed. Errors exactly where reading
+ /// bit by bit would: more than `limit` zeros, or a stream that ends first.
+ #[inline]
+ fn zeros(&mut self, limit: u32) -> Result<u32> {
let mut zeros = 0;
- while self.get(1)? == 0 {
- zeros += 1;
- if zeros > limit {
+ loop {
+ if self.available == 0 {
+ self.refill();
+ if self.available == 0 {
+ return Err(corrupt("truncated entropy stream"));
+ }
+ }
+ // the unread bits, left-aligned; the zeros shifted in below them are cut off by `min`
+ let run = (self.acc << (64 - self.available)).leading_zeros().min(self.available);
+ if zeros + run > limit {
return Err(corrupt("invalid residual escape"));
}
+ zeros += run;
+ if run < self.available {
+ self.available -= run + 1;
+ return Ok(zeros);
+ }
+ self.available = 0;
}
+ }
+
+ fn residual(&mut self, k: u32, bits: u32, range: u32, limit: u32) -> Result<i32> {
+ let zeros = self.zeros(limit)?;
let code = if zeros == limit { self.get(bits)? + 1 } else { (zeros << k) | self.get(k)? };
if code > range {
return Err(corrupt(&format!(
@@ -501,5 +532,48 @@ mod tests {
src.extend_from_slice(&data);
let _ = decode(&src, 780, 6, bits, xtrans, Mode::Full);
}
+
+ /// The word-at-a-time reader returns what reading bit by bit returns, and fails where it fails: long zero
+ /// runs, the escape limit and the end of the stream included.
+ #[test]
+ fn bit_reader_matches_reading_bit_by_bit(
+ data in prop::collection::vec(prop_oneof![3 => Just(0u8), 1 => any::<u8>()], 0..64),
+ reads in prop::collection::vec((any::<bool>(), 0u32..=24, 0u32..=63), 1..200),
+ ) {
+ let bit = |i: usize| data.get(i / 8).map(|b| (b >> (7 - i % 8)) & 1 == 1);
+ let mut at = 0usize;
+ let mut reader = Bits::new(&data);
+ for (prefix, n, limit) in reads {
+ let expected = if prefix {
+ let mut zeros = 0;
+ loop {
+ let Some(b) = bit(at) else { break Err("truncated") };
+ at += 1;
+ if b {
+ break Ok(zeros);
+ }
+ zeros += 1;
+ if zeros > limit {
+ break Err("escape");
+ }
+ }
+ } else if at + n as usize > data.len() * 8 {
+ Err("truncated")
+ } else {
+ let value = (0..n as usize).fold(0, |v, i| (v << 1) | u32::from(bit(at + i) == Some(true)));
+ at += n as usize;
+ Ok(value)
+ };
+ let actual = if prefix { reader.zeros(limit) } else { reader.get(n) };
+ match (expected, actual) {
+ (Ok(want), Ok(got)) => prop_assert_eq!(want, got),
+ (Err(want), Err(got)) => {
+ prop_assert!(got.to_string().contains(want), "{got}");
+ break;
+ }
+ (want, got) => prop_assert!(false, "expected {want:?}, got {got:?}"),
+ }
+ }
+ }
}
}
diff --git a/docs/raf-compression.md b/docs/raf-compression.md
index 0141d893e..29056502e 100644
--- a/docs/raf-compression.md
+++ b/docs/raf-compression.md
@@ -17,7 +17,7 @@ Fujifilm now uses a guarded colour/tone estimate from each file’s own embedded
- Signed prediction errors use adaptive Rice coding: a unary prefix followed by a context-dependent number of bits, with a bounded escape for large errors. Three pair phases have separate even/odd statistics. The 81 signed gradient contexts fold into 41 states.
- Lossy quantizers change the error range and reconstruction step. Low-gradient sites retain finer precision (0, 1 or 2), with five folded contexts per precision and statistics that survive main-quantizer changes. The main context statistics reset when its quantizer changes. Their initial sum is `max(2, (range + 32) / 64)`; using 16 as the minimum breaks more heavily quantized files.
-Stripes decode in parallel using the existing Rayon dependency. Offsets, sizes, dimensions and entropy reads are checked; truncated data, impossible codes and inconsistent headers return errors. Unknown container compression modes, disagreements between container and stream modes, and compressed X-Trans layouts other than the supported CFA return errors. Header-only probing validates the tables and stripe ranges without decompressing the image.
+Stripes decode in parallel using the existing Rayon dependency. The entropy reader keeps up to 64 bits buffered and counts a unary prefix's zeros in one step instead of bit by bit; a property test checks it against reading bit by bit, errors included. Offsets, sizes, dimensions and entropy reads are checked; truncated data, impossible codes and inconsistent headers return errors. Unknown container compression modes, disagreements between container and stream modes, and compressed X-Trans layouts other than the supported CFA return errors. Header-only probing validates the tables and stripe ranges without decompressing the image.
## Verification
@@ -36,7 +36,7 @@ The 17 CC0 files live in the gitignored corpus. Their published SHA-256 identiti
Unconditional procedural tests cover Bayer/X-Trans, 12/14/16 bits, partial final stripes, quantizer changes, residual signs and escapes, malformed headers and truncated/random entropy streams. Regressions also check overflowing uncompressed row strides, unsupported modes and CFA layouts, and container/stream mode mismatches.
-Local release decode measurements: about 0.16–0.24 seconds for 24–40 MP X-Trans files, 0.29 seconds for 50 MP Bayer, and 0.5–0.6 seconds for 100 MP Bayer. These measure sensor decoding, not the full develop/export pipeline.
+Local release decode measurements: about 0.16–0.24 seconds for 24–40 MP X-Trans files, 0.29 seconds for 50 MP Bayer, and 0.5–0.6 seconds for 100 MP Bayer. These measure sensor decoding, not the full develop/export pipeline. The buffered prefix count took 13–17 % off the decode time on an Apple M2 (8 cores, minimum of 5 runs, old and new build alternated): X-T2 178 → 152 ms, X-S10 lossless 265 → 226 ms, X-E5 442 → 385 ms, X-H2 448 → 378 ms, GFX 50S 522 → 435 ms, GFX 100 762 → 644 ms, with sensor arrays unchanged bit for bit.
App verification imported all four supplied compressed files, confirmed `previewOnly: null`, applied +2 EV exposure edits and inspected headless UI screenshots. CLI JPEG exports at a 1600-pixel long edge succeeded for all four files (0.58–0.76 seconds each, including process startup).