Skip to content

Craft Anything: understand how any software works, and craft your own.

Rust

Bound CPIO checksum work across firmware probes (#683)

BFlat636b3ff3d78c
diff --git a/crates/formats/src/firmware.rs b/crates/formats/src/firmware.rs
index 9fb5652f..55d1b9c8 100644
--- a/crates/formats/src/firmware.rs
+++ b/crates/formats/src/firmware.rs
@@ -7,6 +7,7 @@
use crate::bytes::Reader;
use crate::common::entropy;
+use crate::tarcpio::CpioChecksum;
use miniz_oxide::inflate::stream::{InflateState, inflate as mz_inflate};
use miniz_oxide::{DataFormat, MZError, MZFlush, MZStatus};
use serde::Serialize;
@@ -311,6 +312,10 @@ struct Scanner<'a> {
probes: BTreeMap<&'static str, usize>,
opts: &'a ScanOptions,
inflate_left: u64,
+ checksum_left: usize,
+ checksum_exhausted: bool,
+ stop: &'a dyn Fn() -> bool,
+ cancelled: bool,
exhausted: Vec<&'static str>,
}
@@ -791,22 +796,53 @@ impl<'a> Scanner<'a> {
if !matches!(self.r.u8(i + 5)?, b'1' | b'2' | b'7') || !self.probe("cpio") {
return None;
}
- let l = crate::tarcpio::cpio(self.data.get(i..)?, 1_000_000)?;
+ let data = self.data;
+ let before = self.checksum_left;
+ let mut skipped = false;
+ let l = crate::tarcpio::cpio_with_checksum(data.get(i..)?, 1_000_000, &mut |body| {
+ let result = self.cpio_checksum(body);
+ skipped |= matches!(result, CpioChecksum::Skipped);
+ result
+ })?;
if l.members.is_empty() && !l.complete {
return None;
}
+ let mut details = json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings});
+ if l.format == "cpio-crc" {
+ details["checksum_validation_skipped"] = json!(skipped);
+ details["checksum_bytes_visited"] = json!(before.saturating_sub(self.checksum_left));
+ }
Some(reg(
i,
Some(l.length),
l.format,
Family::Archive,
format!("cpio archive ({}), {} member(s)", l.format.trim_start_matches("cpio-"), l.members.len()),
- if l.complete { "high" } else { "medium" },
+ if l.complete && !skipped { "high" } else { "medium" },
Unpack::Members,
- json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings}),
+ details,
))
}
+ /// Sum at most one input's worth of CRC member bytes across the entire scan.
+ fn cpio_checksum(&mut self, body: &[u8]) -> CpioChecksum {
+ let mut sum = 0u32;
+ for chunk in body.chunks(64 << 10) {
+ if (self.stop)() {
+ self.cancelled = true;
+ return CpioChecksum::Cancelled;
+ }
+ let count = chunk.len().min(self.checksum_left);
+ sum = chunk.iter().take(count).fold(sum, |sum, &byte| sum.wrapping_add(u32::from(byte)));
+ self.checksum_left = self.checksum_left.saturating_sub(count);
+ if count < chunk.len() {
+ self.checksum_exhausted = true;
+ return CpioChecksum::Skipped;
+ }
+ }
+ CpioChecksum::Value(sum)
+ }
+
fn squashfs(&mut self, i: usize, be: bool) -> Option<Region> {
let r = self.r;
let u16f = |o: usize| if be { r.u16_be(i + o) } else { r.u16_le(i + o) };
@@ -1499,11 +1535,23 @@ impl<'a> Scanner<'a> {
}
}
-/// Scan `data` for recognised regions. `stop` is polled every MiB; when it returns true the scan
-/// ends early with `cancelled` set. A region with a confirmed length hides its interior from the
-/// scan (nested content is found by scanning the region again after extraction).
+/// Scan `data` for recognised regions. `stop` is polled every MiB and every 64 KiB during CPIO
+/// checksum sums; when it returns true the scan ends early with `cancelled` set. CRC member sums
+/// share a byte allowance equal to the input length; skipped verification is reported. A region
+/// with a confirmed length hides its interior (nested content is found after extraction).
pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanResult {
- let mut s = Scanner { data, r: Reader::new(data), probes: BTreeMap::new(), opts, inflate_left: opts.inflate_budget, exhausted: Vec::new() };
+ let mut s = Scanner {
+ data,
+ r: Reader::new(data),
+ probes: BTreeMap::new(),
+ opts,
+ inflate_left: opts.inflate_budget,
+ checksum_left: data.len(),
+ checksum_exhausted: false,
+ stop,
+ cancelled: false,
+ exhausted: Vec::new(),
+ };
let mut out = ScanResult::default();
let mut skip_until = 0usize;
let mut i = 0usize;
@@ -1513,6 +1561,10 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanRes
break;
}
let found = s.try_backref(i).filter(|r| r.offset as usize >= skip_until).or_else(|| s.try_at(i));
+ if s.cancelled {
+ out.cancelled = true;
+ break;
+ }
if let Some(r) = found {
if out.regions.len() >= opts.max_regions {
out.truncated = true;
@@ -1538,6 +1590,9 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanRes
if !out.probes_exhausted.is_empty() {
out.warnings.push(format!("probe budget spent for: {} (later candidates were not checked)", out.probes_exhausted.join(", ")));
}
+ if s.checksum_exhausted {
+ out.warnings.push("cpio CRC checksum byte budget spent (some member data was not verified)".into());
+ }
out
}
diff --git a/crates/formats/src/tarcpio.rs b/crates/formats/src/tarcpio.rs
index 6039691d..5e7fe4e9 100644
--- a/crates/formats/src/tarcpio.rs
+++ b/crates/formats/src/tarcpio.rs
@@ -275,6 +275,19 @@ fn mode_kind(mode: u32) -> MemberKind {
/// Walk a cpio archive (newc `070701`, crc `070702`, odc `070707`). `None` when the first header
/// is not one of these.
pub fn cpio(data: &[u8], max_members: usize) -> Option<Listing> {
+ cpio_with_checksum(data, max_members, &mut |body| CpioChecksum::Value(body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)))))
+}
+
+/// A checksum sum, or the reason a caller stopped verification before completing it.
+#[derive(Clone, Copy)]
+pub(crate) enum CpioChecksum {
+ Value(u32),
+ Skipped,
+ Cancelled,
+}
+
+/// Let a scanner share its checksum work allowance and cancellation across overlapping probes.
+pub(crate) fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn FnMut(&[u8]) -> CpioChecksum) -> Option<Listing> {
let r = Reader::new(data);
let format = match r.slice(0, 6)? {
b"070701" => "cpio-newc",
@@ -341,12 +354,17 @@ pub fn cpio(data: &[u8], max_members: usize) -> Option<Listing> {
break;
}
if format == "cpio-crc" {
- match h.get(102..110).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {
+ match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {
Some(expected) => {
if let Some(body) = r.slice(data_off, size) {
- let actual = body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)));
- if actual != expected {
- out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})"));
+ match checksum(body) {
+ CpioChecksum::Value(actual) => {
+ if actual != expected {
+ out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})"));
+ }
+ }
+ CpioChecksum::Skipped => out.warnings.push(format!("{name}: data checksum not verified (scan byte budget exhausted)")),
+ CpioChecksum::Cancelled => return None,
}
}
}
diff --git a/crates/formats/tests/cpio_crc_scan_limits.rs b/crates/formats/tests/cpio_crc_scan_limits.rs
new file mode 100644
index 00000000..d161de30
--- /dev/null
+++ b/crates/formats/tests/cpio_crc_scan_limits.rs
@@ -0,0 +1,144 @@
+use std::cell::Cell;
+
+use craft_anything_formats::firmware::{ScanOptions, scan};
+use craft_anything_formats::tarcpio::cpio;
+use craft_anything_testkit::firmware as fx;
+
+const DATA_OFFSET: usize = 112; // 110-byte newc header plus the two-byte name "x\0".
+
+fn crc_archive(payload: &[u8]) -> Vec<u8> {
+ let mut archive = fx::cpio_newc(&[("x", payload, 0o100644)]);
+ assert_eq!(&archive[DATA_OFFSET..DATA_OFFSET + payload.len()], payload);
+ let trailer = (DATA_OFFSET + payload.len() + 3) & !3;
+ let sum = payload.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)));
+ archive[..6].copy_from_slice(b"070702");
+ archive[102..110].copy_from_slice(format!("{sum:08x}").as_bytes());
+ archive[trailer..trailer + 6].copy_from_slice(b"070702");
+ archive
+}
+
+fn odc_archive() -> Vec<u8> {
+ let mut archive = Vec::new();
+ for (name, payload) in [("plain", b"data".as_slice()), ("TRAILER!!!", b"".as_slice())] {
+ let header = format!(
+ "070707{dev:06o}{ino:06o}{mode:06o}{uid:06o}{gid:06o}{nlink:06o}{rdev:06o}{mtime:011o}{namesize:06o}{filesize:011o}",
+ dev = 0,
+ ino = 1,
+ mode = 0o100644,
+ uid = 0,
+ gid = 0,
+ nlink = 1,
+ rdev = 0,
+ mtime = 0,
+ namesize = name.len() + 1,
+ filesize = payload.len(),
+ );
+ assert_eq!(header.len(), 76);
+ archive.extend_from_slice(header.as_bytes());
+ archive.extend_from_slice(name.as_bytes());
+ archive.push(0);
+ archive.extend_from_slice(payload);
+ }
+ archive
+}
+
+#[test]
+fn crc_member_checksum_requires_eight_ascii_hex_digits() {
+ for payload in [b"".as_slice(), &[0, 0]] {
+ for field in [b"+0000000", b"0000000+", b"0000000g", b" 0000000", b"0000000\0"] {
+ let mut archive = crc_archive(payload);
+ archive[102..110].copy_from_slice(field);
+ let listing = cpio(&archive, 10).unwrap();
+ assert!(listing.complete);
+ assert_eq!(listing.members.len(), 1);
+ assert!(listing.warnings.iter().any(|warning| warning.contains("unreadable data checksum")), "{field:?}: {:?}", listing.warnings);
+ assert!(!listing.warnings.iter().any(|warning| warning.contains("checksum mismatch")), "invalid syntax is not a computed mismatch");
+ }
+ }
+}
+
+#[test]
+fn crc_checksum_hex_case_and_zero_length_members_remain_valid() {
+ for field in [b"000001ba", b"000001BA"] {
+ let mut archive = crc_archive(&[255, 171, 16]);
+ archive[102..110].copy_from_slice(field);
+ let listing = cpio(&archive, 10).unwrap();
+ assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");
+ }
+ let empty = crc_archive(&[]);
+ let listing = cpio(&empty, 10).unwrap();
+ assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");
+}
+
+#[test]
+fn non_crc_variants_do_not_interpret_a_crc_field() {
+ let mut newc = fx::cpio_newc(&[("plain", b"data", 0o100644)]);
+ newc[102..110].copy_from_slice(b"+0000000");
+ for archive in [newc, odc_archive()] {
+ let listing = cpio(&archive, 10).unwrap();
+ assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");
+ let result = scan(&archive, &ScanOptions::default(), &|| false);
+ let region = result.regions.iter().find(|region| region.offset == 0).unwrap();
+ assert_eq!(region.confidence, "high");
+ assert!(result.warnings.is_empty());
+ }
+}
+
+#[test]
+fn overlapping_crc_probes_share_the_scan_byte_budget() {
+ // Both checksums are correct. The outer member has no trailer, so its medium-confidence
+ // region does not hide the complete inner archive from the scanner.
+ let inner = crc_archive(&vec![0x5a; 8192]);
+ let mut image = crc_archive(&inner);
+ image.truncate((DATA_OFFSET + inner.len() + 3) & !3);
+ let plain_offset = image.len();
+ image.extend_from_slice(&fx::cpio_newc(&[("plain", b"data", 0o100644)]));
+ let empty_offset = image.len();
+ image.extend_from_slice(&crc_archive(&[]));
+
+ let complete = cpio(&inner, 10).unwrap();
+ assert!(complete.complete && complete.warnings.is_empty(), "the direct parser still checks the entire inner archive");
+ let result = scan(&image, &ScanOptions::default(), &|| false);
+ assert!(!result.cancelled && !result.truncated);
+ assert!(result.warnings.iter().any(|warning| warning.contains("checksum byte budget")), "{:?}", result.warnings);
+ let nested = result.regions.iter().find(|region| region.offset == DATA_OFFSET as u64 && region.kind == "cpio-crc").unwrap();
+ assert_eq!(nested.details["complete"], true, "the trailer is still present");
+ assert_eq!(nested.confidence, "medium", "unverified member data must not gain high confidence");
+ assert_eq!(nested.details["checksum_validation_skipped"], true);
+ let warnings = nested.details["warnings"].as_array().unwrap();
+ assert!(warnings.iter().any(|warning| warning.as_str().unwrap().contains("not verified")));
+ assert!(!warnings.iter().any(|warning| warning.as_str().unwrap().contains("checksum mismatch")));
+ let visited: u64 =
+ result.regions.iter().filter(|region| region.kind == "cpio-crc").map(|region| region.details["checksum_bytes_visited"].as_u64().unwrap()).sum();
+ assert_eq!(visited, image.len() as u64, "the partial probe consumes only the remaining shared allowance");
+
+ let plain = result.regions.iter().find(|region| region.offset == plain_offset as u64).unwrap();
+ assert_eq!(plain.kind, "cpio-newc");
+ assert_eq!(plain.confidence, "high");
+ let empty = result.regions.iter().find(|region| region.offset == empty_offset as u64).unwrap();
+ assert_eq!(empty.kind, "cpio-crc");
+ assert_eq!(empty.confidence, "high");
+ assert_eq!(empty.details["checksum_validation_skipped"], false, "a zero-byte member still verifies with no byte allowance left");
+ assert_eq!(empty.details["checksum_bytes_visited"], 0);
+ assert!(empty.details["warnings"].as_array().unwrap().is_empty());
+}
+
+#[test]
+fn crc_body_cancellation_does_not_publish_a_completed_region() {
+ let archive = crc_archive(&vec![7; 3 * 64 * 1024]);
+ let calls = Cell::new(0usize);
+ let cancelled = scan(&archive, &ScanOptions::default(), &|| {
+ calls.set(calls.get() + 1);
+ calls.get() >= 3
+ });
+ assert!(cancelled.cancelled, "the stop callback must be polled inside the checksum body");
+ assert_eq!(calls.get(), 3);
+ assert!(cancelled.regions.is_empty(), "a cancelled archive probe must not be reported as complete");
+
+ let finished = scan(&archive, &ScanOptions::default(), &|| false);
+ assert!(!finished.cancelled);
+ let region = finished.regions.iter().find(|region| region.offset == 0).unwrap();
+ assert_eq!(region.confidence, "high");
+ assert_eq!(region.details["complete"], true);
+ assert!(region.details["warnings"].as_array().unwrap().is_empty());
+}
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index df5e7095..d00443bb 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -645,6 +645,8 @@ Owner: open · Status: in progress
PAX sizes across other metadata records; CRC cpio checks member data; source maps decode
percent-escaped data URLs and report indexed-map columns in the generated file
(#606–#615, by Afloat16)
+- [x] Bound and cancel CRC cpio verification across firmware probes, and reject non-hex checksum
+ fields — claimed: Afloat16, branch bugfix/r3.12-bound-cpio-crc-scans-20261103
### R3.13 Web inspection correctness
Owner: open · Status: in progress
diff --git a/docs/pitfalls.md b/docs/pitfalls.md
index 6f7560fc..5ae65657 100644
--- a/docs/pitfalls.md
+++ b/docs/pitfalls.md
@@ -671,3 +671,10 @@ opencode-session under settings::config_dir() when the first request was made, s
tests left a file on the developer's machine and two runs could collide. Anything we persist goes
through the hub's stores (AiHub::with_stores, Session::isolated), which tests point at a temp
folder; production code never writes under the real config folder by hand.
+
+**A per-probe walk repeated expensive work on the same bytes.** CRC cpio verification was bounded
+inside one archive, but a firmware scan could rediscover overlapping, trailerless archives and
+sum the same payload repeatedly. Count checksum byte visits across the whole scan, poll its
+cancellation callback inside long sums, and distinguish skipped verification from a mismatch.
+Test overlapping candidates with correct checksums and a complete nested archive so a structural
+trailer cannot turn unchecked data into a high-confidence result.