Craft Anything: understand how any software works, and craft your own.
Rust
Bound CPIO checksum work across firmware probes (#683)
BFlat636b3ff3d78c
diff --git a/crates/formats/src/firmware.rs b/crates/formats/src/firmware.rsindex 9fb5652f..55d1b9c8 100644--- a/crates/formats/src/firmware.rs+++ b/crates/formats/src/firmware.rs@@ -7,6 +7,7 @@use crate::bytes::Reader;use crate::common::entropy;+use crate::tarcpio::CpioChecksum;use miniz_oxide::inflate::stream::{InflateState, inflate as mz_inflate};use miniz_oxide::{DataFormat, MZError, MZFlush, MZStatus};use serde::Serialize;@@ -311,6 +312,10 @@ struct Scanner<'a> {probes: BTreeMap<&'static str, usize>,opts: &'a ScanOptions,inflate_left: u64,+ checksum_left: usize,+ checksum_exhausted: bool,+ stop: &'a dyn Fn() -> bool,+ cancelled: bool,exhausted: Vec<&'static str>,}@@ -791,22 +796,53 @@ impl<'a> Scanner<'a> {if !matches!(self.r.u8(i + 5)?, b'1' | b'2' | b'7') || !self.probe("cpio") {return None;}- let l = crate::tarcpio::cpio(self.data.get(i..)?, 1_000_000)?;+ let data = self.data;+ let before = self.checksum_left;+ let mut skipped = false;+ let l = crate::tarcpio::cpio_with_checksum(data.get(i..)?, 1_000_000, &mut |body| {+ let result = self.cpio_checksum(body);+ skipped |= matches!(result, CpioChecksum::Skipped);+ result+ })?;if l.members.is_empty() && !l.complete {return None;}+ let mut details = json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings});+ if l.format == "cpio-crc" {+ details["checksum_validation_skipped"] = json!(skipped);+ details["checksum_bytes_visited"] = json!(before.saturating_sub(self.checksum_left));+ }Some(reg(i,Some(l.length),l.format,Family::Archive,format!("cpio archive ({}), {} member(s)", l.format.trim_start_matches("cpio-"), l.members.len()),- if l.complete { "high" } else { "medium" },+ if l.complete && !skipped { "high" } else { "medium" },Unpack::Members,- json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings}),+ details,))}+ /// Sum at most one input's worth of CRC member bytes across the entire scan.+ fn cpio_checksum(&mut self, body: &[u8]) -> CpioChecksum {+ let mut sum = 0u32;+ for chunk in body.chunks(64 << 10) {+ if (self.stop)() {+ self.cancelled = true;+ return CpioChecksum::Cancelled;+ }+ let count = chunk.len().min(self.checksum_left);+ sum = chunk.iter().take(count).fold(sum, |sum, &byte| sum.wrapping_add(u32::from(byte)));+ self.checksum_left = self.checksum_left.saturating_sub(count);+ if count < chunk.len() {+ self.checksum_exhausted = true;+ return CpioChecksum::Skipped;+ }+ }+ CpioChecksum::Value(sum)+ }+fn squashfs(&mut self, i: usize, be: bool) -> Option<Region> {let r = self.r;let u16f = |o: usize| if be { r.u16_be(i + o) } else { r.u16_le(i + o) };@@ -1499,11 +1535,23 @@ impl<'a> Scanner<'a> {}}-/// Scan `data` for recognised regions. `stop` is polled every MiB; when it returns true the scan-/// ends early with `cancelled` set. A region with a confirmed length hides its interior from the-/// scan (nested content is found by scanning the region again after extraction).+/// Scan `data` for recognised regions. `stop` is polled every MiB and every 64 KiB during CPIO+/// checksum sums; when it returns true the scan ends early with `cancelled` set. CRC member sums+/// share a byte allowance equal to the input length; skipped verification is reported. A region+/// with a confirmed length hides its interior (nested content is found after extraction).pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanResult {- let mut s = Scanner { data, r: Reader::new(data), probes: BTreeMap::new(), opts, inflate_left: opts.inflate_budget, exhausted: Vec::new() };+ let mut s = Scanner {+ data,+ r: Reader::new(data),+ probes: BTreeMap::new(),+ opts,+ inflate_left: opts.inflate_budget,+ checksum_left: data.len(),+ checksum_exhausted: false,+ stop,+ cancelled: false,+ exhausted: Vec::new(),+ };let mut out = ScanResult::default();let mut skip_until = 0usize;let mut i = 0usize;@@ -1513,6 +1561,10 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanResbreak;}let found = s.try_backref(i).filter(|r| r.offset as usize >= skip_until).or_else(|| s.try_at(i));+ if s.cancelled {+ out.cancelled = true;+ break;+ }if let Some(r) = found {if out.regions.len() >= opts.max_regions {out.truncated = true;@@ -1538,6 +1590,9 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanResif !out.probes_exhausted.is_empty() {out.warnings.push(format!("probe budget spent for: {} (later candidates were not checked)", out.probes_exhausted.join(", ")));}+ if s.checksum_exhausted {+ out.warnings.push("cpio CRC checksum byte budget spent (some member data was not verified)".into());+ }out}diff --git a/crates/formats/src/tarcpio.rs b/crates/formats/src/tarcpio.rsindex 6039691d..5e7fe4e9 100644--- a/crates/formats/src/tarcpio.rs+++ b/crates/formats/src/tarcpio.rs@@ -275,6 +275,19 @@ fn mode_kind(mode: u32) -> MemberKind {/// Walk a cpio archive (newc `070701`, crc `070702`, odc `070707`). `None` when the first header/// is not one of these.pub fn cpio(data: &[u8], max_members: usize) -> Option<Listing> {+ cpio_with_checksum(data, max_members, &mut |body| CpioChecksum::Value(body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)))))+}++/// A checksum sum, or the reason a caller stopped verification before completing it.+#[derive(Clone, Copy)]+pub(crate) enum CpioChecksum {+ Value(u32),+ Skipped,+ Cancelled,+}++/// Let a scanner share its checksum work allowance and cancellation across overlapping probes.+pub(crate) fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn FnMut(&[u8]) -> CpioChecksum) -> Option<Listing> {let r = Reader::new(data);let format = match r.slice(0, 6)? {b"070701" => "cpio-newc",@@ -341,12 +354,17 @@ pub fn cpio(data: &[u8], max_members: usize) -> Option<Listing> {break;}if format == "cpio-crc" {- match h.get(102..110).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {+ match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {Some(expected) => {if let Some(body) = r.slice(data_off, size) {- let actual = body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)));- if actual != expected {- out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})"));+ match checksum(body) {+ CpioChecksum::Value(actual) => {+ if actual != expected {+ out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})"));+ }+ }+ CpioChecksum::Skipped => out.warnings.push(format!("{name}: data checksum not verified (scan byte budget exhausted)")),+ CpioChecksum::Cancelled => return None,}}}diff --git a/crates/formats/tests/cpio_crc_scan_limits.rs b/crates/formats/tests/cpio_crc_scan_limits.rsnew file mode 100644index 00000000..d161de30--- /dev/null+++ b/crates/formats/tests/cpio_crc_scan_limits.rs@@ -0,0 +1,144 @@+use std::cell::Cell;++use craft_anything_formats::firmware::{ScanOptions, scan};+use craft_anything_formats::tarcpio::cpio;+use craft_anything_testkit::firmware as fx;++const DATA_OFFSET: usize = 112; // 110-byte newc header plus the two-byte name "x\0".++fn crc_archive(payload: &[u8]) -> Vec<u8> {+ let mut archive = fx::cpio_newc(&[("x", payload, 0o100644)]);+ assert_eq!(&archive[DATA_OFFSET..DATA_OFFSET + payload.len()], payload);+ let trailer = (DATA_OFFSET + payload.len() + 3) & !3;+ let sum = payload.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)));+ archive[..6].copy_from_slice(b"070702");+ archive[102..110].copy_from_slice(format!("{sum:08x}").as_bytes());+ archive[trailer..trailer + 6].copy_from_slice(b"070702");+ archive+}++fn odc_archive() -> Vec<u8> {+ let mut archive = Vec::new();+ for (name, payload) in [("plain", b"data".as_slice()), ("TRAILER!!!", b"".as_slice())] {+ let header = format!(+ "070707{dev:06o}{ino:06o}{mode:06o}{uid:06o}{gid:06o}{nlink:06o}{rdev:06o}{mtime:011o}{namesize:06o}{filesize:011o}",+ dev = 0,+ ino = 1,+ mode = 0o100644,+ uid = 0,+ gid = 0,+ nlink = 1,+ rdev = 0,+ mtime = 0,+ namesize = name.len() + 1,+ filesize = payload.len(),+ );+ assert_eq!(header.len(), 76);+ archive.extend_from_slice(header.as_bytes());+ archive.extend_from_slice(name.as_bytes());+ archive.push(0);+ archive.extend_from_slice(payload);+ }+ archive+}++#[test]+fn crc_member_checksum_requires_eight_ascii_hex_digits() {+ for payload in [b"".as_slice(), &[0, 0]] {+ for field in [b"+0000000", b"0000000+", b"0000000g", b" 0000000", b"0000000\0"] {+ let mut archive = crc_archive(payload);+ archive[102..110].copy_from_slice(field);+ let listing = cpio(&archive, 10).unwrap();+ assert!(listing.complete);+ assert_eq!(listing.members.len(), 1);+ assert!(listing.warnings.iter().any(|warning| warning.contains("unreadable data checksum")), "{field:?}: {:?}", listing.warnings);+ assert!(!listing.warnings.iter().any(|warning| warning.contains("checksum mismatch")), "invalid syntax is not a computed mismatch");+ }+ }+}++#[test]+fn crc_checksum_hex_case_and_zero_length_members_remain_valid() {+ for field in [b"000001ba", b"000001BA"] {+ let mut archive = crc_archive(&[255, 171, 16]);+ archive[102..110].copy_from_slice(field);+ let listing = cpio(&archive, 10).unwrap();+ assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");+ }+ let empty = crc_archive(&[]);+ let listing = cpio(&empty, 10).unwrap();+ assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");+}++#[test]+fn non_crc_variants_do_not_interpret_a_crc_field() {+ let mut newc = fx::cpio_newc(&[("plain", b"data", 0o100644)]);+ newc[102..110].copy_from_slice(b"+0000000");+ for archive in [newc, odc_archive()] {+ let listing = cpio(&archive, 10).unwrap();+ assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");+ let result = scan(&archive, &ScanOptions::default(), &|| false);+ let region = result.regions.iter().find(|region| region.offset == 0).unwrap();+ assert_eq!(region.confidence, "high");+ assert!(result.warnings.is_empty());+ }+}++#[test]+fn overlapping_crc_probes_share_the_scan_byte_budget() {+ // Both checksums are correct. The outer member has no trailer, so its medium-confidence+ // region does not hide the complete inner archive from the scanner.+ let inner = crc_archive(&vec![0x5a; 8192]);+ let mut image = crc_archive(&inner);+ image.truncate((DATA_OFFSET + inner.len() + 3) & !3);+ let plain_offset = image.len();+ image.extend_from_slice(&fx::cpio_newc(&[("plain", b"data", 0o100644)]));+ let empty_offset = image.len();+ image.extend_from_slice(&crc_archive(&[]));++ let complete = cpio(&inner, 10).unwrap();+ assert!(complete.complete && complete.warnings.is_empty(), "the direct parser still checks the entire inner archive");+ let result = scan(&image, &ScanOptions::default(), &|| false);+ assert!(!result.cancelled && !result.truncated);+ assert!(result.warnings.iter().any(|warning| warning.contains("checksum byte budget")), "{:?}", result.warnings);+ let nested = result.regions.iter().find(|region| region.offset == DATA_OFFSET as u64 && region.kind == "cpio-crc").unwrap();+ assert_eq!(nested.details["complete"], true, "the trailer is still present");+ assert_eq!(nested.confidence, "medium", "unverified member data must not gain high confidence");+ assert_eq!(nested.details["checksum_validation_skipped"], true);+ let warnings = nested.details["warnings"].as_array().unwrap();+ assert!(warnings.iter().any(|warning| warning.as_str().unwrap().contains("not verified")));+ assert!(!warnings.iter().any(|warning| warning.as_str().unwrap().contains("checksum mismatch")));+ let visited: u64 =+ result.regions.iter().filter(|region| region.kind == "cpio-crc").map(|region| region.details["checksum_bytes_visited"].as_u64().unwrap()).sum();+ assert_eq!(visited, image.len() as u64, "the partial probe consumes only the remaining shared allowance");++ let plain = result.regions.iter().find(|region| region.offset == plain_offset as u64).unwrap();+ assert_eq!(plain.kind, "cpio-newc");+ assert_eq!(plain.confidence, "high");+ let empty = result.regions.iter().find(|region| region.offset == empty_offset as u64).unwrap();+ assert_eq!(empty.kind, "cpio-crc");+ assert_eq!(empty.confidence, "high");+ assert_eq!(empty.details["checksum_validation_skipped"], false, "a zero-byte member still verifies with no byte allowance left");+ assert_eq!(empty.details["checksum_bytes_visited"], 0);+ assert!(empty.details["warnings"].as_array().unwrap().is_empty());+}++#[test]+fn crc_body_cancellation_does_not_publish_a_completed_region() {+ let archive = crc_archive(&vec![7; 3 * 64 * 1024]);+ let calls = Cell::new(0usize);+ let cancelled = scan(&archive, &ScanOptions::default(), &|| {+ calls.set(calls.get() + 1);+ calls.get() >= 3+ });+ assert!(cancelled.cancelled, "the stop callback must be polled inside the checksum body");+ assert_eq!(calls.get(), 3);+ assert!(cancelled.regions.is_empty(), "a cancelled archive probe must not be reported as complete");++ let finished = scan(&archive, &ScanOptions::default(), &|| false);+ assert!(!finished.cancelled);+ let region = finished.regions.iter().find(|region| region.offset == 0).unwrap();+ assert_eq!(region.confidence, "high");+ assert_eq!(region.details["complete"], true);+ assert!(region.details["warnings"].as_array().unwrap().is_empty());+}diff --git a/docs/ROADMAP.md b/docs/ROADMAP.mdindex df5e7095..d00443bb 100644--- a/docs/ROADMAP.md+++ b/docs/ROADMAP.md@@ -645,6 +645,8 @@ Owner: open · Status: in progressPAX sizes across other metadata records; CRC cpio checks member data; source maps decodepercent-escaped data URLs and report indexed-map columns in the generated file(#606–#615, by Afloat16)+- [x] Bound and cancel CRC cpio verification across firmware probes, and reject non-hex checksum+ fields — claimed: Afloat16, branch bugfix/r3.12-bound-cpio-crc-scans-20261103### R3.13 Web inspection correctnessOwner: open · Status: in progressdiff --git a/docs/pitfalls.md b/docs/pitfalls.mdindex 6f7560fc..5ae65657 100644--- a/docs/pitfalls.md+++ b/docs/pitfalls.md@@ -671,3 +671,10 @@ opencode-session under settings::config_dir() when the first request was made, stests left a file on the developer's machine and two runs could collide. Anything we persist goesthrough the hub's stores (AiHub::with_stores, Session::isolated), which tests point at a tempfolder; production code never writes under the real config folder by hand.++**A per-probe walk repeated expensive work on the same bytes.** CRC cpio verification was bounded+inside one archive, but a firmware scan could rediscover overlapping, trailerless archives and+sum the same payload repeatedly. Count checksum byte visits across the whole scan, poll its+cancellation callback inside long sums, and distinguish skipped verification from a mismatch.+Test overlapping candidates with correct checksums and a complete nested archive so a structural+trailer cannot turn unchecked data into a high-confidence result.