Skip to content

Craft Anything: understand how any software works, and craft your own.

Rust

Address the parser review findings from #722 (#766)

Afloateab6a0f01bf9
diff --git a/crates/engine/src/tools/java_tools.rs b/crates/engine/src/tools/java_tools.rs
index 988c7f37..af2f690c 100644
--- a/crates/engine/src/tools/java_tools.rs
+++ b/crates/engine/src/tools/java_tools.rs
@@ -747,7 +747,10 @@ fn java_class_users_find(ctx: &mut Ctx<'_>, params: Value) -> Result<ToolOutput,
"Only direct bytecode references are found; reflection, invokedynamic bootstrap targets, service loaders and names built at run time are not.",
));
if truncated {
- lims.push(Limitation::new("truncated", format!("stopped after {MAX_REFS} references")));
+ lims.push(Limitation::new(
+ "truncated",
+ format!("The scan stopped early because a method could not be fully decoded or the {MAX_REFS}-reference limit was reached."),
+ ));
cov.status = CoverageStatus::Partial;
}
let b = ev(ctx, "java_class_users_find", &src, json!({"path": p.path, "class_name": p.class_name, "method_name": p.method_name, "offset": p.offset, "limit": p.limit}))
diff --git a/crates/formats/src/axml.rs b/crates/formats/src/axml.rs
index caeae09f..db4fc093 100644
--- a/crates/formats/src/axml.rs
+++ b/crates/formats/src/axml.rs
@@ -20,6 +20,8 @@ const XML_RESOURCE_MAP: u16 = 0x0180;
const MAX_STRINGS: usize = 1_000_000;
/// Longest string kept (characters).
const MAX_STRING: usize = 64 * 1024;
+/// Most extra UTF-8 length candidates examined across one string pool.
+const MAX_RECOVERY_PROBES: usize = 65_536;
/// Most XML nodes decoded.
const MAX_NODES: usize = 200_000;
/// Deepest element nesting tracked.
@@ -57,6 +59,11 @@ impl StringPool {
/// Parse the pool whose chunk starts at `off`.
pub fn parse(data: &[u8], off: usize) -> Result<StringPool, FormatError> {
+ let mut recovery_left = MAX_RECOVERY_PROBES;
+ Self::parse_with_recovery_budget(data, off, &mut recovery_left)
+ }
+
+ fn parse_with_recovery_budget(data: &[u8], off: usize, recovery_left: &mut usize) -> Result<StringPool, FormatError> {
let r = Reader::new(data);
let (t, hdr, size) = chunk(&r, off).ok_or_else(|| FormatError::Truncated("string pool header".into()))?;
if t != RES_STRING_POOL {
@@ -92,7 +99,7 @@ impl StringPool {
let s = if at >= string_bytes.len() {
None
} else if utf8 {
- utf8_string(&strings_reader, at, string_bytes.len())
+ utf8_string(&strings_reader, at, string_bytes.len(), recovery_left)
} else {
utf16_string(&strings_reader, at, string_bytes.len())
};
@@ -112,13 +119,24 @@ fn utf8_len(r: &Reader<'_>, at: usize) -> Option<(usize, usize)> {
}
}
-fn utf8_string(r: &Reader<'_>, at: usize, end: usize) -> Option<String> {
+fn utf8_string(r: &Reader<'_>, at: usize, end: usize, recovery_left: &mut usize) -> Option<String> {
let (_chars, n1) = utf8_len(r, at)?;
- let (bytes, n2) = utf8_len(r, at.checked_add(n1)?)?;
+ let (mut bytes, n2) = utf8_len(r, at.checked_add(n1)?)?;
let start = at.checked_add(n1 + n2)?;
- let terminator = start.checked_add(bytes)?;
- if terminator >= end || r.u8(terminator)? != 0 {
- return None;
+ // Older AAPT keeps only the low 15 length bits; probe matching lengths inside this pool.
+ loop {
+ let terminator = start.checked_add(bytes)?;
+ if terminator >= end {
+ return None;
+ }
+ // The declared candidate costs one read per entry, already bounded by MAX_STRINGS.
+ if bytes >= 0x8000 {
+ *recovery_left = recovery_left.checked_sub(1)?;
+ }
+ if r.u8(terminator)? == 0 {
+ break;
+ }
+ bytes = bytes.checked_add(0x8000)?;
}
let b = r.slice(start, bytes.min(MAX_STRING * 4))?;
Some(String::from_utf8_lossy(b).into_owned())
@@ -677,6 +695,115 @@ mod tests {
use super::*;
use craft_anything_testkit::android as fx;
+ fn pool_with_offsets(offsets: &[u32], string_data: &[u8]) -> Vec<u8> {
+ let start = 28 + offsets.len() * 4;
+ let mut data = vec![0; start];
+ data[..2].copy_from_slice(&RES_STRING_POOL.to_le_bytes());
+ data[2..4].copy_from_slice(&28u16.to_le_bytes());
+ data[8..12].copy_from_slice(&(offsets.len() as u32).to_le_bytes());
+ data[16..20].copy_from_slice(&0x100u32.to_le_bytes());
+ data[20..24].copy_from_slice(&(start as u32).to_le_bytes());
+ for (i, offset) in offsets.iter().enumerate() {
+ let at = 28 + i * 4;
+ data[at..at + 4].copy_from_slice(&offset.to_le_bytes());
+ }
+ data.extend_from_slice(string_data);
+ data.resize(data.len().next_multiple_of(4), b'x');
+ let size = data.len() as u32;
+ data[4..8].copy_from_slice(&size.to_le_bytes());
+ data
+ }
+
+ fn append_legacy_ascii(data: &mut Vec<u8>, len: usize) -> u32 {
+ let offset = data.len() as u32;
+ let encoded = [0x80 | ((len >> 8) & 0x7f) as u8, (len & 0xff) as u8];
+ data.extend_from_slice(&encoded);
+ data.extend_from_slice(&encoded);
+ data.resize(data.len() + len, b'x');
+ data.push(0);
+ offset
+ }
+
+ #[test]
+ fn repeated_offsets_share_the_utf8_recovery_budget() {
+ let mut strings = Vec::new();
+ let long = append_legacy_ascii(&mut strings, 0x8000);
+ let short = strings.len() as u32;
+ strings.extend_from_slice(b"\x02\x02ok\0");
+ let data = pool_with_offsets(&[long, long, long, short], &strings);
+ let mut left = 2;
+ let parsed = StringPool::parse_with_recovery_budget(&data, 0, &mut left).unwrap();
+ assert_eq!(left, 0);
+ assert_eq!(parsed.declared, 4);
+ assert_eq!(parsed.strings.len(), 4);
+ assert_eq!(parsed.get(0).map(str::len), Some(0x8000));
+ assert_eq!(parsed.get(1), parsed.get(0));
+ assert_eq!(parsed.get(2), Some(""));
+ assert_eq!(parsed.get(3), Some("ok"));
+ }
+
+ #[test]
+ fn distinct_offsets_share_the_utf8_recovery_budget() {
+ let mut strings = Vec::new();
+ let mut offsets: Vec<u32> = (0..3).map(|_| append_legacy_ascii(&mut strings, 0x8000)).collect();
+ offsets.push(strings.len() as u32);
+ strings.extend_from_slice(b"\x02\x02ok\0");
+ let data = pool_with_offsets(&offsets, &strings);
+ let mut left = 2;
+ let parsed = StringPool::parse_with_recovery_budget(&data, 0, &mut left).unwrap();
+ assert_eq!(left, 0);
+ assert_eq!(parsed.get(0).map(str::len), Some(0x8000));
+ assert_eq!(parsed.get(1).map(str::len), Some(0x8000));
+ assert_eq!(parsed.get(2), Some(""));
+ assert_eq!(parsed.get(3), Some("ok"));
+ }
+
+ #[test]
+ fn utf8_recovery_stops_before_a_probe_without_credit() {
+ let mut strings = Vec::new();
+ let long = append_legacy_ascii(&mut strings, 0x10000);
+ let short = strings.len() as u32;
+ strings.extend_from_slice(b"\x02\x02ok\0");
+ let data = pool_with_offsets(&[long, short], &strings);
+ for (allowance, length, remaining) in [(0, 0, 0), (1, 0, 0), (2, 0x10000, 0), (3, 0x10000, 1)] {
+ let mut left = allowance;
+ let parsed = StringPool::parse_with_recovery_budget(&data, 0, &mut left).unwrap();
+ assert_eq!(left, remaining);
+ assert_eq!(parsed.get(0).map(str::len), Some(length));
+ assert_eq!(parsed.get(1), Some("ok"));
+ }
+ }
+
+ #[test]
+ fn short_strings_and_out_of_bounds_candidates_spend_no_recovery_probes() {
+ let data = pool_with_offsets(&[0, 5], b"\x80\0\x80\0x\x02\x02ok\0");
+ let mut left = 7;
+ let parsed = StringPool::parse_with_recovery_budget(&data, 0, &mut left).unwrap();
+ assert_eq!(left, 7);
+ assert_eq!(parsed.get(0), Some(""));
+ assert_eq!(parsed.get(1), Some("ok"));
+ }
+
+ #[test]
+ fn default_pool_budget_stops_repeated_recovery_before_later_long_strings() {
+ let mut strings = vec![b'x'; 2 * 0x8000 + 16];
+ strings[..4].copy_from_slice(&[0x80, 0, 0x80, 0]);
+ let long = append_legacy_ascii(&mut strings, 0x8000);
+ let short = strings.len() as u32;
+ strings.extend_from_slice(b"\x02\x02ok\0");
+ // Offset zero has three in-bounds extra candidates, all nonzero.
+ let repeated = MAX_RECOVERY_PROBES / 3 + 1;
+ let mut offsets = vec![0; repeated];
+ offsets.extend([long, short]);
+ let data = pool_with_offsets(&offsets, &strings);
+ let parsed = StringPool::parse(&data, 0).unwrap();
+ assert_eq!(parsed.declared, repeated + 2);
+ assert_eq!(parsed.strings.len(), repeated + 2);
+ assert!(parsed.strings[..repeated].iter().all(String::is_empty));
+ assert_eq!(parsed.get(repeated as u32), Some(""));
+ assert_eq!(parsed.get((repeated + 1) as u32), Some("ok"));
+ }
+
#[test]
fn decodes_manifest() {
let doc = parse(&fx::manifest_axml()).unwrap();
diff --git a/crates/formats/src/dex.rs b/crates/formats/src/dex.rs
index b01a1b24..6b23d2e7 100644
--- a/crates/formats/src/dex.rs
+++ b/crates/formats/src/dex.rs
@@ -156,14 +156,14 @@ pub fn mutf8(r: &Reader<'_>, mut at: usize, max_units: usize) -> (String, usize)
(String::from_utf16_lossy(&units), consumed)
}
-/// Signed LEB128 at `off`: (value, bytes).
+/// Signed 32-bit DEX LEB128 at `off`: (value, bytes), using at most five bytes.
pub fn sleb128(r: &Reader<'_>, off: usize) -> Option<(i64, usize)> {
let mut result = 0i64;
let mut shift = 0u32;
- for i in 0..10usize {
+ for i in 0..5usize {
let b = r.u8(off.checked_add(i)?)?;
- // Only one value bit remains in the tenth byte; the rest must extend its sign.
- if i == 9 && !matches!(b, 0x00 | 0x7f) {
+ // Four value bits remain in byte five; the other bits must extend the sign.
+ if i == 4 && !matches!(b, 0x00..=0x07 | 0x78..=0x7f) {
return None;
}
result |= i64::from(b & 0x7f).checked_shl(shift)?;
diff --git a/crates/formats/src/fsread/extfs.rs b/crates/formats/src/fsread/extfs.rs
index 3a6f834e..bb3f23a9 100644
--- a/crates/formats/src/fsread/extfs.rs
+++ b/crates/formats/src/fsread/extfs.rs
@@ -75,11 +75,11 @@ fn block_bytes<'a>(data: &'a [u8], sb: &Sb, block: u64) -> Option<&'a [u8]> {
}
/// Collect the physical block numbers for logical blocks 0..n of an inode, via block pointers or
-/// an extent tree. Unmapped logical blocks are `None` (sparse → zeros).
-fn file_block_map(data: &[u8], sb: &Sb, inode_off: u64, flags: u32, nblocks: u64, img: &mut FsImage) -> Vec<Option<u64>> {
+/// an extent tree. Unmapped logical blocks are `None` (sparse → zeros); unreadable indirect
+/// metadata makes the whole map unavailable rather than turning unknown mappings into holes.
+fn file_block_map(data: &[u8], sb: &Sb, inode_off: u64, flags: u32, nblocks: u64, img: &mut FsImage) -> Option<Vec<Option<u64>>> {
let r = Reader::new(data);
- let ib = usize::try_from(inode_off).ok().and_then(|o| o.checked_add(40));
- let Some(ib) = ib else { return Vec::new() };
+ let ib = usize::try_from(inode_off).ok().and_then(|o| o.checked_add(40))?;
let n = usize::try_from(nblocks.min(1 << 20)).unwrap_or(0);
let mut map = vec![None; n];
if flags & 0x8_0000 != 0 {
@@ -103,33 +103,38 @@ fn file_block_map(data: &[u8], sb: &Sb, inode_off: u64, flags: u32, nblocks: u64
set(&mut logical, p, &mut map);
}
}
- // Single indirect.
- if let Some(ind) = r.u32_le(ib + 48).filter(|&p| p != 0) {
- read_indirect(data, sb, u64::from(ind), 1, per, &mut logical, &mut map);
- }
- // The single-indirect span still occupies its logical blocks when its root is absent.
- logical = 12 + per;
- // Double indirect.
- if let Some(dind) = r.u32_le(ib + 52).filter(|&p| p != 0) {
- read_indirect(data, sb, u64::from(dind), 2, per, &mut logical, &mut map);
+ // Each root owns its full logical span, including when its pointer is zero.
+ for (level, slot, first) in [(1, 48, 12), (2, 52, 12 + per)] {
+ logical = first;
+ if let Some(root) = r.u32_le(ib + slot).filter(|&p| p != 0)
+ && let Err(block) = read_indirect(data, sb, u64::from(root), level, per, &mut logical, &mut map)
+ {
+ img.warn(format!("ext: indirect block {block} is unavailable; file data not recovered"));
+ img.complete = false;
+ return None;
+ }
}
- if r.u32_le(ib + 56).is_some_and(|p| p != 0) {
+ if nblocks > 12 + per + per * per && r.u32_le(ib + 56).is_some_and(|p| p != 0) {
img.warn("ext: triple-indirect blocks are not followed");
img.complete = false;
+ return None;
}
}
- map
+ Some(map)
}
-fn read_indirect(data: &[u8], sb: &Sb, block: u64, level: u8, per: u64, logical: &mut u64, map: &mut Vec<Option<u64>>) {
+fn read_indirect(data: &[u8], sb: &Sb, block: u64, level: u8, per: u64, logical: &mut u64, map: &mut Vec<Option<u64>>) -> Result<(), u64> {
if *logical as usize >= map.len() {
- return;
+ return Ok(());
}
- let Some(b) = block_bytes(data, sb, block) else { return };
+ let b = block_bytes(data, sb, block).ok_or(block)?;
let r = Reader::new(b);
for i in 0..per {
- let Some(off) = usize::try_from(i.saturating_mul(4)).ok() else { break };
- let Some(ptr) = r.u32_le(off) else { break };
+ if *logical as usize >= map.len() {
+ return Ok(());
+ }
+ let off = usize::try_from(i.saturating_mul(4)).map_err(|_| block)?;
+ let ptr = r.u32_le(off).ok_or(block)?;
if ptr == 0 {
*logical += if level == 1 { 1 } else { per };
continue;
@@ -142,12 +147,10 @@ fn read_indirect(data: &[u8], sb: &Sb, block: u64, level: u8, per: u64, logical:
}
*logical += 1;
} else {
- read_indirect(data, sb, u64::from(ptr), level - 1, per, logical, map);
- }
- if *logical as usize >= map.len() {
- return;
+ read_indirect(data, sb, u64::from(ptr), level - 1, per, logical, map)?;
}
}
+ Ok(())
}
fn extents(data: &[u8], sb: &Sb, node_off: usize, map: &mut Vec<Option<u64>>, depth_guard: u8, img: &mut FsImage) {
@@ -200,7 +203,7 @@ fn file_data(data: &[u8], sb: &Sb, inode_off: u64, flags: u32, size: u64, limits
return None;
}
let nblocks = size.div_ceil(sb.block_size);
- let map = file_block_map(data, sb, inode_off, flags, nblocks, img);
+ let map = file_block_map(data, sb, inode_off, flags, nblocks, img)?;
let mut out = Vec::with_capacity(usize::try_from(size).unwrap_or(0));
for phys in &map {
match phys {
diff --git a/crates/formats/src/java_class.rs b/crates/formats/src/java_class.rs
index 22d336b3..e0ccdbfd 100644
--- a/crates/formats/src/java_class.rs
+++ b/crates/formats/src/java_class.rs
@@ -762,13 +762,15 @@ pub struct Reference {
}
/// Every instruction in `cf` that refers to the internal class `owner` (optionally only calls to
-/// its methods named `method`). Stops after `max` hits; the bool is true when it stopped early.
+/// its methods named `method`). The bool is true when decoding or the reference limit stops a scan early.
pub fn references(cf: &ClassFile, owner: &str, method: Option<&str>, max: usize) -> (Vec<Reference>, bool) {
let mut out = Vec::new();
+ let mut incomplete = false;
for m in &cf.methods {
let Some(code) = &m.code else { continue };
let caller = Ref { owner: cf.this_class.clone(), name: m.name.clone(), descriptor: m.descriptor.clone() }.display();
- let (insns, _) = java_ops::decode(&code.bytecode, 200_000);
+ let (insns, complete) = java_ops::decode(&code.bytecode, 200_000);
+ incomplete |= !complete;
for i in &insns {
let Some(cp) = i.cp_index else { continue };
let hit = match i.opcode {
@@ -793,7 +795,7 @@ pub fn references(cf: &ClassFile, owner: &str, method: Option<&str>, max: usize)
}
}
}
- (out, false)
+ (out, incomplete)
}
#[cfg(test)]
diff --git a/crates/formats/tests/android_string_pool_bounds.rs b/crates/formats/tests/android_string_pool_bounds.rs
index 2d295953..94c2e477 100644
--- a/crates/formats/tests/android_string_pool_bounds.rs
+++ b/crates/formats/tests/android_string_pool_bounds.rs
@@ -24,6 +24,20 @@ fn pool(utf8: bool, string_data: &[u8], styles: &[u8]) -> Vec<u8> {
data
}
+fn legacy_utf8(text: &str) -> Vec<u8> {
+ let mut data = Vec::new();
+ // Legacy AAPT writes the low 15 bits of both lengths even when the full value is larger.
+ for len in [text.encode_utf16().count(), text.len()] {
+ if len > 0x7f {
+ data.push(((len >> 8) & 0x7f) as u8 | 0x80);
+ }
+ data.push((len & 0xff) as u8);
+ }
+ data.extend_from_slice(text.as_bytes());
+ data.push(0);
+ data
+}
+
fn valid_pool() -> Vec<u8> {
pool(true, b"\x04\x04kept\0", &[])
}
@@ -90,3 +104,65 @@ fn empty_pools_with_no_string_data_remain_valid() {
u32_at(&mut data, 4, 28);
assert!(StringPool::parse(&data, 0).unwrap().strings.is_empty());
}
+
+#[test]
+fn legacy_aapt_utf8_recovers_a_65535_byte_string() {
+ let expected = "x".repeat(65_535);
+ let string_data = legacy_utf8(&expected);
+ assert_eq!(&string_data[..4], &[0xff; 4]);
+ let parsed = StringPool::parse(&pool(true, &string_data, &[]), 0).unwrap();
+ assert_eq!(parsed.get(0), Some(expected.as_str()));
+}
+
+#[test]
+fn legacy_aapt_utf8_output_remains_capped() {
+ let text = "x".repeat(64 * 1024 * 4 + 0x7fff);
+ let parsed = StringPool::parse(&pool(true, &legacy_utf8(&text), &[]), 0).unwrap();
+ let kept = parsed.get(0).unwrap();
+ assert_eq!(kept.len(), 64 * 1024 * 4);
+ assert!(kept.bytes().all(|b| b == b'x'));
+}
+
+#[test]
+fn legacy_aapt_utf8_cannot_borrow_a_terminator_after_its_chunk() {
+ let string_data = legacy_utf8(&"x".repeat(65_535));
+ let mut data = pool(true, &string_data, &[]);
+ let declared_size = data.len() - 4;
+ u32_at(&mut data, 4, declared_size as u32);
+ assert_eq!(data[data.len() - 1], 0);
+ assert_eq!(StringPool::parse(&data, 0).unwrap().get(0), Some(""));
+}
+
+#[test]
+fn legacy_aapt_utf8_cannot_borrow_a_terminator_from_styles() {
+ let mut string_data = legacy_utf8(&"x".repeat(65_532));
+ assert_eq!(string_data.pop(), Some(0));
+ let data = pool(true, &string_data, &[0; 12]);
+ assert_eq!(StringPool::parse(&data, 0).unwrap().get(0), Some(""));
+}
+
+#[test]
+fn legacy_aapt_utf8_needs_a_terminator_at_a_matching_length() {
+ let mut string_data = legacy_utf8(&"x".repeat(65_535));
+ *string_data.last_mut().unwrap() = b'x';
+ string_data[4 + 40_000] = 0;
+ let data = pool(true, &string_data, &[]);
+ assert_eq!(StringPool::parse(&data, 0).unwrap().get(0), Some(""));
+}
+
+#[test]
+fn truncated_utf8_length_prefixes_and_bodies_are_not_recovered() {
+ let string_data = legacy_utf8(&"x".repeat(65_535));
+ for cut in [0, 1, 2, 3, 4, 32_770, 65_539] {
+ let data = pool(true, &string_data[..cut], &[0; 12]);
+ assert_eq!(StringPool::parse(&data, 0).unwrap().get(0), Some(""), "cut at {cut}");
+ }
+}
+
+#[test]
+fn ordinary_utf8_lengths_keep_their_exact_content() {
+ for text in [String::new(), "kept".into(), "a".repeat(127), "b".repeat(128), "c".repeat(32_767), "🙂".repeat(128)] {
+ let parsed = StringPool::parse(&pool(true, &legacy_utf8(&text), &[]), 0).unwrap();
+ assert_eq!(parsed.get(0), Some(text.as_str()));
+ }
+}
diff --git a/crates/formats/tests/dex_signed_leb128.rs b/crates/formats/tests/dex_signed_leb128.rs
index 3a5c36d0..699a17e7 100644
--- a/crates/formats/tests/dex_signed_leb128.rs
+++ b/crates/formats/tests/dex_signed_leb128.rs
@@ -1,10 +1,11 @@
-use craft_anything_formats::{bytes::Reader, dex::sleb128};
+use craft_anything_formats::bytes::Reader;
+use craft_anything_formats::dex::{self, Code, Handler, sleb128};
#[test]
-fn rejects_every_overflowing_tenth_byte() {
+fn rejects_invalid_fifth_byte_sign_extensions() {
for prefix in [0x80, 0xff] {
- for last in 1..0x7f {
- let mut encoded = vec![prefix; 9];
+ for last in 0x08..=0x77 {
+ let mut encoded = vec![prefix; 4];
encoded.push(last);
assert_eq!(sleb128(&Reader::new(&encoded), 0), None, "last byte {last:#x}");
}
@@ -12,41 +13,82 @@ fn rejects_every_overflowing_tenth_byte() {
}
#[test]
-fn retains_signed_limits_and_ordinary_values() {
- let mut maximum = vec![0xff; 9];
- maximum.push(0);
- let mut minimum = vec![0x80; 9];
- minimum.push(0x7f);
- assert_eq!(sleb128(&Reader::new(&maximum), 0), Some((i64::MAX, 10)));
- assert_eq!(sleb128(&Reader::new(&minimum), 0), Some((i64::MIN, 10)));
+fn retains_signed_32_bit_limits_and_ordinary_values() {
+ assert_eq!(sleb128(&Reader::new(&[0xff, 0xff, 0xff, 0xff, 0x07]), 0), Some((i64::from(i32::MAX), 5)));
+ assert_eq!(sleb128(&Reader::new(&[0x80, 0x80, 0x80, 0x80, 0x78]), 0), Some((i64::from(i32::MIN), 5)));
assert_eq!(sleb128(&Reader::new(&[0x9b, 0xf1, 0x59]), 0), Some((-624485, 3)));
+ assert_eq!(sleb128(&Reader::new(&[0xe5, 0x8e, 0x26]), 0), Some((624485, 3)));
assert_eq!(sleb128(&Reader::new(&[0xff, 0x7f]), 0), Some((-1, 2)));
assert_eq!(sleb128(&Reader::new(&[0x80, 0]), 0), Some((0, 2)));
+ assert_eq!(sleb128(&Reader::new(&[0x55, 0x7f, 0x55]), 1), Some((-1, 1)));
+}
+
+#[test]
+fn retains_valid_five_byte_sign_extensions() {
+ assert_eq!(sleb128(&Reader::new(&[0x80, 0x80, 0x80, 0x80, 0]), 0), Some((0, 5)));
+ assert_eq!(sleb128(&Reader::new(&[0x81, 0x80, 0x80, 0x80, 0]), 0), Some((1, 5)));
+ assert_eq!(sleb128(&Reader::new(&[0xff, 0xff, 0xff, 0xff, 0x7f]), 0), Some((-1, 5)));
+}
+
+#[test]
+fn rejects_values_longer_than_five_bytes() {
+ for (prefix, last) in [(0x80, 0), (0xff, 0x7f)] {
+ for length in [6, 10] {
+ let mut encoded = vec![prefix; length - 1];
+ encoded.push(last);
+ assert_eq!(sleb128(&Reader::new(&encoded), 0), None, "{encoded:x?}");
+ }
+ }
}
#[test]
fn rejects_truncated_values_and_out_of_range_offsets() {
- assert_eq!(sleb128(&Reader::new(&[0x80; 10]), 0), None);
+ assert_eq!(sleb128(&Reader::new(&[0x80; 5]), 0), None);
assert_eq!(sleb128(&Reader::new(&[0x80]), 0), None);
+ assert_eq!(sleb128(&Reader::new(&[]), 0), None);
assert_eq!(sleb128(&Reader::new(&[0]), usize::MAX), None);
}
-#[test]
-fn overflow_does_not_create_a_catch_all_handler() {
+fn code_with_handler(size: &[u8], payload: &[u8]) -> Code {
let mut bytes = vec![0; 0x70];
bytes[..8].copy_from_slice(b"dex\n035\0");
bytes[36..40].copy_from_slice(&0x70u32.to_le_bytes());
bytes[40..44].copy_from_slice(&0x1234_5678u32.to_le_bytes());
- // Empty code item with one try entry pointing to the first encoded handler.
- bytes.extend_from_slice(&[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
- bytes.extend_from_slice(&[0, 0, 0, 0, 0, 0, 1, 0]);
+ // Eight nop instructions and one try entry pointing to the first encoded handler.
+ bytes.extend_from_slice(&[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0]);
+ bytes.extend_from_slice(&[0; 16]);
+ bytes.extend_from_slice(&[0, 0, 0, 0, 1, 0, 1, 0]);
bytes.push(1);
- bytes.extend_from_slice(&[0x80; 9]);
- bytes.extend_from_slice(&[2, 7]);
+ bytes.extend_from_slice(size);
+ bytes.extend_from_slice(payload);
let len = bytes.len() as u32;
bytes[32..36].copy_from_slice(&len.to_le_bytes());
- let dex = craft_anything_formats::dex::parse(&bytes).unwrap();
- let code = dex.code(0x70).unwrap();
- assert_eq!(code.tries.len(), 1);
- assert!(code.tries[0].handlers.is_empty());
+ dex::parse(&bytes).unwrap().code(0x70).unwrap()
+}
+
+#[test]
+fn invalid_sizes_do_not_create_handlers() {
+ let sizes: &[&[u8]] = &[
+ &[0x80, 0x80, 0x80, 0x80, 0x08],
+ &[0x80, 0x80, 0x80, 0x80, 0x77],
+ &[0x80, 0x80, 0x80, 0x80, 0x80, 0],
+ &[0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0],
+ ];
+ for size in sizes {
+ // These bytes could be a type/address pair or a catch-all address if size is accepted.
+ let code = code_with_handler(size, &[0, 7]);
+ assert_eq!(code.tries.len(), 1);
+ assert!(code.tries[0].handlers.is_empty(), "invalid size {size:x?}");
+ }
+}
+
+#[test]
+fn valid_catch_all_handlers_are_retained() {
+ for size in [&[0][..], &[0x80, 0x80, 0x80, 0x80, 0][..]] {
+ let code = code_with_handler(size, &[7]);
+ assert_eq!(code.tries.len(), 1);
+ assert_eq!(code.tries[0].start, 0);
+ assert_eq!(code.tries[0].end, 1);
+ assert_eq!(code.tries[0].handlers, vec![Handler { exception: None, address: 7 }]);
+ }
}
diff --git a/crates/formats/tests/ext_indirect_metadata.rs b/crates/formats/tests/ext_indirect_metadata.rs
new file mode 100644
index 00000000..0bf4c02a
--- /dev/null
+++ b/crates/formats/tests/ext_indirect_metadata.rs
@@ -0,0 +1,142 @@
+use craft_anything_formats::fsread::{self, Limits};
+
+const BS: usize = 1024;
+const PER: usize = BS / 4;
+const FILE_INODE: usize = 5120 + 10 * 128;
+const FIRST_DOUBLE: usize = 12 + PER;
+const FIRST_TRIPLE: usize = FIRST_DOUBLE + PER * PER;
+
+fn put(data: &mut [u8], at: usize, value: u32) {
+ data[at..at + 4].copy_from_slice(&value.to_le_bytes());
+}
+
+fn sparse_file(nblocks: usize) -> Vec<u8> {
+ let mut data = craft_anything_testkit::firmware::ext_image(false);
+ data.resize(15 * BS, 0);
+ put(&mut data, 1024 + 4, 15);
+ data[FILE_INODE + 40..FILE_INODE + 100].fill(0);
+ put(&mut data, FILE_INODE + 4, (nblocks * BS) as u32);
+ data
+}
+
+fn assert_unrecovered(data: &[u8], nblocks: usize, warning: &str) {
+ let image = fsread::extfs::read(data, &Limits::default());
+ let hello = image.entries.iter().find(|e| e.path == "hello.txt").unwrap();
+ assert_eq!(hello.size, (nblocks * BS) as u64);
+ assert!(hello.data.is_none(), "unknown indirect mappings must not become recovered zero bytes");
+ assert!(!image.complete);
+ assert!(image.warnings.iter().any(|w| w.contains(warning)), "{:?}", image.warnings);
+ assert_eq!(image.entries.iter().find(|e| e.path == "sub/note").unwrap().data.as_deref(), Some(b"nested note".as_slice()));
+}
+
+fn recovered_file(data: &[u8], nblocks: usize) -> Vec<u8> {
+ let image = fsread::extfs::read(data, &Limits::default());
+ assert!(image.complete, "{:?}", image.warnings);
+ assert!(image.warnings.is_empty(), "{:?}", image.warnings);
+ assert_eq!(image.entries.iter().find(|e| e.path == "sub/note").unwrap().data.as_deref(), Some(b"nested note".as_slice()));
+ let hello = image.entries.into_iter().find(|e| e.path == "hello.txt").unwrap();
+ assert_eq!(hello.size, (nblocks * BS) as u64);
+ let bytes = hello.data.expect("known sparse holes and readable mappings are recovered");
+ assert_eq!(bytes.len(), nblocks * BS);
+ bytes
+}
+
+#[test]
+fn nonzero_indirect_roots_outside_the_image_are_unrecovered() {
+ for (slot, first) in [(88, 12), (92, FIRST_DOUBLE)] {
+ for block in [15, u32::MAX] {
+ let mut data = sparse_file(first + 1);
+ put(&mut data, FILE_INODE + slot, block);
+ assert_unrecovered(&data, first + 1, "indirect block");
+ }
+ }
+}
+
+#[test]
+fn a_truncated_indirect_root_is_not_a_sparse_hole() {
+ for (slot, first) in [(88, 12), (92, FIRST_DOUBLE)] {
+ let mut data = sparse_file(first + 1);
+ put(&mut data, FILE_INODE + slot, 14);
+ data.truncate(15 * BS - 1);
+ assert_unrecovered(&data, first + 1, "indirect block");
+ }
+}
+
+#[test]
+fn a_missing_double_indirect_child_cannot_shift_later_mappings() {
+ let nblocks = FIRST_DOUBLE + PER + 1;
+ for (missing, truncated) in [(15, false), (u32::MAX, false), (14, true)] {
+ let mut data = sparse_file(nblocks);
+ put(&mut data, FILE_INODE + 92, 11);
+ put(&mut data, 11 * BS, missing);
+ put(&mut data, 11 * BS + 4, 12);
+ put(&mut data, 12 * BS, 13);
+ data[13 * BS..13 * BS + 4].copy_from_slice(b"tail");
+ if truncated {
+ data.truncate(15 * BS - 1);
+ }
+ assert_unrecovered(&data, nblocks, "indirect block");
+ }
+}
+
+#[test]
+fn zero_indirect_roots_remain_sparse_through_the_first_triple_block() {
+ let nblocks = FIRST_TRIPLE + 1;
+ let bytes = recovered_file(&sparse_file(nblocks), nblocks);
+ assert!(bytes.iter().all(|b| *b == 0));
+}
+
+#[test]
+fn a_zero_double_indirect_child_keeps_the_following_data_in_place() {
+ let logical = FIRST_DOUBLE + PER;
+ let mut data = sparse_file(logical + 1);
+ put(&mut data, FILE_INODE + 92, 11);
+ put(&mut data, 11 * BS + 4, 12);
+ put(&mut data, 12 * BS, 13);
+ data[13 * BS..13 * BS + 4].copy_from_slice(b"tail");
+ let bytes = recovered_file(&data, logical + 1);
+ assert!(bytes[..logical * BS].iter().all(|b| *b == 0));
+ assert_eq!(&bytes[logical * BS..logical * BS + 4], b"tail");
+ assert!(bytes[logical * BS + 4..].iter().all(|b| *b == 0));
+}
+
+#[test]
+fn bad_indirect_roots_past_the_file_end_are_not_followed() {
+ for (slot, first) in [(88, 12), (92, FIRST_DOUBLE), (96, FIRST_TRIPLE)] {
+ let mut data = sparse_file(first);
+ put(&mut data, FILE_INODE + slot, u32::MAX);
+ let bytes = recovered_file(&data, first);
+ assert!(bytes.iter().all(|b| *b == 0));
+ }
+}
+
+#[test]
+fn bad_indirect_children_past_the_file_end_are_not_followed() {
+ for (slot, first, double) in [(88, 12, false), (92, FIRST_DOUBLE, true)] {
+ let mut data = sparse_file(first + 1);
+ put(&mut data, FILE_INODE + slot, 11);
+ put(&mut data, 11 * BS + 4, u32::MAX);
+ if double {
+ put(&mut data, 11 * BS, 12);
+ put(&mut data, 12 * BS, 13);
+ put(&mut data, 12 * BS + 4, u32::MAX);
+ } else {
+ put(&mut data, 11 * BS, 13);
+ }
+ data[13 * BS..13 * BS + 4].copy_from_slice(b"tail");
+ let bytes = recovered_file(&data, first + 1);
+ assert!(bytes[..first * BS].iter().all(|b| *b == 0));
+ assert_eq!(&bytes[first * BS..first * BS + 4], b"tail");
+ assert!(bytes[first * BS + 4..].iter().all(|b| *b == 0));
+ }
+}
+
+#[test]
+fn a_needed_nonzero_triple_root_is_unrecovered_until_triples_are_supported() {
+ let nblocks = FIRST_TRIPLE + 1;
+ for block in [11, 15, u32::MAX] {
+ let mut data = sparse_file(nblocks);
+ put(&mut data, FILE_INODE + 96, block);
+ assert_unrecovered(&data, nblocks, "triple-indirect");
+ }
+}
diff --git a/crates/formats/tests/java_invoke_operands.rs b/crates/formats/tests/java_invoke_operands.rs
index a0ab4139..c0b7c22e 100644
--- a/crates/formats/tests/java_invoke_operands.rs
+++ b/crates/formats/tests/java_invoke_operands.rs
@@ -1,3 +1,4 @@
+use craft_anything_formats::java_class;
use craft_anything_formats::java_ops::{decode, decode_one};
#[test]
@@ -35,3 +36,41 @@ fn valid_and_truncated_invocations_keep_their_boundaries() {
assert_eq!(instructions.len(), 3);
assert_eq!(instructions[2].pc, 10);
}
+
+fn reference_class() -> java_class::ClassFile {
+ let mut class = java_class::parse(&craft_anything_testkit::java::hello_class()).unwrap();
+ class.methods.retain(|method| method.name == "run");
+ class
+}
+
+#[test]
+fn malformed_invocations_mark_reference_scans_incomplete() {
+ let valid = reference_class();
+ let (expected, incomplete) = java_class::references(&valid, "java/io/PrintStream", None, 10);
+ assert!(!incomplete);
+ assert_eq!(expected.len(), 1);
+ for invalid in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1]] {
+ let mut class = valid.clone();
+ let code = &mut class.methods[0].code.as_mut().unwrap().bytecode;
+ let later_reference = code.clone();
+ *code = invalid.to_vec();
+ code.extend_from_slice(&later_reference);
+ let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10);
+ assert!(found.is_empty());
+ assert!(incomplete, "invalid invocation must not hide an unfinished reference scan: {invalid:?}");
+ }
+}
+
+#[test]
+fn incomplete_scans_keep_earlier_references_and_scan_other_methods() {
+ let mut class = reference_class();
+ let mut later_method = class.methods[0].clone();
+ later_method.name = "later".into();
+ class.methods[0].code.as_mut().unwrap().bytecode.extend_from_slice(&[0xba, 0, 1, 1, 0]);
+ class.methods.push(later_method);
+ let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10);
+ assert!(incomplete);
+ assert_eq!(found.len(), 2);
+ assert!(found[0].caller.contains(".run:("));
+ assert!(found[1].caller.contains(".later:("));
+}
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index 89d7e7c4..cff01714 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -778,6 +778,14 @@ Owner: open · Status: in progress
PYZ, JSON config edits (comments kept, every duplicate key removed), and UDP, TCP, DNS over
TCP, pcapng and TLS lengths (#688–#715, by Afloat16)
+- [x] Address the combined parser review: preserve partial Java reference coverage,
+ enforce DEX signed 32-bit LEB128 limits, recover legacy AAPT UTF-8 lengths within
+ their pool with a shared recovery probe limit, and distinguish missing ext indirect
+ metadata from sparse holes.
+ — claimed: Afloat16, branch bugfix/r3.12-parser-review-followups-main-20261104
+ Shared-budget revision `29c48e03` passed [full Linux CI](https://github.com/storytold/craft-anything/actions/runs/41762780268).
+ The merge with main `41d2175e` awaits a new CI run.
+
### R3.13 Web inspection correctness
Owner: open · Status: in progress
- [x] The HTML scanner finds script preloads in `rel` token lists, decodes asset URLs once (numeric
diff --git a/docs/STATUS.md b/docs/STATUS.md
index 376cbf34..f381fe78 100644
--- a/docs/STATUS.md
+++ b/docs/STATUS.md
@@ -51,6 +51,7 @@ workspace clippy, layers, fmt and the secret scan of all history were clean. Chi
| SQLite database files (`db_sqlite_show`) | A rowid alias column (`INTEGER PRIMARY KEY`, including the table-constraint form) shows the rowid; `DESC`, non-`INTEGER` and composite keys keep their stored values | Hand-built fixtures; generated files compared with the official SQLite library |
| SQLite WITHOUT ROWID tables | Read from the index b-tree, interior records and overflow included; rows in primary-key order, values in declaration order, `rowid` null | Hand-built fixtures; generated files compared with the official SQLite library |
| Builder discovery smoke fixture | The missing-builder smoke fixture controls discovery state and supplies a synthetic result. | All original warning, modal, action and build-availability assertions are retained. Source inspection only; automated validation has not yet run for this change. |
+| Parser review follow-ups | Java reference scans retain partial coverage; DEX signed LEB128 enforces 32-bit values; legacy AAPT strings stay inside their pool with a shared recovery probe limit; missing classic ext indirect metadata is not recovered as zeros. | Shared-budget revision `29c48e03` passed [full Linux CI](https://github.com/storytold/craft-anything/actions/runs/41762780268). The merge with main `41d2175e` awaits a new CI run. Extent-tree metadata handling is unchanged. |
### Builder retry policy