A fast, helpful, and open-source document parser
improve file type from bytes detection
Logan Markewich29a35b38be7e
diff --git a/Cargo.lock b/Cargo.lockindex 0a11802..194b2f3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -264,17 +264,6 @@ dependencies = ["nom",]-[[package]]-name = "cfb"-version = "0.7.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"-dependencies = [- "byteorder",- "fnv",- "uuid",-]-[[package]]name = "cfg-if"version = "1.0.4"@@ -598,6 +587,12 @@ dependencies = ["simd-adler32",]+[[package]]+name = "file-format"+version = "0.29.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "55d9ccda37e95b4f0978a3074b4a9939979103a7256459cfb449c9c84d1adf23"+[[package]]name = "filetime"version = "0.2.27"@@ -1082,15 +1077,6 @@ dependencies = ["serde_core",]-[[package]]-name = "infer"-version = "0.19.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7"-dependencies = [- "cfb",-]-[[package]]name = "inout"version = "0.1.4"@@ -1250,8 +1236,8 @@ version = "2.2.1"dependencies = ["blake3","clap",+ "file-format","image",- "infer","liteparse-pdfium","liteparse-pdfium-sys","ordered-float",@@ -2813,16 +2799,6 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"-[[package]]-name = "uuid"-version = "1.23.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"-dependencies = [- "js-sys",- "wasm-bindgen",-]-[[package]]name = "vcpkg"version = "0.2.15"diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.tomlindex 48477fc..e4b085a 100644--- a/crates/liteparse/Cargo.toml+++ b/crates/liteparse/Cargo.toml@@ -21,8 +21,8 @@ tesseract = ["dep:tesseract-rs"][dependencies]blake3 = "1"clap = { version = "4.5.55", features = ["derive"] }+file-format = { version = "0.29.0", features = ["reader"] }image = { version = "0.25", default-features = false, features = ["png"] }-infer = "0.19.0"ordered-float = "5.3.0"pdfium = { package = "liteparse-pdfium", version = "1.3.0", path = "../pdfium" }pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.3.0", path = "../pdfium-sys" }diff --git a/crates/liteparse/src/conversion.rs b/crates/liteparse/src/conversion.rsindex e6e83d5..95d8d2f 100644--- a/crates/liteparse/src/conversion.rs+++ b/crates/liteparse/src/conversion.rs@@ -1,3 +1,4 @@+use file_format::FileFormat;use tempfile::TempDir;use crate::error::LiteParseError;@@ -553,7 +554,16 @@ pub async fn convert_image_to_pdf(}pub fn guess_extension_from_data(data: &[u8]) -> Option<String> {- infer::get(data).map(|k| k.extension().to_string())+ // `file-format` inspects ZIP-based containers via their central directory+ // (requires the `reader` feature), so DOCX/XLSX/PPTX/ODF resolve to their+ // specific format instead of a generic "zip" regardless of entry ordering+ // or file size. Unrecognized input falls back to the generic binary format,+ // which we surface as `None` to match the prior contract.+ let fmt = FileFormat::from_bytes(data);+ if fmt == FileFormat::ArbitraryBinaryData {+ return None;+ }+ Some(fmt.extension().to_string())}pub async fn convert_data_to_pdf(@@ -602,6 +612,74 @@ mod tests {assert!(!is_supported_extension("noext"));}+ /// Build a minimal but structurally valid ZIP (stored, empty entries) whose+ /// central directory lists `names`. This exercises the same code path+ /// `file-format` uses to disambiguate ZIP-based office formats.+ fn build_zip(names: &[&str]) -> Vec<u8> {+ let mut out = Vec::new();+ let mut offsets = Vec::new();+ for name in names {+ offsets.push(out.len() as u32);+ let nb = name.as_bytes();+ out.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); // local header sig+ out.extend_from_slice(&20u16.to_le_bytes()); // version needed+ out.extend_from_slice(&[0u8; 20]); // flags..uncompressed size (all zero)+ out.extend_from_slice(&(nb.len() as u16).to_le_bytes());+ out.extend_from_slice(&0u16.to_le_bytes()); // extra len+ out.extend_from_slice(nb);+ }+ let cd_offset = out.len() as u32;+ let mut central = Vec::new();+ for (i, name) in names.iter().enumerate() {+ let nb = name.as_bytes();+ central.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); // central header sig+ central.extend_from_slice(&20u16.to_le_bytes()); // version made by+ central.extend_from_slice(&20u16.to_le_bytes()); // version needed+ central.extend_from_slice(&[0u8; 20]); // flags..uncompressed size+ central.extend_from_slice(&(nb.len() as u16).to_le_bytes());+ central.extend_from_slice(&[0u8; 8]); // extra/comment/disk/internal attrs+ central.extend_from_slice(&0u32.to_le_bytes()); // external attrs+ central.extend_from_slice(&offsets[i].to_le_bytes()); // local header offset+ central.extend_from_slice(nb);+ }+ let cd_size = central.len() as u32;+ out.extend_from_slice(¢ral);+ out.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); // EOCD sig+ out.extend_from_slice(&[0u8; 4]); // disk numbers+ out.extend_from_slice(&(names.len() as u16).to_le_bytes());+ out.extend_from_slice(&(names.len() as u16).to_le_bytes());+ out.extend_from_slice(&cd_size.to_le_bytes());+ out.extend_from_slice(&cd_offset.to_le_bytes());+ out.extend_from_slice(&0u16.to_le_bytes()); // comment len+ out+ }++ #[test]+ fn test_guess_extension_disambiguates_zip_office_formats() {+ // ZIP-based office formats must resolve to their specific type, not a+ // generic "zip", even though they share the PK signature.+ assert_eq!(+ guess_extension_from_data(&build_zip(&["[Content_Types].xml", "word/document.xml"]))+ .as_deref(),+ Some("docx")+ );+ assert_eq!(+ guess_extension_from_data(&build_zip(&["[Content_Types].xml", "xl/workbook.xml"]))+ .as_deref(),+ Some("xlsx")+ );+ assert_eq!(+ guess_extension_from_data(&build_zip(&["[Content_Types].xml", "ppt/presentation.xml"]))+ .as_deref(),+ Some("pptx")+ );+ // A plain ZIP with no office markers stays "zip".+ assert_eq!(+ guess_extension_from_data(&build_zip(&["random/file.txt"])).as_deref(),+ Some("zip")+ );+ }+#[test]fn test_conversion_tool_display() {assert_eq!(ConversionTool::ImageMagick.to_string(), "ImageMagick");