A fast, helpful, and open-source document parser
fix: fix failures in parsing images and office docs, add conversion for bytes to pdf when parsing and add integration tests to verify that everything works
Clelia (Astra) Bertellifefd28725a3b
diff --git a/.gitignore b/.gitignoreindex 4f90917..a620ebc 100644--- a/.gitignore+++ b/.gitignore@@ -42,6 +42,9 @@ tmp/temp/*.txt*.png+!integration_tests_data/receipt.png+!integration_tests_data/sample3.docx.doc+!integration_tests_data/sample.pdfdeno.json# Python filesdiff --git a/Cargo.lock b/Cargo.lockindex 97e6cf5..8202db0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -642,6 +642,17 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"+[[package]]+name = "futures-executor"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"+dependencies = [+ "futures-core",+ "futures-task",+ "futures-util",+]+[[package]]name = "futures-io"version = "0.3.32"@@ -1204,6 +1215,7 @@ dependencies = ["reqwest 0.13.3","serde","serde_json",+ "serial_test","tempfile","tesseract-rs","thiserror 2.0.18",@@ -2090,6 +2102,15 @@ dependencies = ["winapi-util",]+[[package]]+name = "scc"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc"+dependencies = [+ "sdd",+]+[[package]]name = "schannel"version = "0.1.29"@@ -2105,6 +2126,12 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"+[[package]]+name = "sdd"+version = "3.0.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"+[[package]]name = "security-framework"version = "3.7.0"@@ -2200,6 +2227,32 @@ dependencies = ["serde",]+[[package]]+name = "serial_test"+version = "3.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f"+dependencies = [+ "futures-executor",+ "futures-util",+ "log",+ "once_cell",+ "parking_lot",+ "scc",+ "serial_test_derive",+]++[[package]]+name = "serial_test_derive"+version = "3.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "sha1"version = "0.10.6"diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.tomlindex 5f0a6ae..233540c 100644--- a/crates/liteparse/Cargo.toml+++ b/crates/liteparse/Cargo.toml@@ -34,3 +34,6 @@ web-time = "1.1.0"[target.'cfg(not(target_arch = "wasm32"))'.dependencies]tokio = { version = "1.52.3", features = ["rt-multi-thread", "fs", "process"] }++[dev-dependencies]+serial_test = "3"diff --git a/crates/liteparse/src/conversion.rs b/crates/liteparse/src/conversion.rsindex b2f6f73..3e4f3c1 100644--- a/crates/liteparse/src/conversion.rs+++ b/crates/liteparse/src/conversion.rs@@ -1,3 +1,5 @@+use tempfile::TempDir;+use crate::error::LiteParseError;use std::{fmt::{self, Display},@@ -82,7 +84,8 @@ pub fn is_supported_extension(path: &str) -> bool {pub async fn convert_to_pdf(path: &str,password: Option<&str>,-) -> Result<ConversionResult, LiteParseError> {+ is_temporary_path: bool,+) -> Result<(ConversionResult, Option<TempDir>), LiteParseError> {let ext = Path::new(path).extension().and_then(|e| e.to_str())@@ -90,10 +93,13 @@ pub async fn convert_to_pdf(.unwrap_or_default();if ext == "pdf" {- return Ok(ConversionResult {- pdf_path: path.to_string(),- original_extension: ext,- });+ return Ok((+ ConversionResult {+ pdf_path: path.to_string(),+ original_extension: ext,+ },+ None,+ ));}let tool = if OFFICE_EXTENSIONS.contains(&ext.as_str())@@ -119,11 +125,18 @@ pub async fn convert_to_pdf(convert_office_document(path, tmp_dir.path().to_str().unwrap(), password).await?}};+ // Remove temporary path for Office docs/images bytes+ if is_temporary_path {+ tokio::fs::remove_dir_all(Path::new(path).parent().unwrap()).await?;+ }- Ok(ConversionResult {- pdf_path,- original_extension: ext,- })+ Ok((+ ConversionResult {+ pdf_path,+ original_extension: ext,+ },+ Some(tmp_dir),+ ))}/// Execute command with timeout@@ -340,9 +353,20 @@ pub async fn convert_office_document(On Windows: choco install libreoffice-fresh".into())})?;-+ // LibreOffice serializes on a per-user profile lock. Concurrent invocations+ // sharing the same profile race for the lock: the loser either silently+ // exits with status 0 producing no output, or crashes on shared state.+ // Give every invocation its own throwaway UserInstallation profile.+ let user_profile_dir = tempfile::Builder::new()+ .prefix("liteparse-lo-profile-")+ .tempdir()?;+ let user_profile_url = format!(+ "-env:UserInstallation=file://{}",+ user_profile_dir.path().to_string_lossy()+ );let infilter_arg;let mut args: Vec<&str> = vec![+ &user_profile_url,"--headless","--invisible","--convert-to",@@ -458,14 +482,14 @@ pub fn guess_extension_from_data(data: &[u8]) -> Option<String> {pub async fn convert_data_to_pdf(data: Vec<u8>,password: Option<&str>,-) -> Result<ConversionResult, LiteParseError> {+) -> Result<(ConversionResult, Option<TempDir>), LiteParseError> {let ext = guess_extension_from_data(&data);let tmp_dir = tempfile::Builder::new().prefix("liteparse-").tempdir()?;let tmp_path = tmp_dir- .path()+ .keep().join(format!("input.{}", ext.unwrap_or("bin".to_string())));tokio::fs::write(&tmp_path, data).await?;- convert_to_pdf(tmp_path.to_str().unwrap(), password).await+ convert_to_pdf(tmp_path.to_str().unwrap(), password, true).await}#[cfg(test)]@@ -572,14 +596,14 @@ mod tests {#[tokio::test]async fn test_convert_to_pdf_passthrough_pdf() {- let res = convert_to_pdf("/some/file.pdf", None).await.unwrap();+ let (res, _) = convert_to_pdf("/some/file.pdf", None, false).await.unwrap();assert_eq!(res.pdf_path, "/some/file.pdf");assert_eq!(res.original_extension, "pdf");}#[tokio::test]async fn test_convert_to_pdf_unsupported() {- let r = convert_to_pdf("/some/file.xyz", None).await;+ let r = convert_to_pdf("/some/file.xyz", None, false).await;assert!(r.is_err());assert!(r.unwrap_err().to_string().contains("unsupported"));}diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex dadc62a..3495db1 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -1,6 +1,7 @@use crate::config::{LiteParseConfig, parse_target_pages};#[cfg(not(target_arch = "wasm32"))]use crate::conversion;+use crate::conversion::convert_data_to_pdf;use crate::error::LiteParseError;use crate::extract;use crate::ocr::OcrEngine;@@ -55,11 +56,13 @@ impl LiteParse {#[cfg(not(target_arch = "wasm32"))]pub async fn parse(&self, input: &str) -> Result<ParseResult, LiteParseError> {// Resolve file path to a PdfInput (convert non-PDFs first)+ let _tmp_dir;let pdf_input = if conversion::is_pdf(input) {PdfInput::Path(input.to_string())} else {- let converted =- conversion::convert_to_pdf(input, self.config.password.as_deref()).await?;+ let (converted, tmp_dir) =+ conversion::convert_to_pdf(input, self.config.password.as_deref(), false).await?;+ _tmp_dir = tmp_dir;PdfInput::Path(converted.pdf_path)};@@ -79,6 +82,18 @@ impl LiteParse {let t0 = web_time::Instant::now();+ let _tmp_dir;++ let validated_input = match input {+ PdfInput::Path(p) => PdfInput::Path(p),+ PdfInput::Bytes(b) => {+ let (converted, tmp_dir) =+ convert_data_to_pdf(b, self.config.password.as_deref()).await?;+ _tmp_dir = tmp_dir;+ PdfInput::Path(converted.pdf_path)+ }+ };+// Determine which pages to extractlet target_pages = self.config@@ -90,7 +105,7 @@ impl LiteParse {// Extract text items from PDF pageslet mut pages = extract::extract_pages_from_input(- &input,+ &validated_input,target_pages.as_deref(),self.config.max_pages,self.config.password.as_deref(),@@ -135,7 +150,7 @@ impl LiteParse {};ocr_merge::ocr_and_merge_pages_from_input(&mut pages,- &input,+ &validated_input,self.config.dpi,engine,&self.config.ocr_language,diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsnew file mode 100644index 0000000..d6a0bb4--- /dev/null+++ b/crates/liteparse/tests/integration_test.rs@@ -0,0 +1,100 @@+use std::path::Path;++use liteparse::conversion::convert_data_to_pdf;+use liteparse::types::PdfInput;+use liteparse::{LiteParse, LiteParseConfig};+use serial_test::serial;++#[tokio::test]+#[serial]+async fn test_convert_data_to_pdf_integration() {+ let fixture_path = "../../integration_tests_data/receipt.png";+ let data = tokio::fs::read(fixture_path)+ .await+ .expect("Should be able to read file");+ let (converted, _tmp_dir) = convert_data_to_pdf(data, None)+ .await+ .expect("Should be able to convert data to PDF");+ assert!(Path::new(&converted.pdf_path).exists());+}++#[tokio::test]+#[serial]+async fn test_parse_bytes_image_integration() {+ let fixture_path = "../../integration_tests_data/receipt.png";+ let lit = LiteParse::new(LiteParseConfig::default());+ let data = tokio::fs::read(fixture_path)+ .await+ .expect("Should be able to read file");+ let input = PdfInput::Bytes(data);+ let parsed = lit+ .parse_input(input)+ .await+ .expect("Should be able to parse");+ assert_eq!(parsed.pages.len(), 1);+}++#[tokio::test]+#[serial]+async fn test_parse_bytes_office_integration() {+ let fixture_path = "../../integration_tests_data/sample3.doc";+ let lit = LiteParse::new(LiteParseConfig::default());+ let data = tokio::fs::read(fixture_path)+ .await+ .expect("Should be able to read file");+ let input = PdfInput::Bytes(data);+ let parsed = lit+ .parse_input(input)+ .await+ .expect("Should be able to parse");+ assert_eq!(parsed.pages.len(), 2);+}++#[tokio::test]+#[serial]+async fn test_parse_image_integration() {+ let lit = LiteParse::new(LiteParseConfig::default());+ let parsed = lit+ .parse("../../integration_tests_data/receipt.png")+ .await+ .expect("Should be able to parse");+ assert_eq!(parsed.pages.len(), 1);+}++#[tokio::test]+#[serial]+async fn test_parse_office_doc_integration() {+ let lit = LiteParse::new(LiteParseConfig::default());+ let parsed = lit+ .parse("../../integration_tests_data/sample3.doc")+ .await+ .expect("Should be able to parse");+ assert_eq!(parsed.pages.len(), 2);+}++#[tokio::test]+#[serial]+async fn test_parse_pdf_integration() {+ let lit = LiteParse::new(LiteParseConfig::default());+ let parsed = lit+ .parse("../../integration_tests_data/sample.pdf")+ .await+ .expect("Should be able to parse");+ assert_eq!(parsed.pages.len(), 1);+}++#[tokio::test]+#[serial]+async fn test_parse_bytes_pdf_integration() {+ let fixture_path = "../../integration_tests_data/sample.pdf";+ let lit = LiteParse::new(LiteParseConfig::default());+ let data = tokio::fs::read(fixture_path)+ .await+ .expect("Should be able to read file");+ let input = PdfInput::Bytes(data);+ let parsed = lit+ .parse_input(input)+ .await+ .expect("Should be able to parse");+ assert_eq!(parsed.pages.len(), 1);+}diff --git a/integration_tests_data/receipt.png b/integration_tests_data/receipt.pngnew file mode 100644index 0000000..426800fBinary files /dev/null and b/integration_tests_data/receipt.png differdiff --git a/integration_tests_data/sample.pdf b/integration_tests_data/sample.pdfnew file mode 100644index 0000000..c01805eBinary files /dev/null and b/integration_tests_data/sample.pdf differdiff --git a/integration_tests_data/sample3.doc b/integration_tests_data/sample3.docnew file mode 100644index 0000000..1bac61dBinary files /dev/null and b/integration_tests_data/sample3.doc differ