Skip to content

A fast, helpful, and open-source document parser

fix: fix failures in parsing images and office docs, add conversion for bytes to pdf when parsing and add integration tests to verify that everything works

Clelia (Astra) Bertellifefd28725a3b
diff --git a/.gitignore b/.gitignore
index 4f90917..a620ebc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,6 +42,9 @@ tmp/
temp/
*.txt
*.png
+!integration_tests_data/receipt.png
+!integration_tests_data/sample3.docx.doc
+!integration_tests_data/sample.pdf
deno.json
# Python files
diff --git a/Cargo.lock b/Cargo.lock
index 97e6cf5..8202db0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -642,6 +642,17 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+[[package]]
+name = "futures-executor"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-io"
version = "0.3.32"
@@ -1204,6 +1215,7 @@ dependencies = [
"reqwest 0.13.3",
"serde",
"serde_json",
+ "serial_test",
"tempfile",
"tesseract-rs",
"thiserror 2.0.18",
@@ -2090,6 +2102,15 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "scc"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc"
+dependencies = [
+ "sdd",
+]
+
[[package]]
name = "schannel"
version = "0.1.29"
@@ -2105,6 +2126,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "sdd"
+version = "3.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"
+
[[package]]
name = "security-framework"
version = "3.7.0"
@@ -2200,6 +2227,32 @@ dependencies = [
"serde",
]
+[[package]]
+name = "serial_test"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f"
+dependencies = [
+ "futures-executor",
+ "futures-util",
+ "log",
+ "once_cell",
+ "parking_lot",
+ "scc",
+ "serial_test_derive",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "sha1"
version = "0.10.6"
diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.toml
index 5f0a6ae..233540c 100644
--- a/crates/liteparse/Cargo.toml
+++ b/crates/liteparse/Cargo.toml
@@ -34,3 +34,6 @@ web-time = "1.1.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
tokio = { version = "1.52.3", features = ["rt-multi-thread", "fs", "process"] }
+
+[dev-dependencies]
+serial_test = "3"
diff --git a/crates/liteparse/src/conversion.rs b/crates/liteparse/src/conversion.rs
index b2f6f73..3e4f3c1 100644
--- a/crates/liteparse/src/conversion.rs
+++ b/crates/liteparse/src/conversion.rs
@@ -1,3 +1,5 @@
+use tempfile::TempDir;
+
use crate::error::LiteParseError;
use std::{
fmt::{self, Display},
@@ -82,7 +84,8 @@ pub fn is_supported_extension(path: &str) -> bool {
pub async fn convert_to_pdf(
path: &str,
password: Option<&str>,
-) -> Result<ConversionResult, LiteParseError> {
+ is_temporary_path: bool,
+) -> Result<(ConversionResult, Option<TempDir>), LiteParseError> {
let ext = Path::new(path)
.extension()
.and_then(|e| e.to_str())
@@ -90,10 +93,13 @@ pub async fn convert_to_pdf(
.unwrap_or_default();
if ext == "pdf" {
- return Ok(ConversionResult {
- pdf_path: path.to_string(),
- original_extension: ext,
- });
+ return Ok((
+ ConversionResult {
+ pdf_path: path.to_string(),
+ original_extension: ext,
+ },
+ None,
+ ));
}
let tool = if OFFICE_EXTENSIONS.contains(&ext.as_str())
@@ -119,11 +125,18 @@ pub async fn convert_to_pdf(
convert_office_document(path, tmp_dir.path().to_str().unwrap(), password).await?
}
};
+ // Remove temporary path for Office docs/images bytes
+ if is_temporary_path {
+ tokio::fs::remove_dir_all(Path::new(path).parent().unwrap()).await?;
+ }
- Ok(ConversionResult {
- pdf_path,
- original_extension: ext,
- })
+ Ok((
+ ConversionResult {
+ pdf_path,
+ original_extension: ext,
+ },
+ Some(tmp_dir),
+ ))
}
/// Execute command with timeout
@@ -340,9 +353,20 @@ pub async fn convert_office_document(
On Windows: choco install libreoffice-fresh".into()
)
})?;
-
+ // LibreOffice serializes on a per-user profile lock. Concurrent invocations
+ // sharing the same profile race for the lock: the loser either silently
+ // exits with status 0 producing no output, or crashes on shared state.
+ // Give every invocation its own throwaway UserInstallation profile.
+ let user_profile_dir = tempfile::Builder::new()
+ .prefix("liteparse-lo-profile-")
+ .tempdir()?;
+ let user_profile_url = format!(
+ "-env:UserInstallation=file://{}",
+ user_profile_dir.path().to_string_lossy()
+ );
let infilter_arg;
let mut args: Vec<&str> = vec![
+ &user_profile_url,
"--headless",
"--invisible",
"--convert-to",
@@ -458,14 +482,14 @@ pub fn guess_extension_from_data(data: &[u8]) -> Option<String> {
pub async fn convert_data_to_pdf(
data: Vec<u8>,
password: Option<&str>,
-) -> Result<ConversionResult, LiteParseError> {
+) -> Result<(ConversionResult, Option<TempDir>), LiteParseError> {
let ext = guess_extension_from_data(&data);
let tmp_dir = tempfile::Builder::new().prefix("liteparse-").tempdir()?;
let tmp_path = tmp_dir
- .path()
+ .keep()
.join(format!("input.{}", ext.unwrap_or("bin".to_string())));
tokio::fs::write(&tmp_path, data).await?;
- convert_to_pdf(tmp_path.to_str().unwrap(), password).await
+ convert_to_pdf(tmp_path.to_str().unwrap(), password, true).await
}
#[cfg(test)]
@@ -572,14 +596,14 @@ mod tests {
#[tokio::test]
async fn test_convert_to_pdf_passthrough_pdf() {
- let res = convert_to_pdf("/some/file.pdf", None).await.unwrap();
+ let (res, _) = convert_to_pdf("/some/file.pdf", None, false).await.unwrap();
assert_eq!(res.pdf_path, "/some/file.pdf");
assert_eq!(res.original_extension, "pdf");
}
#[tokio::test]
async fn test_convert_to_pdf_unsupported() {
- let r = convert_to_pdf("/some/file.xyz", None).await;
+ let r = convert_to_pdf("/some/file.xyz", None, false).await;
assert!(r.is_err());
assert!(r.unwrap_err().to_string().contains("unsupported"));
}
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index dadc62a..3495db1 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -1,6 +1,7 @@
use crate::config::{LiteParseConfig, parse_target_pages};
#[cfg(not(target_arch = "wasm32"))]
use crate::conversion;
+use crate::conversion::convert_data_to_pdf;
use crate::error::LiteParseError;
use crate::extract;
use crate::ocr::OcrEngine;
@@ -55,11 +56,13 @@ impl LiteParse {
#[cfg(not(target_arch = "wasm32"))]
pub async fn parse(&self, input: &str) -> Result<ParseResult, LiteParseError> {
// Resolve file path to a PdfInput (convert non-PDFs first)
+ let _tmp_dir;
let pdf_input = if conversion::is_pdf(input) {
PdfInput::Path(input.to_string())
} else {
- let converted =
- conversion::convert_to_pdf(input, self.config.password.as_deref()).await?;
+ let (converted, tmp_dir) =
+ conversion::convert_to_pdf(input, self.config.password.as_deref(), false).await?;
+ _tmp_dir = tmp_dir;
PdfInput::Path(converted.pdf_path)
};
@@ -79,6 +82,18 @@ impl LiteParse {
let t0 = web_time::Instant::now();
+ let _tmp_dir;
+
+ let validated_input = match input {
+ PdfInput::Path(p) => PdfInput::Path(p),
+ PdfInput::Bytes(b) => {
+ let (converted, tmp_dir) =
+ convert_data_to_pdf(b, self.config.password.as_deref()).await?;
+ _tmp_dir = tmp_dir;
+ PdfInput::Path(converted.pdf_path)
+ }
+ };
+
// Determine which pages to extract
let target_pages = self
.config
@@ -90,7 +105,7 @@ impl LiteParse {
// Extract text items from PDF pages
let mut pages = extract::extract_pages_from_input(
- &input,
+ &validated_input,
target_pages.as_deref(),
self.config.max_pages,
self.config.password.as_deref(),
@@ -135,7 +150,7 @@ impl LiteParse {
};
ocr_merge::ocr_and_merge_pages_from_input(
&mut pages,
- &input,
+ &validated_input,
self.config.dpi,
engine,
&self.config.ocr_language,
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
new file mode 100644
index 0000000..d6a0bb4
--- /dev/null
+++ b/crates/liteparse/tests/integration_test.rs
@@ -0,0 +1,100 @@
+use std::path::Path;
+
+use liteparse::conversion::convert_data_to_pdf;
+use liteparse::types::PdfInput;
+use liteparse::{LiteParse, LiteParseConfig};
+use serial_test::serial;
+
+#[tokio::test]
+#[serial]
+async fn test_convert_data_to_pdf_integration() {
+ let fixture_path = "../../integration_tests_data/receipt.png";
+ let data = tokio::fs::read(fixture_path)
+ .await
+ .expect("Should be able to read file");
+ let (converted, _tmp_dir) = convert_data_to_pdf(data, None)
+ .await
+ .expect("Should be able to convert data to PDF");
+ assert!(Path::new(&converted.pdf_path).exists());
+}
+
+#[tokio::test]
+#[serial]
+async fn test_parse_bytes_image_integration() {
+ let fixture_path = "../../integration_tests_data/receipt.png";
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let data = tokio::fs::read(fixture_path)
+ .await
+ .expect("Should be able to read file");
+ let input = PdfInput::Bytes(data);
+ let parsed = lit
+ .parse_input(input)
+ .await
+ .expect("Should be able to parse");
+ assert_eq!(parsed.pages.len(), 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_parse_bytes_office_integration() {
+ let fixture_path = "../../integration_tests_data/sample3.doc";
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let data = tokio::fs::read(fixture_path)
+ .await
+ .expect("Should be able to read file");
+ let input = PdfInput::Bytes(data);
+ let parsed = lit
+ .parse_input(input)
+ .await
+ .expect("Should be able to parse");
+ assert_eq!(parsed.pages.len(), 2);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_parse_image_integration() {
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let parsed = lit
+ .parse("../../integration_tests_data/receipt.png")
+ .await
+ .expect("Should be able to parse");
+ assert_eq!(parsed.pages.len(), 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_parse_office_doc_integration() {
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let parsed = lit
+ .parse("../../integration_tests_data/sample3.doc")
+ .await
+ .expect("Should be able to parse");
+ assert_eq!(parsed.pages.len(), 2);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_parse_pdf_integration() {
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let parsed = lit
+ .parse("../../integration_tests_data/sample.pdf")
+ .await
+ .expect("Should be able to parse");
+ assert_eq!(parsed.pages.len(), 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_parse_bytes_pdf_integration() {
+ let fixture_path = "../../integration_tests_data/sample.pdf";
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let data = tokio::fs::read(fixture_path)
+ .await
+ .expect("Should be able to read file");
+ let input = PdfInput::Bytes(data);
+ let parsed = lit
+ .parse_input(input)
+ .await
+ .expect("Should be able to parse");
+ assert_eq!(parsed.pages.len(), 1);
+}
diff --git a/integration_tests_data/receipt.png b/integration_tests_data/receipt.png
new file mode 100644
index 0000000..426800f
Binary files /dev/null and b/integration_tests_data/receipt.png differ
diff --git a/integration_tests_data/sample.pdf b/integration_tests_data/sample.pdf
new file mode 100644
index 0000000..c01805e
Binary files /dev/null and b/integration_tests_data/sample.pdf differ
diff --git a/integration_tests_data/sample3.doc b/integration_tests_data/sample3.doc
new file mode 100644
index 0000000..1bac61d
Binary files /dev/null and b/integration_tests_data/sample3.doc differ