A fast, helpful, and open-source document parser
feat: finish imagemagick and libreoffice
Clelia (Astra) Bertelli0d0263e19272
diff --git a/.gitignore b/.gitignoreindex 75358a4..4f90917 100644--- a/.gitignore+++ b/.gitignore@@ -73,4 +73,5 @@ docs/src/content/docs/liteparse/.api-tmp/# Rustpackages/rust/target/-packages/rust/vendor/\ No newline at end of file+packages/rust/vendor/+target/diff --git a/Cargo.lock b/Cargo.lockindex 25430db..7975755 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -216,6 +216,17 @@ dependencies = ["nom",]+[[package]]+name = "cfb"+version = "0.7.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"+dependencies = [+ "byteorder",+ "fnv",+ "uuid",+]+[[package]]name = "cfg-if"version = "1.0.4"@@ -938,6 +949,15 @@ dependencies = ["serde_core",]+[[package]]+name = "infer"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7"+dependencies = [+ "cfb",+]+[[package]]name = "inout"version = "0.1.4"@@ -1004,9 +1024,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"[[package]]name = "libc"-version = "0.2.183"+version = "0.2.186"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"[[package]]name = "libloading"@@ -1027,7 +1047,7 @@ dependencies = ["bitflags","libc","plain",- "redox_syscall",+ "redox_syscall 0.7.5",][[package]]@@ -1046,13 +1066,26 @@ checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"name = "liteparse-rs"version = "0.1.0"dependencies = [+ "anyhow","clap","image",+ "infer","pdfium","pdfium-sys","serde","serde_json",+ "tempfile","tesseract-rs",+ "tokio",+]++[[package]]+name = "lock_api"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"+dependencies = [+ "scopeguard",][[package]]@@ -1228,6 +1261,29 @@ dependencies = ["vcpkg",]+[[package]]+name = "parking_lot"+version = "0.12.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"+dependencies = [+ "lock_api",+ "parking_lot_core",+]++[[package]]+name = "parking_lot_core"+version = "0.9.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"+dependencies = [+ "cfg-if",+ "libc",+ "redox_syscall 0.5.18",+ "smallvec",+ "windows-link",+]+[[package]]name = "pbkdf2"version = "0.12.2"@@ -1353,6 +1409,15 @@ version = "6.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"+[[package]]+name = "redox_syscall"+version = "0.5.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"+dependencies = [+ "bitflags",+]+[[package]]name = "redox_syscall"version = "0.7.5"@@ -1522,6 +1587,12 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "scopeguard"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"+[[package]]name = "security-framework"version = "3.7.0"@@ -1623,6 +1694,16 @@ version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"+[[package]]+name = "signal-hook-registry"+version = "1.4.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"+dependencies = [+ "errno",+ "libc",+]+[[package]]name = "simd-adler32"version = "0.3.9"@@ -1838,11 +1919,25 @@ dependencies = ["bytes","libc","mio",+ "parking_lot","pin-project-lite",+ "signal-hook-registry","socket2",+ "tokio-macros","windows-sys 0.61.2",]+[[package]]+name = "tokio-macros"+version = "2.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "tokio-native-tls"version = "0.3.1"@@ -2029,6 +2124,16 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"+[[package]]+name = "uuid"+version = "1.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]+[[package]]name = "vcpkg"version = "0.2.15"diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.tomlindex 4923620..861d1f7 100644--- a/crates/liteparse/Cargo.toml+++ b/crates/liteparse/Cargo.toml@@ -8,10 +8,14 @@ name = "lit"path = "src/main.rs"[dependencies]+anyhow = "1.0.102"clap = { version = "4.5.55", features = ["derive"] }image = { version = "0.25", default-features = false, features = ["png"] }+infer = "0.19.0"pdfium = { path = "../pdfium" }pdfium-sys = { path = "../pdfium-sys" }serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.149"+tempfile = "3.27.0"tesseract-rs = { version = "0.2", features = ["build-tesseract"] }+tokio = { version = "1.52.3", features = ["full"] }diff --git a/crates/liteparse/src/conversion.rs b/crates/liteparse/src/conversion.rsindex 9d953dc..70d784a 100644--- a/crates/liteparse/src/conversion.rs+++ b/crates/liteparse/src/conversion.rs@@ -1,4 +1,7 @@-use std::path::Path;+use std::{+ fmt::{self, Display},+ path::Path,+};/// Supported file extensions for conversion (non-PDF formats).const OFFICE_EXTENSIONS: &[&str] = &[@@ -14,6 +17,38 @@ const IMAGE_EXTENSIONS: &[&str] = &["jpg", "jpeg", "png", "gif", "bmp", "tiff", "tif", "webp", "svg",];+/// Extensions that require Ghostscript for ImageMagick conversion.+const GHOSTSCRIPT_REQUIRED_EXTENSIONS: &[&str] = &["svg", "eps", "ps", "ai"];++/// A resolved external command with its executable path and any required prefix args.+#[derive(Debug, Clone)]+pub struct ResolvedCommand {+ pub command: String,+ pub args: Vec<String>,+ pub resolved_path: String,+}++#[derive(Debug, Clone)]+pub struct ConversionResult {+ pub pdf_path: String,+ pub original_extension: String,+}++enum ConverstionTool {+ LibreOffice,+ ImageMagick,+}++impl Display for ConverstionTool {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ let s = match self {+ Self::ImageMagick => "ImageMagick",+ Self::LibreOffice => "LibreOffice",+ };+ write!(f, "{}", s)+ }+}+/// Check if a file is a PDF (no conversion needed).pub fn is_pdf(path: &str) -> bool {Path::new(path)@@ -43,10 +78,10 @@ pub fn is_supported_extension(path: &str) -> bool {////// Currently stubbed out — returns an error directing users to install/// LibreOffice (for office documents) or ImageMagick (for images).-pub fn convert_to_pdf(+pub async fn convert_to_pdf(path: &str,- _password: Option<&str>,-) -> Result<String, Box<dyn std::error::Error>> {+ password: Option<&str>,+) -> Result<ConversionResult, Box<dyn std::error::Error>> {let ext = Path::new(path).extension().and_then(|e| e.to_str())@@ -54,23 +89,376 @@ pub fn convert_to_pdf(.unwrap_or_default();if ext == "pdf" {- return Ok(path.to_string());+ return Ok(ConversionResult {+ pdf_path: path.to_string(),+ original_extension: ext,+ });}let tool = if OFFICE_EXTENSIONS.contains(&ext.as_str())|| PRESENTATION_EXTENSIONS.contains(&ext.as_str())|| SPREADSHEET_EXTENSIONS.contains(&ext.as_str()){- "LibreOffice"+ ConverstionTool::LibreOffice} else if IMAGE_EXTENSIONS.contains(&ext.as_str()) {- "ImageMagick"+ ConverstionTool::ImageMagick} else {return Err(format!("unsupported file format: .{}", ext).into());};- Err(format!(- "conversion from .{ext} to PDF requires {tool}, which is not yet supported in the Rust port. \- Convert the file to PDF manually, or use the TypeScript version of LiteParse."+ let tmp_dir = tempfile::Builder::new().prefix("liteparse-").tempdir()?;+ let pdf_path = match tool {+ ConverstionTool::ImageMagick => {+ convert_image_to_pdf(path, tmp_dir.path().to_str().unwrap()).await?+ }+ ConverstionTool::LibreOffice => {+ convert_office_document(path, tmp_dir.path().to_str().unwrap(), password).await?+ }+ };++ Ok(ConversionResult {+ pdf_path,+ original_extension: ext,+ })+}++/// Execute command with timeout+pub async fn execute_command(+ command: &str,+ args: Vec<&str>,+ timeout_ms: u64,+) -> Result<String, Box<dyn std::error::Error>> {+ let proc = tokio::process::Command::new(command)+ .args(args)+ .stdin(std::process::Stdio::null())+ .stdout(std::process::Stdio::piped())+ .stderr(std::process::Stdio::piped())+ .kill_on_drop(true)+ .spawn()?;+ match tokio::time::timeout(+ tokio::time::Duration::from_millis(timeout_ms),+ proc.wait_with_output(),+ )+ .await+ {+ Ok(Ok(output)) => {+ let stdout = String::from_utf8_lossy(&output.stdout).to_string();+ let stderr = String::from_utf8_lossy(&output.stderr).to_string();+ if output.status.success() {+ Ok(stdout)+ } else {+ Err(anyhow::anyhow!("Command failed: {stderr}").into())+ }+ }+ Ok(Err(e)) => Err(anyhow::anyhow!("Command error: {e}").into()),+ Err(_) => Err(anyhow::anyhow!("Command timed out after {timeout_ms}ms").into()),+ }+}++/// Execute a command for PowerShel+pub async fn execute_powershell(+ command: &str,+ timeout_ms: u64,+) -> Result<String, Box<dyn std::error::Error>> {+ execute_command(+ "powershell",+ vec!["-NoProfile", "-Command", command],+ timeout_ms,)- .into())+ .await+}++fn get_resolved_path_from_output(output: &str, use_last_line: bool) -> Option<String> {+ let lines: Vec<String> = output+ .lines()+ .map(|l| l.trim().to_string())+ .filter(|l| !l.is_empty())+ .collect();+ if lines.is_empty() {+ return None;+ }+ let l = if use_last_line {+ lines.last()+ } else {+ lines.first()+ };++ l.cloned()+}++/// Resolve the actual executable path for a command.+pub async fn resolve_command_path(command: &str) -> Option<String> {+ if std::env::consts::FAMILY == "windows" {+ let ps = format!("(Get-Command '{command}' -ErrorAction Stop).Source");+ match execute_powershell(&ps, 5000).await {+ Ok(out) => get_resolved_path_from_output(&out, true),+ Err(_) => None,+ }+ } else {+ match execute_command("which", vec![command], 5000).await {+ Ok(out) => get_resolved_path_from_output(&out, false),+ Err(_) => None,+ }+ }+}++/// Check if a command is available on Unix-like platforms (via `which`).+pub async fn is_command_available(command: &str) -> bool {+ execute_command("which", vec![command], 5000).await.is_ok()+}++/// Check if a command is available on Windows (via PowerShell `Get-Command`).+pub async fn is_command_available_windows(command: &str) -> bool {+ execute_powershell(&format!("Get-Command {command}"), 5000)+ .await+ .is_ok()+}++/// Check if a file path exists and is executable.+pub async fn is_path_executable(file_path: &str) -> bool {+ let p = std::path::PathBuf::from(file_path);+ match tokio::fs::metadata(&p).await {+ Ok(meta) => {+ if !meta.is_file() {+ return false;+ }+ if std::env::consts::FAMILY == "unix" {+ use std::os::unix::fs::PermissionsExt;+ meta.permissions().mode() & 0o111 != 0+ } else {+ true+ }+ }+ Err(_) => false,+ }+}++/// Detect whether a resolved path points at the built-in Windows+/// `System32\convert.exe` (which is unrelated to ImageMagick).+fn is_windows_system_convert(file_path: &str) -> bool {+ let normalized = file_path.replace('/', "\\").to_lowercase();+ let system_root = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string());+ let system32_convert = format!("{system_root}\\System32\\convert.exe")+ .replace('/', "\\")+ .to_lowercase();+ normalized == system32_convert+}++/// Verify an executable identifies itself as ImageMagick via `-version`.+async fn is_image_magick_binary(executable_path: &str, args: &[&str]) -> bool {+ let mut full_args: Vec<&str> = args.to_vec();+ full_args.push("-version");+ match execute_command(executable_path, full_args, 5000).await {+ Ok(out) => out.to_lowercase().contains("imagemagick"),+ Err(_) => false,+ }+}++async fn resolve_image_magick_command(command: &str) -> Option<ResolvedCommand> {+ let resolved_path = resolve_command_path(command).await?;++ if command == "convert"+ && std::env::consts::FAMILY == "windows"+ && is_windows_system_convert(&resolved_path)+ {+ return None;+ }++ if !is_image_magick_binary(&resolved_path, &[]).await {+ return None;+ }++ Some(ResolvedCommand {+ command: resolved_path.clone(),+ args: Vec::new(),+ resolved_path,+ })+}++/// Find LibreOffice command - handles different installation methods.+pub async fn find_libre_office_command() -> Option<String> {+ if is_command_available("libreoffice").await+ || is_command_available_windows("libreoffice").await+ {+ return Some("libreoffice".to_string());+ }++ if is_command_available("soffice").await || is_command_available_windows("soffice").await {+ return Some("soffice".to_string());+ }++ let mac_os_paths = [+ "/Applications/LibreOffice.app/Contents/MacOS/soffice",+ "/Applications/LibreOffice.app/Contents/MacOS/libreoffice",+ ];++ let windows_paths = ["C:\\Program Files\\Libreoffice\\program\\soffice.exe"];++ for lib_path in mac_os_paths.iter() {+ if is_path_executable(lib_path).await {+ return Some(lib_path.to_string());+ }+ }++ for lib_path in windows_paths.iter() {+ if is_path_executable(lib_path).await {+ return Some(lib_path.to_string());+ }+ }++ None+}++/// Find ImageMagick command - handles v6 (`convert`) and v7 (`magick`).+pub async fn find_image_magick_command() -> Option<ResolvedCommand> {+ if let Some(cmd) = resolve_image_magick_command("magick").await {+ return Some(cmd);+ }+ resolve_image_magick_command("convert").await+}++/// Convert office documents using LibreOffice.+pub async fn convert_office_document(+ file_path: &str,+ output_dir: &str,+ password: Option<&str>,+) -> Result<String, Box<dyn std::error::Error>> {+ let libre_office_cmd = find_libre_office_command().await.ok_or_else(|| {+ anyhow::anyhow!(+ "LibreOffice is not installed. Please install LibreOffice to convert office documents. \+ On macOS: brew install --cask libreoffice, On Ubuntu: apt-get install libreoffice, \+ On Windows: choco install libreoffice-fresh"+ )+ })?;++ let infilter_arg;+ let mut args: Vec<&str> = vec![+ "--headless",+ "--invisible",+ "--convert-to",+ "pdf",+ "--outdir",+ output_dir,+ ];+ if let Some(pw) = password {+ infilter_arg = format!("--infilter=:{pw}");+ args.push(&infilter_arg);+ }+ args.push(file_path);++ execute_command(&libre_office_cmd, args, 120_000).await?;++ let base_name = Path::new(file_path)+ .file_stem()+ .and_then(|s| s.to_str())+ .ok_or_else(|| anyhow::anyhow!("invalid file path: {file_path}"))?;+ let pdf_path = Path::new(output_dir)+ .join(format!("{base_name}.pdf"))+ .to_string_lossy()+ .to_string();++ if tokio::fs::metadata(&pdf_path).await.is_ok() {+ Ok(pdf_path)+ } else {+ Err(anyhow::anyhow!("LibreOffice conversion succeeded but output PDF not found").into())+ }+}++/// Convert images to PDF using ImageMagick.+pub async fn convert_image_to_pdf(+ file_path: &str,+ output_dir: &str,+) -> Result<String, Box<dyn std::error::Error>> {+ let image_magick = find_image_magick_command().await.ok_or_else(|| {+ anyhow::anyhow!(+ "ImageMagick is not installed. Please install ImageMagick to convert images. \+ On macOS: brew install imagemagick, On Ubuntu: apt-get install imagemagick, \+ On Windows: choco install imagemagick.app"+ )+ })?;++ let ext = Path::new(file_path)+ .extension()+ .and_then(|e| e.to_str())+ .map(|e| e.to_lowercase())+ .unwrap_or_default();+ let needs_ghostscript = GHOSTSCRIPT_REQUIRED_EXTENSIONS.contains(&ext.as_str());++ if needs_ghostscript {+ let has_ghostscript =+ is_command_available("gs").await || is_command_available_windows("gs").await;+ if !has_ghostscript {+ return Err(anyhow::anyhow!(+ "Ghostscript is required to convert {} files but is not installed. \+ On macOS: brew install ghostscript, On Ubuntu: apt-get install ghostscript, \+ On Windows: choco install ghostscript",+ ext.to_uppercase()+ )+ .into());+ }+ }++ let base_name = Path::new(file_path)+ .file_stem()+ .and_then(|s| s.to_str())+ .ok_or_else(|| anyhow::anyhow!("invalid file path: {file_path}"))?;+ let pdf_path = Path::new(output_dir)+ .join(format!("{base_name}.pdf"))+ .to_string_lossy()+ .to_string();++ let mut args: Vec<&str> = image_magick.args.iter().map(|s| s.as_str()).collect();+ args.push(file_path);+ args.push("-density");+ args.push("150");+ args.push("-units");+ args.push("PixelsPerInch");+ args.push(&pdf_path);++ match execute_command(&image_magick.command, args, 60_000).await {+ Ok(_) => Ok(pdf_path),+ Err(error) => {+ let error_msg = error.to_string();+ if error_msg.contains("gs") && error_msg.contains("command not found") {+ return Err(anyhow::anyhow!(+ "Ghostscript is required to convert {} files but is not installed. \+ On macOS: brew install ghostscript, On Ubuntu: apt-get install ghostscript, \+ On Windows: choco install ghostscript",+ ext.to_uppercase()+ )+ .into());+ }+ if error_msg.contains("FailedToExecuteCommand") && error_msg.contains("gs") {+ return Err(anyhow::anyhow!(+ "Ghostscript failed during {} conversion. \+ Ensure Ghostscript is properly installed: brew install ghostscript",+ ext.to_uppercase()+ )+ .into());+ }+ Err(error)+ }+ }+}++pub fn guess_extension_from_data(data: &[u8]) -> Option<String> {+ let kind = infer::get(data);+ let ext = kind.map(|k| k.extension());+ if let Some(e) = ext {+ return Some(e.to_string());+ }+ None+}++pub async fn convert_data_to_pdf(+ data: Vec<u8>,+ password: Option<&str>,+) -> Result<ConversionResult, Box<dyn std::error::Error>> {+ let ext = guess_extension_from_data(&data);+ let tmp_dir = tempfile::Builder::new().prefix("liteparse-").tempdir()?;+ let tmp_path = tmp_dir+ .path()+ .join(format!("input.{}", ext.unwrap_or("bin".to_string())));+ tokio::fs::write(&tmp_path, data).await?;+ convert_to_pdf(tmp_path.to_str().unwrap(), password).await}diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rsindex 2dd2602..7c353ff 100644--- a/crates/liteparse/src/main.rs+++ b/crates/liteparse/src/main.rs@@ -174,7 +174,8 @@ fn parse_output_format(s: &str) -> Result<OutputFormat, String> {}}-fn main() -> Result<(), Box<dyn std::error::Error>> {+#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {let cli = Cli::parse();match cli.command {@@ -195,7 +196,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {};let lp = LiteParse::new(config);- let result = lp.parse(&cmd.file)?;+ let result = lp.parse(&cmd.file).await?;let formatted = lp.format(&result)?;match cmd.output {@@ -306,7 +307,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {std::fs::create_dir_all(parent)?;}- match lp.parse(file_path) {+ match lp.parse(file_path).await {Ok(result) => match lp.format(&result) {Ok(formatted) => {std::fs::write(&out_path, &formatted)?;diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 455e507..f69070b 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -26,7 +26,7 @@ impl LiteParse {}/// Parse a document file, returning structured results.- pub fn parse(&self, input: &str) -> Result<ParseResult, Box<dyn std::error::Error>> {+ pub async fn parse(&self, input: &str) -> Result<ParseResult, Box<dyn std::error::Error>> {let log = |msg: &str| {if !self.config.quiet {eprintln!("{}", msg);@@ -37,7 +37,9 @@ impl LiteParse {let pdf_path = if conversion::is_pdf(input) {input.to_string()} else {- conversion::convert_to_pdf(input, self.config.password.as_deref())?+ conversion::convert_to_pdf(input, self.config.password.as_deref())+ .await?+ .pdf_path};let t0 = std::time::Instant::now();