A fast, helpful, and open-source document parser
wip
Logan Markewich7ac8a25b9548
diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rsindex 8fbb377..21dec20 100644--- a/crates/liteparse-python/src/cli.rs+++ b/crates/liteparse-python/src/cli.rs@@ -1,5 +1,5 @@use clap::{Args, Parser, Subcommand};-use liteparse::config::{LiteParseConfig, OutputFormat};+use liteparse::config::{ImageMode, LiteParseConfig, OutputFormat};use liteparse::conversion;use liteparse::output::{json, text};use liteparse::parser::LiteParse;@@ -54,6 +54,20 @@ struct ParseCommand {quiet: bool,#[arg(long)]num_workers: Option<usize>,+ /// How to surface raster images in markdown output: `off`, `placeholder`+ /// (default), or `embed` (extracts PNG bytes, written next to the output+ /// when `--image-output-dir` is set).+ #[arg(long, default_value = "placeholder")]+ image_mode: String,+ /// Directory to write embedded images to when `--image-mode embed` is set.+ /// Each image is written as `image_{id}.png` to match the markdown+ /// references. Created if missing.+ #[arg(long)]+ image_output_dir: Option<String>,+ /// Disable hyperlink extraction. By default URI link annotations render as+ /// `[text](url)` in markdown output; pass this to emit plain anchor text.+ #[arg(long)]+ no_links: bool,}#[derive(Args, Debug)]@@ -113,6 +127,18 @@ fn parse_output_format(s: &str) -> Result<OutputFormat, String> {}}+fn parse_image_mode(s: &str) -> Result<ImageMode, String> {+ match s.to_lowercase().as_str() {+ "off" | "none" => Ok(ImageMode::Off),+ "placeholder" => Ok(ImageMode::Placeholder),+ "embed" => Ok(ImageMode::Embed),+ _ => Err(format!(+ "unknown image-mode '{}', expected 'off', 'placeholder', or 'embed'",+ s+ )),+ }+}+/// Run the CLI with the given args (typically from sys.argv).pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {let cli = Cli::parse_from(args);@@ -121,6 +147,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {match cli.command {Commands::Parse(cmd) => {let format = parse_output_format(&cmd.format)?;+ let image_mode = parse_image_mode(&cmd.image_mode)?;let mut config = LiteParseConfig {ocr_language: cmd.ocr_language,ocr_enabled: !cmd.no_ocr,@@ -133,6 +160,8 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {password: cmd.password,quiet: cmd.quiet,ocr_server_url: cmd.ocr_server_url,+ image_mode,+ extract_links: !cmd.no_links,..Default::default()};if let Some(n) = cmd.num_workers {@@ -145,6 +174,22 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {OutputFormat::Text => text::format_text(&result.pages),OutputFormat::Markdown => result.text.clone(),};+ if let Some(dir) = cmd.image_output_dir.as_deref()+ && !result.images.is_empty()+ {+ std::fs::create_dir_all(dir)?;+ for img in &result.images {+ let path = format!("{}/image_{}.{}", dir, img.id, img.format);+ std::fs::write(&path, &img.bytes)?;+ }+ if !cmd.quiet {+ eprintln!(+ "[liteparse] wrote {} image(s) to {}",+ result.images.len(),+ dir+ );+ }+ }match cmd.output {Some(path) => {std::fs::write(&path, &formatted)?;diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rsindex 867d7e6..cfc9b57 100644--- a/crates/liteparse-wasm/src/lib.rs+++ b/crates/liteparse-wasm/src/lib.rs@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};use wasm_bindgen::prelude::*;use wasm_bindgen_futures::JsFuture;-use liteparse::config::{LiteParseConfig, OutputFormat};+use liteparse::config::{ImageMode, LiteParseConfig, OutputFormat};use liteparse::ocr::{OcrEngine, OcrOptions, OcrResult};use liteparse::parser::LiteParse as CoreLiteParse;use liteparse::search;@@ -45,6 +45,8 @@ struct JsLiteParseConfig {target_pages: Option<String>,dpi: Option<f32>,output_format: Option<String>,+ image_mode: Option<String>,+ extract_links: Option<bool>,preserve_very_small_text: Option<bool>,password: Option<String>,quiet: Option<bool>,@@ -87,6 +89,16 @@ impl JsLiteParseConfig {}};}+ if let Some(v) = self.image_mode {+ cfg.image_mode = match v.as_str() {+ "off" | "none" => ImageMode::Off,+ "embed" => ImageMode::Embed,+ _ => ImageMode::Placeholder,+ };+ }+ if let Some(v) = self.extract_links {+ cfg.extract_links = v;+ }if let Some(v) = self.preserve_very_small_text {cfg.preserve_very_small_text = v;}@@ -114,6 +126,12 @@ impl JsLiteParseConfig {OutputFormat::Text => "text".into(),OutputFormat::Markdown => "markdown".into(),}),+ image_mode: Some(match cfg.image_mode {+ ImageMode::Off => "off".into(),+ ImageMode::Placeholder => "placeholder".into(),+ ImageMode::Embed => "embed".into(),+ }),+ extract_links: Some(cfg.extract_links),preserve_very_small_text: Some(cfg.preserve_very_small_text),password: cfg.password.clone(),quiet: Some(cfg.quiet),diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex 2f4fd72..fb5ee35 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -1,10 +1,10 @@use crate::error::LiteParseError;use crate::glyph_names::resolve_glyph_name;-use crate::render::encode_png;use crate::types::{ExtractedImage, GraphicPrimitive, ImageRef, OutlineTarget, Page as LitePage, PdfInput, Rect,StructNode, TextItem,};+use image::ImageEncoder;use pdfium::{Document, Font, FontType, Library, Page, PathObject, PdfLink, RectF, SegmentKind, TextPage,};@@ -310,6 +310,16 @@ pub(crate) fn render_page_images(out}+/// Encode RGBA pixel bytes to PNG. Lives here (always-compiled) rather than in+/// `render` so the image-embed path is available on wasm, where the `render`+/// module (page rasterization / screenshots) is compiled out.+pub(crate) fn encode_png(rgba: &[u8], width: u32, height: u32) -> Result<Vec<u8>, LiteParseError> {+ let mut png_buf = Vec::new();+ let encoder = image::codecs::png::PngEncoder::new(&mut png_buf);+ encoder.write_image(rgba, width, height, image::ColorType::Rgba8.into())?;+ Ok(png_buf)+}+/// Walk image objects on a page and return a stable per-page `ImageRef` for/// each one. `obj_index` is the index among image-typed page objects (not all/// page objects), so a later embed pass can pull pixel bytes via@@ -967,7 +977,7 @@ fn median_f32(values: &[f32]) -> Option<f32> {let mut v: Vec<f32> = values.to_vec();v.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));let mid = v.len() / 2;- if v.len() % 2 == 0 {+ if v.len().is_multiple_of(2) {Some((v[mid - 1] + v[mid]) / 2.0)} else {Some(v[mid])diff --git a/crates/liteparse/src/glyph_names.rs b/crates/liteparse/src/glyph_names.rsindex cee16d3..1374a51 100644--- a/crates/liteparse/src/glyph_names.rs+++ b/crates/liteparse/src/glyph_names.rs@@ -49,24 +49,27 @@ fn resolve_component(comp: &str) -> Option<String> {}// uniXXXX (one or more groups of exactly 4 uppercase hex digits)- if let Some(hex) = comp.strip_prefix("uni") {- if !hex.is_empty() && hex.len() % 4 == 0 && hex.bytes().all(is_agl_hex_digit) {- let mut s = String::new();- for group in hex.as_bytes().chunks(4) {- let v = u32::from_str_radix(std::str::from_utf8(group).ok()?, 16).ok()?;- // Surrogate range is invalid scalar values- s.push(char::from_u32(v)?);- }- return Some(s);+ if let Some(hex) = comp.strip_prefix("uni")+ && !hex.is_empty()+ && hex.len() % 4 == 0+ && hex.bytes().all(is_agl_hex_digit)+ {+ let mut s = String::new();+ for group in hex.as_bytes().chunks(4) {+ let v = u32::from_str_radix(std::str::from_utf8(group).ok()?, 16).ok()?;+ // Surrogate range is invalid scalar values+ s.push(char::from_u32(v)?);}+ return Some(s);}// uXXXX / uXXXXX / uXXXXXX- if let Some(hex) = comp.strip_prefix('u') {- if (4..=6).contains(&hex.len()) && hex.bytes().all(is_agl_hex_digit) {- let v = u32::from_str_radix(hex, 16).ok()?;- return char::from_u32(v).map(String::from);- }+ if let Some(hex) = comp.strip_prefix('u')+ && (4..=6).contains(&hex.len())+ && hex.bytes().all(is_agl_hex_digit)+ {+ let v = u32::from_str_radix(hex, 16).ok()?;+ return char::from_u32(v).map(String::from);}// Static table lookupdiff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex 0ffcd75..e2d091b 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -926,8 +926,7 @@ fn classify_region(/// Classify a line that is *purely decorative* — no alphanumeric content, made/// up only of divider symbols (`* * * *`, `———`, `____`). Returns:/// - `Some(true)` → a section divider (≥3 symbols): emit a thematic break.-/// - `Some(false)` → a lone 1–2 char flourish (a stray em-dash under a-/// title): drop it (too small to be a meaningful rule).+/// - `Some(false)` → a lone 1–2 char, drop it, tool small/// - `None` → not decorative; classify normally.////// Without this, a `* * * *` divider line flows into the paragraph accumulatordiff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 1c54bfa..f1a1a6c 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -3101,9 +3101,12 @@ fn merge_stacked_header(/// continuation rows./// - **flattened header**: a fired colspan header flatten is table evidence on/// par with a spine.+////// With a spine but above the higher WITH_SPINE ceiling, still reject (unless a/// long-prose table). `flattened` is whether `flatten_header_band` fired.+////// Returns `true` to keep the table, `false` to reject it.+///fn passes_density_gate(cells: &[Vec<String>],cell_has_text: &[Vec<bool>],diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rsindex 5008c46..1e2e791 100644--- a/crates/liteparse/src/projection.rs+++ b/crates/liteparse/src/projection.rs@@ -737,8 +737,7 @@ fn form_lines(// inter-cell gap, never a digit butted directly against a// leading-decimal run, so this is safe to join silently.let is_decimal_continuation = y_compatible- && delta_x <= 1.0- && delta_x >= -2.0+ && (-2.0..=1.0).contains(&delta_x)&& prev.item.text@@ -4155,10 +4154,9 @@ fn xy_cut_rec(if !candidates.iter().any(|c| c.axis == CutAxis::Vertical && (c.score - 1.0e9).abs() < 1.0)+ && let Some(cc) = column_full{- if let Some(cp) = column_full {- candidates.push(cp);- }+ candidates.push(cc);}for cut in candidates {diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rsindex 7185cdd..b33680b 100644--- a/crates/liteparse/src/render.rs+++ b/crates/liteparse/src/render.rs@@ -1,7 +1,6 @@use crate::error::LiteParseError;-use crate::extract::load_document_from_input;+use crate::extract::{encode_png, load_document_from_input};use crate::types::PdfInput;-use image::ImageEncoder;use pdfium::Library;use serde::Serialize;@@ -68,13 +67,6 @@ fn render_document_pages(Ok(results)}-pub(crate) fn encode_png(rgba: &[u8], width: u32, height: u32) -> Result<Vec<u8>, LiteParseError> {- let mut png_buf = Vec::new();- let encoder = image::codecs::png::PngEncoder::new(&mut png_buf);- encoder.write_image(rgba, width, height, image::ColorType::Rgba8.into())?;- Ok(png_buf)-}-/// Render a single page to a PNG file.pub fn screenshot(pdf_path: &str,diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.tsindex 71c792c..be68713 100644--- a/packages/node/src/cli.ts+++ b/packages/node/src/cli.ts@@ -16,6 +16,15 @@ program.argument("<file>", "Path to the document file").option("-o, --output <file>", "Output file path").option("--format <format>", 'Output format: json|text|markdown (default: "text")')+ .option(+ "--image-mode <mode>",+ "How to surface raster images in markdown: off|placeholder|embed (default: placeholder)",+ )+ .option(+ "--image-output-dir <dir>",+ "Directory to write embedded images to when --image-mode embed is set",+ )+ .option("--no-links", "Disable hyperlink extraction (emit plain anchor text)").option("--ocr-server-url <url>", "HTTP OCR server URL").option("--no-ocr", "Disable OCR").option("--ocr-language <lang>", "OCR language (default: eng)")@@ -44,6 +53,9 @@ program// CLI options override config fileif (opts.format) config.outputFormat = opts.format as "json" | "text" | "markdown";+ if (opts.imageMode)+ config.imageMode = opts.imageMode as "off" | "placeholder" | "embed";+ if (opts.links === false) config.extractLinks = false;if (opts.ocrServerUrl)config.ocrServerUrl = opts.ocrServerUrl as string;if (opts.ocr === false) config.ocrEnabled = false;@@ -79,6 +91,19 @@ program): result.text;+ if (opts.imageOutputDir && result.images.length > 0) {+ const dir = opts.imageOutputDir as string;+ mkdirSync(dir, { recursive: true });+ for (const img of result.images) {+ writeFileSync(join(dir, `image_${img.id}.${img.format}`), img.bytes);+ }+ if (!opts.quiet) {+ console.error(+ `[liteparse] wrote ${result.images.length} image(s) to ${dir}`,+ );+ }+ }+if (opts.output) {writeFileSync(opts.output as string, output, "utf-8");} else {diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex e69387b..4127472 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -5,6 +5,7 @@ import {type NativeParseResult,type NativeParsedPage,type NativeTextItem,+ type NativeExtractedImage,} from "./native.js";// ---------------------------------------------------------------------------@@ -13,6 +14,7 @@ import {export type LiteParseInput = string | Buffer | Uint8Array;export type OutputFormat = "json" | "text" | "markdown";+export type ImageMode = "off" | "placeholder" | "embed";export interface LiteParseConfig {ocrLanguage: string;@@ -23,6 +25,10 @@ export interface LiteParseConfig {targetPages?: string;dpi: number;outputFormat: OutputFormat;+ /** How to surface raster images in markdown output (default: "placeholder"). */+ imageMode: ImageMode;+ /** Render hyperlink annotations as `[text](url)` in markdown output (default: true). */+ extractLinks: boolean;preserveVerySmallText: boolean;password?: string;quiet: boolean;@@ -48,9 +54,19 @@ export interface ParsedPage {textItems: TextItem[];}+export interface ExtractedImage {+ /** Reference id used in the markdown output (e.g. `` → `"p1_0"`). */+ id: string;+ page: number;+ format: string;+ bytes: Buffer;+}+export interface ParseResult {pages: ParsedPage[];text: string;+ /** Populated only when configured with `imageMode: "embed"`. */+ images: ExtractedImage[];}export interface ScreenshotResult {@@ -78,6 +94,8 @@ export class LiteParse {targetPages: userConfig.targetPages,dpi: userConfig.dpi,outputFormat: userConfig.outputFormat,+ imageMode: userConfig.imageMode,+ extractLinks: userConfig.extractLinks,preserveVerySmallText: userConfig.preserveVerySmallText,password: userConfig.password,quiet: userConfig.quiet,@@ -97,6 +115,8 @@ export class LiteParse {targetPages: resolved.targetPages ?? undefined,dpi: resolved.dpi ?? 150,outputFormat: (resolved.outputFormat as OutputFormat) ?? "json",+ imageMode: (resolved.imageMode as ImageMode) ?? "placeholder",+ extractLinks: resolved.extractLinks ?? true,preserveVerySmallText: resolved.preserveVerySmallText ?? false,password: resolved.password ?? undefined,quiet: resolved.quiet ?? false,@@ -112,6 +132,7 @@ export class LiteParse {return {pages: result.pages.map(toPage),text: result.text,+ images: (result.images ?? []).map(toImage),};}@@ -148,6 +169,15 @@ function toPage(p: NativeParsedPage): ParsedPage {};}+function toImage(img: NativeExtractedImage): ExtractedImage {+ return {+ id: img.id,+ page: img.page,+ format: img.format,+ bytes: img.bytes,+ };+}+function toTextItem(item: NativeTextItem): TextItem {return {text: item.text,diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex d6b3192..7133024 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -28,6 +28,8 @@ export interface LiteParseNativeConfig {targetPages?: string;dpi?: number;outputFormat?: string;+ imageMode?: string;+ extractLinks?: boolean;preserveVerySmallText?: boolean;password?: string;quiet?: boolean;@@ -53,9 +55,17 @@ export interface NativeParsedPage {textItems: NativeTextItem[];}+export interface NativeExtractedImage {+ id: string;+ page: number;+ format: string;+ bytes: Buffer;+}+export interface NativeParseResult {pages: NativeParsedPage[];text: string;+ images: NativeExtractedImage[];}export interface NativeScreenshotResult {