Skip to content

A fast, helpful, and open-source document parser

wip

Logan Markewich7ac8a25b9548
diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rs
index 8fbb377..21dec20 100644
--- a/crates/liteparse-python/src/cli.rs
+++ b/crates/liteparse-python/src/cli.rs
@@ -1,5 +1,5 @@
use clap::{Args, Parser, Subcommand};
-use liteparse::config::{LiteParseConfig, OutputFormat};
+use liteparse::config::{ImageMode, LiteParseConfig, OutputFormat};
use liteparse::conversion;
use liteparse::output::{json, text};
use liteparse::parser::LiteParse;
@@ -54,6 +54,20 @@ struct ParseCommand {
quiet: bool,
#[arg(long)]
num_workers: Option<usize>,
+ /// How to surface raster images in markdown output: `off`, `placeholder`
+ /// (default), or `embed` (extracts PNG bytes, written next to the output
+ /// when `--image-output-dir` is set).
+ #[arg(long, default_value = "placeholder")]
+ image_mode: String,
+ /// Directory to write embedded images to when `--image-mode embed` is set.
+ /// Each image is written as `image_{id}.png` to match the markdown
+ /// references. Created if missing.
+ #[arg(long)]
+ image_output_dir: Option<String>,
+ /// Disable hyperlink extraction. By default URI link annotations render as
+ /// `[text](url)` in markdown output; pass this to emit plain anchor text.
+ #[arg(long)]
+ no_links: bool,
}
#[derive(Args, Debug)]
@@ -113,6 +127,18 @@ fn parse_output_format(s: &str) -> Result<OutputFormat, String> {
}
}
+fn parse_image_mode(s: &str) -> Result<ImageMode, String> {
+ match s.to_lowercase().as_str() {
+ "off" | "none" => Ok(ImageMode::Off),
+ "placeholder" => Ok(ImageMode::Placeholder),
+ "embed" => Ok(ImageMode::Embed),
+ _ => Err(format!(
+ "unknown image-mode '{}', expected 'off', 'placeholder', or 'embed'",
+ s
+ )),
+ }
+}
+
/// Run the CLI with the given args (typically from sys.argv).
pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
let cli = Cli::parse_from(args);
@@ -121,6 +147,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
match cli.command {
Commands::Parse(cmd) => {
let format = parse_output_format(&cmd.format)?;
+ let image_mode = parse_image_mode(&cmd.image_mode)?;
let mut config = LiteParseConfig {
ocr_language: cmd.ocr_language,
ocr_enabled: !cmd.no_ocr,
@@ -133,6 +160,8 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
password: cmd.password,
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
+ image_mode,
+ extract_links: !cmd.no_links,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -145,6 +174,22 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
};
+ if let Some(dir) = cmd.image_output_dir.as_deref()
+ && !result.images.is_empty()
+ {
+ std::fs::create_dir_all(dir)?;
+ for img in &result.images {
+ let path = format!("{}/image_{}.{}", dir, img.id, img.format);
+ std::fs::write(&path, &img.bytes)?;
+ }
+ if !cmd.quiet {
+ eprintln!(
+ "[liteparse] wrote {} image(s) to {}",
+ result.images.len(),
+ dir
+ );
+ }
+ }
match cmd.output {
Some(path) => {
std::fs::write(&path, &formatted)?;
diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs
index 867d7e6..cfc9b57 100644
--- a/crates/liteparse-wasm/src/lib.rs
+++ b/crates/liteparse-wasm/src/lib.rs
@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};
use wasm_bindgen::prelude::*;
use wasm_bindgen_futures::JsFuture;
-use liteparse::config::{LiteParseConfig, OutputFormat};
+use liteparse::config::{ImageMode, LiteParseConfig, OutputFormat};
use liteparse::ocr::{OcrEngine, OcrOptions, OcrResult};
use liteparse::parser::LiteParse as CoreLiteParse;
use liteparse::search;
@@ -45,6 +45,8 @@ struct JsLiteParseConfig {
target_pages: Option<String>,
dpi: Option<f32>,
output_format: Option<String>,
+ image_mode: Option<String>,
+ extract_links: Option<bool>,
preserve_very_small_text: Option<bool>,
password: Option<String>,
quiet: Option<bool>,
@@ -87,6 +89,16 @@ impl JsLiteParseConfig {
}
};
}
+ if let Some(v) = self.image_mode {
+ cfg.image_mode = match v.as_str() {
+ "off" | "none" => ImageMode::Off,
+ "embed" => ImageMode::Embed,
+ _ => ImageMode::Placeholder,
+ };
+ }
+ if let Some(v) = self.extract_links {
+ cfg.extract_links = v;
+ }
if let Some(v) = self.preserve_very_small_text {
cfg.preserve_very_small_text = v;
}
@@ -114,6 +126,12 @@ impl JsLiteParseConfig {
OutputFormat::Text => "text".into(),
OutputFormat::Markdown => "markdown".into(),
}),
+ image_mode: Some(match cfg.image_mode {
+ ImageMode::Off => "off".into(),
+ ImageMode::Placeholder => "placeholder".into(),
+ ImageMode::Embed => "embed".into(),
+ }),
+ extract_links: Some(cfg.extract_links),
preserve_very_small_text: Some(cfg.preserve_very_small_text),
password: cfg.password.clone(),
quiet: Some(cfg.quiet),
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 2f4fd72..fb5ee35 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -1,10 +1,10 @@
use crate::error::LiteParseError;
use crate::glyph_names::resolve_glyph_name;
-use crate::render::encode_png;
use crate::types::{
ExtractedImage, GraphicPrimitive, ImageRef, OutlineTarget, Page as LitePage, PdfInput, Rect,
StructNode, TextItem,
};
+use image::ImageEncoder;
use pdfium::{
Document, Font, FontType, Library, Page, PathObject, PdfLink, RectF, SegmentKind, TextPage,
};
@@ -310,6 +310,16 @@ pub(crate) fn render_page_images(
out
}
+/// Encode RGBA pixel bytes to PNG. Lives here (always-compiled) rather than in
+/// `render` so the image-embed path is available on wasm, where the `render`
+/// module (page rasterization / screenshots) is compiled out.
+pub(crate) fn encode_png(rgba: &[u8], width: u32, height: u32) -> Result<Vec<u8>, LiteParseError> {
+ let mut png_buf = Vec::new();
+ let encoder = image::codecs::png::PngEncoder::new(&mut png_buf);
+ encoder.write_image(rgba, width, height, image::ColorType::Rgba8.into())?;
+ Ok(png_buf)
+}
+
/// Walk image objects on a page and return a stable per-page `ImageRef` for
/// each one. `obj_index` is the index among image-typed page objects (not all
/// page objects), so a later embed pass can pull pixel bytes via
@@ -967,7 +977,7 @@ fn median_f32(values: &[f32]) -> Option<f32> {
let mut v: Vec<f32> = values.to_vec();
v.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let mid = v.len() / 2;
- if v.len() % 2 == 0 {
+ if v.len().is_multiple_of(2) {
Some((v[mid - 1] + v[mid]) / 2.0)
} else {
Some(v[mid])
diff --git a/crates/liteparse/src/glyph_names.rs b/crates/liteparse/src/glyph_names.rs
index cee16d3..1374a51 100644
--- a/crates/liteparse/src/glyph_names.rs
+++ b/crates/liteparse/src/glyph_names.rs
@@ -49,24 +49,27 @@ fn resolve_component(comp: &str) -> Option<String> {
}
// uniXXXX (one or more groups of exactly 4 uppercase hex digits)
- if let Some(hex) = comp.strip_prefix("uni") {
- if !hex.is_empty() && hex.len() % 4 == 0 && hex.bytes().all(is_agl_hex_digit) {
- let mut s = String::new();
- for group in hex.as_bytes().chunks(4) {
- let v = u32::from_str_radix(std::str::from_utf8(group).ok()?, 16).ok()?;
- // Surrogate range is invalid scalar values
- s.push(char::from_u32(v)?);
- }
- return Some(s);
+ if let Some(hex) = comp.strip_prefix("uni")
+ && !hex.is_empty()
+ && hex.len() % 4 == 0
+ && hex.bytes().all(is_agl_hex_digit)
+ {
+ let mut s = String::new();
+ for group in hex.as_bytes().chunks(4) {
+ let v = u32::from_str_radix(std::str::from_utf8(group).ok()?, 16).ok()?;
+ // Surrogate range is invalid scalar values
+ s.push(char::from_u32(v)?);
}
+ return Some(s);
}
// uXXXX / uXXXXX / uXXXXXX
- if let Some(hex) = comp.strip_prefix('u') {
- if (4..=6).contains(&hex.len()) && hex.bytes().all(is_agl_hex_digit) {
- let v = u32::from_str_radix(hex, 16).ok()?;
- return char::from_u32(v).map(String::from);
- }
+ if let Some(hex) = comp.strip_prefix('u')
+ && (4..=6).contains(&hex.len())
+ && hex.bytes().all(is_agl_hex_digit)
+ {
+ let v = u32::from_str_radix(hex, 16).ok()?;
+ return char::from_u32(v).map(String::from);
}
// Static table lookup
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index 0ffcd75..e2d091b 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -926,8 +926,7 @@ fn classify_region(
/// Classify a line that is *purely decorative* — no alphanumeric content, made
/// up only of divider symbols (`* * * *`, `———`, `____`). Returns:
/// - `Some(true)` → a section divider (≥3 symbols): emit a thematic break.
-/// - `Some(false)` → a lone 1–2 char flourish (a stray em-dash under a
-/// title): drop it (too small to be a meaningful rule).
+/// - `Some(false)` → a lone 1–2 char, drop it, tool small
/// - `None` → not decorative; classify normally.
///
/// Without this, a `* * * *` divider line flows into the paragraph accumulator
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 1c54bfa..f1a1a6c 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -3101,9 +3101,12 @@ fn merge_stacked_header(
/// continuation rows.
/// - **flattened header**: a fired colspan header flatten is table evidence on
/// par with a spine.
+///
/// With a spine but above the higher WITH_SPINE ceiling, still reject (unless a
/// long-prose table). `flattened` is whether `flatten_header_band` fired.
+///
/// Returns `true` to keep the table, `false` to reject it.
+///
fn passes_density_gate(
cells: &[Vec<String>],
cell_has_text: &[Vec<bool>],
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index 5008c46..1e2e791 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -737,8 +737,7 @@ fn form_lines(
// inter-cell gap, never a digit butted directly against a
// leading-decimal run, so this is safe to join silently.
let is_decimal_continuation = y_compatible
- && delta_x <= 1.0
- && delta_x >= -2.0
+ && (-2.0..=1.0).contains(&delta_x)
&& prev
.item
.text
@@ -4155,10 +4154,9 @@ fn xy_cut_rec(
if !candidates
.iter()
.any(|c| c.axis == CutAxis::Vertical && (c.score - 1.0e9).abs() < 1.0)
+ && let Some(cc) = column_full
{
- if let Some(cp) = column_full {
- candidates.push(cp);
- }
+ candidates.push(cc);
}
for cut in candidates {
diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rs
index 7185cdd..b33680b 100644
--- a/crates/liteparse/src/render.rs
+++ b/crates/liteparse/src/render.rs
@@ -1,7 +1,6 @@
use crate::error::LiteParseError;
-use crate::extract::load_document_from_input;
+use crate::extract::{encode_png, load_document_from_input};
use crate::types::PdfInput;
-use image::ImageEncoder;
use pdfium::Library;
use serde::Serialize;
@@ -68,13 +67,6 @@ fn render_document_pages(
Ok(results)
}
-pub(crate) fn encode_png(rgba: &[u8], width: u32, height: u32) -> Result<Vec<u8>, LiteParseError> {
- let mut png_buf = Vec::new();
- let encoder = image::codecs::png::PngEncoder::new(&mut png_buf);
- encoder.write_image(rgba, width, height, image::ColorType::Rgba8.into())?;
- Ok(png_buf)
-}
-
/// Render a single page to a PNG file.
pub fn screenshot(
pdf_path: &str,
diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts
index 71c792c..be68713 100644
--- a/packages/node/src/cli.ts
+++ b/packages/node/src/cli.ts
@@ -16,6 +16,15 @@ program
.argument("<file>", "Path to the document file")
.option("-o, --output <file>", "Output file path")
.option("--format <format>", 'Output format: json|text|markdown (default: "text")')
+ .option(
+ "--image-mode <mode>",
+ "How to surface raster images in markdown: off|placeholder|embed (default: placeholder)",
+ )
+ .option(
+ "--image-output-dir <dir>",
+ "Directory to write embedded images to when --image-mode embed is set",
+ )
+ .option("--no-links", "Disable hyperlink extraction (emit plain anchor text)")
.option("--ocr-server-url <url>", "HTTP OCR server URL")
.option("--no-ocr", "Disable OCR")
.option("--ocr-language <lang>", "OCR language (default: eng)")
@@ -44,6 +53,9 @@ program
// CLI options override config file
if (opts.format) config.outputFormat = opts.format as "json" | "text" | "markdown";
+ if (opts.imageMode)
+ config.imageMode = opts.imageMode as "off" | "placeholder" | "embed";
+ if (opts.links === false) config.extractLinks = false;
if (opts.ocrServerUrl)
config.ocrServerUrl = opts.ocrServerUrl as string;
if (opts.ocr === false) config.ocrEnabled = false;
@@ -79,6 +91,19 @@ program
)
: result.text;
+ if (opts.imageOutputDir && result.images.length > 0) {
+ const dir = opts.imageOutputDir as string;
+ mkdirSync(dir, { recursive: true });
+ for (const img of result.images) {
+ writeFileSync(join(dir, `image_${img.id}.${img.format}`), img.bytes);
+ }
+ if (!opts.quiet) {
+ console.error(
+ `[liteparse] wrote ${result.images.length} image(s) to ${dir}`,
+ );
+ }
+ }
+
if (opts.output) {
writeFileSync(opts.output as string, output, "utf-8");
} else {
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index e69387b..4127472 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -5,6 +5,7 @@ import {
type NativeParseResult,
type NativeParsedPage,
type NativeTextItem,
+ type NativeExtractedImage,
} from "./native.js";
// ---------------------------------------------------------------------------
@@ -13,6 +14,7 @@ import {
export type LiteParseInput = string | Buffer | Uint8Array;
export type OutputFormat = "json" | "text" | "markdown";
+export type ImageMode = "off" | "placeholder" | "embed";
export interface LiteParseConfig {
ocrLanguage: string;
@@ -23,6 +25,10 @@ export interface LiteParseConfig {
targetPages?: string;
dpi: number;
outputFormat: OutputFormat;
+ /** How to surface raster images in markdown output (default: "placeholder"). */
+ imageMode: ImageMode;
+ /** Render hyperlink annotations as `[text](url)` in markdown output (default: true). */
+ extractLinks: boolean;
preserveVerySmallText: boolean;
password?: string;
quiet: boolean;
@@ -48,9 +54,19 @@ export interface ParsedPage {
textItems: TextItem[];
}
+export interface ExtractedImage {
+ /** Reference id used in the markdown output (e.g. `![](image_p1_0.png)` → `"p1_0"`). */
+ id: string;
+ page: number;
+ format: string;
+ bytes: Buffer;
+}
+
export interface ParseResult {
pages: ParsedPage[];
text: string;
+ /** Populated only when configured with `imageMode: "embed"`. */
+ images: ExtractedImage[];
}
export interface ScreenshotResult {
@@ -78,6 +94,8 @@ export class LiteParse {
targetPages: userConfig.targetPages,
dpi: userConfig.dpi,
outputFormat: userConfig.outputFormat,
+ imageMode: userConfig.imageMode,
+ extractLinks: userConfig.extractLinks,
preserveVerySmallText: userConfig.preserveVerySmallText,
password: userConfig.password,
quiet: userConfig.quiet,
@@ -97,6 +115,8 @@ export class LiteParse {
targetPages: resolved.targetPages ?? undefined,
dpi: resolved.dpi ?? 150,
outputFormat: (resolved.outputFormat as OutputFormat) ?? "json",
+ imageMode: (resolved.imageMode as ImageMode) ?? "placeholder",
+ extractLinks: resolved.extractLinks ?? true,
preserveVerySmallText: resolved.preserveVerySmallText ?? false,
password: resolved.password ?? undefined,
quiet: resolved.quiet ?? false,
@@ -112,6 +132,7 @@ export class LiteParse {
return {
pages: result.pages.map(toPage),
text: result.text,
+ images: (result.images ?? []).map(toImage),
};
}
@@ -148,6 +169,15 @@ function toPage(p: NativeParsedPage): ParsedPage {
};
}
+function toImage(img: NativeExtractedImage): ExtractedImage {
+ return {
+ id: img.id,
+ page: img.page,
+ format: img.format,
+ bytes: img.bytes,
+ };
+}
+
function toTextItem(item: NativeTextItem): TextItem {
return {
text: item.text,
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index d6b3192..7133024 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -28,6 +28,8 @@ export interface LiteParseNativeConfig {
targetPages?: string;
dpi?: number;
outputFormat?: string;
+ imageMode?: string;
+ extractLinks?: boolean;
preserveVerySmallText?: boolean;
password?: string;
quiet?: boolean;
@@ -53,9 +55,17 @@ export interface NativeParsedPage {
textItems: NativeTextItem[];
}
+export interface NativeExtractedImage {
+ id: string;
+ page: number;
+ format: string;
+ bytes: Buffer;
+}
+
export interface NativeParseResult {
pages: NativeParsedPage[];
text: string;
+ images: NativeExtractedImage[];
}
export interface NativeScreenshotResult {