Skip to content

A fast, helpful, and open-source document parser

nits

Logan Markewich5c7f46f44249
diff --git a/README.md b/README.md
index f9a0e04..c7fc119 100644
--- a/README.md
+++ b/README.md
@@ -206,7 +206,7 @@ Image handling is controlled by `--image-mode`:
| Mode | Behavior |
|------|----------|
-| `placeholder` (default) | Emits `![](image_pN_K.png)` references in reading order |
+| `placeholder` (default) | Emits `![](img_pN_K.png)` references in reading order |
| `off` | Strips images entirely |
| `embed` | Emits the same image references as `placeholder` |
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs
index 57f2142..670986d 100644
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@ -45,7 +45,7 @@ pub struct JsLiteParseConfig {
/// Number of concurrent OCR workers (default: CPU cores - 1).
pub num_workers: Option<u32>,
/// How to surface raster images in markdown output: "off", "placeholder"
- /// (default — emits `![](image_pN_K.png)` references with no bytes), or
+ /// (default — emits `![](img_pN_K.png)` references with no bytes), or
/// "embed" (same presentation as placeholder; extraction is independent).
pub image_mode: Option<String>,
/// Extract embedded image bytes and metadata (default false).
@@ -71,6 +71,9 @@ pub struct JsLiteParseConfig {
/// Detect solid rectangles/lines in rendered page screenshots and attach
/// them to each screenshot result. Default false.
pub detect_screenshot_rects: Option<bool>,
+ /// Draw AcroForm field appearances into rendered rasters (screenshots and
+ /// OCR inputs). Runs the document's open/JS actions. Default false.
+ pub render_form_fields: Option<bool>,
/// Whether a systemic OCR failure aborts the whole parse (default true).
/// Set false to keep already-recovered native text and return partial
/// results when OCR is unavailable, instead of rejecting.
@@ -191,6 +194,9 @@ impl JsLiteParseConfig {
if let Some(v) = self.detect_screenshot_rects {
cfg.detect_screenshot_rects = v;
}
+ if let Some(v) = self.render_form_fields {
+ cfg.render_form_fields = v;
+ }
if let Some(v) = self.ocr_failure_fatal {
cfg.ocr_failure_fatal = v;
}
@@ -260,6 +266,7 @@ impl JsLiteParseConfig {
extract_xfa_packets: Some(cfg.extract_xfa_packets),
extract_content_bounds: Some(cfg.extract_content_bounds),
detect_screenshot_rects: Some(cfg.detect_screenshot_rects),
+ render_form_fields: Some(cfg.render_form_fields),
ocr_failure_fatal: Some(cfg.ocr_failure_fatal),
ocr_hedge_delays_ms: Some(
cfg.ocr_hedge_delays_ms
@@ -395,22 +402,25 @@ impl JsTextItem {
}
}
+ /// `from_rust` with the rich-metadata fields taken from the core-gated
+ /// [`liteparse::types::TextMetadata`] view, so the "what counts as text
+ /// metadata" list lives in one place instead of per binding.
fn from_rust_for_output(item: &TextItem, extract_text_metadata: bool) -> Self {
- let mut output = Self::from_rust(item);
- if !extract_text_metadata {
- output.font_height = None;
- output.font_ascent = None;
- output.font_descent = None;
- output.font_weight = None;
- output.text_width = None;
- output.font_is_buggy = None;
- output.mcid = None;
- output.fill_color = None;
- output.stroke_color = None;
- output.char_codes = None;
- output.trailing_space_generated = None;
+ let meta = item.text_metadata(extract_text_metadata);
+ Self {
+ font_height: meta.font_height.map(|v| v as f64),
+ font_ascent: meta.font_ascent.map(|v| v as f64),
+ font_descent: meta.font_descent.map(|v| v as f64),
+ font_weight: meta.font_weight,
+ text_width: meta.text_width.map(|v| v as f64),
+ font_is_buggy: meta.font_is_buggy,
+ mcid: meta.mcid,
+ fill_color: meta.fill_color.map(str::to_owned),
+ stroke_color: meta.stroke_color.map(str::to_owned),
+ char_codes: meta.char_codes.map(<[u32]>::to_vec),
+ trailing_space_generated: meta.trailing_space_generated,
+ ..Self::from_rust(item)
}
- output
}
}
@@ -1076,7 +1086,7 @@ impl JsParseResult {
rotation: img.rotation as f64,
format: img.format.clone(),
duplicate_of: img.duplicate_of.clone(),
- bytes: img.bytes.clone().into(),
+ bytes: img.bytes.as_slice().to_vec().into(),
})
.collect(),
}
diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rs
index cde165a..5bff1eb 100644
--- a/crates/liteparse-python/src/cli.rs
+++ b/crates/liteparse-python/src/cli.rs
@@ -181,6 +181,12 @@ struct BatchParseCommand {
/// Include page-scoped vector shapes and merged horizontal/vertical lines.
#[arg(long)]
extract_vector_graphics: bool,
+ /// Include all PDF annotations as page-scoped structured data.
+ #[arg(long)]
+ extract_annotations: bool,
+ /// Include AcroForm widget fields and values.
+ #[arg(long)]
+ extract_form_fields: bool,
/// Include the tagged-PDF logical structure tree.
#[arg(long)]
extract_structure_tree: bool,
@@ -367,6 +373,8 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
extract_text_metadata: cmd.extract_text_metadata,
extract_images: cmd.extract_images,
extract_vector_graphics: cmd.extract_vector_graphics,
+ extract_annotations: cmd.extract_annotations,
+ extract_form_fields: cmd.extract_form_fields,
extract_structure_tree: cmd.extract_structure_tree,
extract_xfa_packets: cmd.extract_xfa_packets,
extract_content_bounds: cmd.extract_content_bounds,
diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs
index 0359968..9cf5839 100644
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@ -378,24 +378,30 @@ impl PyTextItem {
}
}
- fn from_rust_for_output(
- mut item: liteparse::types::TextItem,
- extract_text_metadata: bool,
- ) -> Self {
- if !extract_text_metadata {
- item.font_height = None;
- item.font_ascent = None;
- item.font_descent = None;
- item.font_weight = None;
- item.text_width = None;
- item.font_is_buggy = false;
- item.mcid = None;
- item.fill_color = None;
- item.stroke_color = None;
- item.char_codes.clear();
- item.trailing_space_generated = false;
+ /// `from_rust` with the rich-metadata fields taken from the core-gated
+ /// [`liteparse::types::TextMetadata`] view, so the "what counts as text
+ /// metadata" list lives in one place instead of per binding.
+ fn from_rust_for_output(item: liteparse::types::TextItem, extract_text_metadata: bool) -> Self {
+ let meta = item.text_metadata(extract_text_metadata);
+ let (fill_color, stroke_color, char_codes) = (
+ meta.fill_color.map(str::to_owned),
+ meta.stroke_color.map(str::to_owned),
+ meta.char_codes.map(<[u32]>::to_vec),
+ );
+ Self {
+ font_height: meta.font_height.map(|v| v as f64),
+ font_ascent: meta.font_ascent.map(|v| v as f64),
+ font_descent: meta.font_descent.map(|v| v as f64),
+ font_weight: meta.font_weight,
+ text_width: meta.text_width.map(|v| v as f64),
+ font_is_buggy: meta.font_is_buggy.unwrap_or(false),
+ mcid: meta.mcid,
+ fill_color,
+ stroke_color,
+ char_codes: char_codes.unwrap_or_default(),
+ trailing_space_generated: meta.trailing_space_generated.unwrap_or(false),
+ ..Self::from_rust(item)
}
- Self::from_rust(item)
}
}
@@ -708,7 +714,7 @@ struct PyExtractedImage {
format: String,
#[pyo3(get)]
duplicate_of: Option<String>,
- bytes_buffer: Vec<u8>,
+ bytes_buffer: std::sync::Arc<Vec<u8>>,
}
#[pyclass(frozen, from_py_object)]
@@ -728,7 +734,7 @@ struct PyImageRect {
impl PyExtractedImage {
#[getter]
fn bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
- PyBytes::new(py, &self.bytes_buffer)
+ PyBytes::new(py, self.bytes_buffer.as_slice())
}
fn __repr__(&self) -> String {
@@ -996,6 +1002,8 @@ struct PyLiteParseConfig {
#[pyo3(get)]
detect_screenshot_rects: bool,
#[pyo3(get)]
+ render_form_fields: bool,
+ #[pyo3(get)]
ocr_failure_fatal: bool,
#[pyo3(get)]
ocr_hedge_delays_ms: Vec<u64>,
@@ -1063,6 +1071,7 @@ impl PyLiteParseConfig {
extract_xfa_packets: cfg.extract_xfa_packets,
extract_content_bounds: cfg.extract_content_bounds,
detect_screenshot_rects: cfg.detect_screenshot_rects,
+ render_form_fields: cfg.render_form_fields,
ocr_failure_fatal: cfg.ocr_failure_fatal,
ocr_hedge_delays_ms: cfg.ocr_hedge_delays_ms.clone(),
emit_word_boxes: cfg.emit_word_boxes,
@@ -1119,6 +1128,7 @@ impl LiteParse {
extract_xfa_packets = None,
extract_content_bounds = None,
detect_screenshot_rects = None,
+ render_form_fields = None,
ocr_failure_fatal = None,
ocr_hedge_delays_ms = None,
emit_word_boxes = None,
@@ -1152,6 +1162,7 @@ impl LiteParse {
extract_xfa_packets: Option<bool>,
extract_content_bounds: Option<bool>,
detect_screenshot_rects: Option<bool>,
+ render_form_fields: Option<bool>,
ocr_failure_fatal: Option<bool>,
ocr_hedge_delays_ms: Option<Vec<u64>>,
emit_word_boxes: Option<bool>,
@@ -1239,6 +1250,9 @@ impl LiteParse {
if let Some(v) = detect_screenshot_rects {
cfg.detect_screenshot_rects = v;
}
+ if let Some(v) = render_form_fields {
+ cfg.render_form_fields = v;
+ }
if let Some(v) = ocr_failure_fatal {
cfg.ocr_failure_fatal = v;
}
diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs
index 8e4fe7a..94b6759 100644
--- a/crates/liteparse-wasm/src/lib.rs
+++ b/crates/liteparse-wasm/src/lib.rs
@@ -84,6 +84,13 @@ pub struct LiteParseConfig {
include_complexity: Option<bool>,
/// Expose page-scoped vector shapes and merged H/V line segments.
extract_vector_graphics: Option<bool>,
+ /// Include rich PDF text metadata on text items (font metrics/weight,
+ /// buggy state, MCID, fill/stroke colors, raw char codes, generated
+ /// trailing-space state). Default false.
+ extract_text_metadata: Option<bool>,
+ /// Draw AcroForm field appearances into OCR rasters (runs document
+ /// open/JS actions). Default false.
+ render_form_fields: Option<bool>,
}
/// A page sub-region as the fraction cropped from each side (top-left origin,
@@ -201,6 +208,12 @@ impl LiteParseConfig {
if let Some(v) = self.extract_vector_graphics {
cfg.extract_vector_graphics = v;
}
+ if let Some(v) = self.extract_text_metadata {
+ cfg.extract_text_metadata = v;
+ }
+ if let Some(v) = self.render_form_fields {
+ cfg.render_form_fields = v;
+ }
cfg.num_workers = 1;
Ok(cfg)
}
@@ -251,6 +264,8 @@ impl LiteParseConfig {
skip_diagonal_text: Some(cfg.skip_diagonal_text),
include_complexity: Some(cfg.include_complexity),
extract_vector_graphics: Some(cfg.extract_vector_graphics),
+ extract_text_metadata: Some(cfg.extract_text_metadata),
+ render_form_fields: Some(cfg.render_form_fields),
}
}
}
@@ -287,6 +302,29 @@ pub struct TextItem {
pub confidence: Option<f32>,
/// Rotation in degrees (viewport space).
pub rotation: f32,
+ // Rich text metadata — present only when `extractTextMetadata` is set.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub font_height: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub font_ascent: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub font_descent: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub font_weight: Option<i32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub text_width: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub font_is_buggy: Option<bool>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub mcid: Option<i32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub fill_color: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub stroke_color: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub char_codes: Option<Vec<u32>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub trailing_space_generated: Option<bool>,
/// Per-word sub-boxes for attribution. Omitted when empty (the default —
/// only populated when parsing with `emitWordBoxes: true`).
#[serde(skip_serializing_if = "Option::is_none")]
@@ -315,6 +353,8 @@ pub struct ParsedPage {
pub annotations: Option<Vec<DocumentAnnotation>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub form_fields: Option<Vec<FormField>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub structure_tree: Option<StructureTree>,
}
#[derive(Serialize, Tsify)]
@@ -466,6 +506,96 @@ impl FormField {
}
}
+/// Scalar value from a tagged-PDF structure element's `/A` dictionary.
+#[derive(Serialize, Tsify)]
+#[tsify(into_wasm_abi)]
+#[serde(untagged)]
+pub enum StructureAttributeValue {
+ Boolean(bool),
+ Number(f32),
+ String(String),
+}
+
+#[derive(Serialize, Tsify)]
+#[tsify(into_wasm_abi)]
+#[serde(rename_all = "camelCase")]
+pub struct StructureTree {
+ pub roots: Vec<StructureTreeElement>,
+}
+
+#[derive(Serialize, Tsify)]
+#[tsify(into_wasm_abi)]
+#[serde(rename_all = "camelCase")]
+pub struct StructureTreeElement {
+ #[serde(rename = "type")]
+ pub element_type: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub actual_text: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub alt_text: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub title: Option<String>,
+ pub attributes: std::collections::BTreeMap<String, StructureAttributeValue>,
+ pub marked_content_ids: Vec<i32>,
+ pub children: Vec<StructureTreeElement>,
+ pub annotations: Vec<DocumentAnnotation>,
+}
+
+impl StructureTree {
+ fn from_rust(tree: &liteparse::types::StructureTree) -> Self {
+ Self {
+ roots: tree
+ .roots
+ .iter()
+ .map(StructureTreeElement::from_rust)
+ .collect(),
+ }
+ }
+}
+
+impl StructureTreeElement {
+ fn from_rust(element: &liteparse::types::StructureTreeElement) -> Self {
+ Self {
+ element_type: element.element_type.clone(),
+ id: element.id.clone(),
+ actual_text: element.actual_text.clone(),
+ alt_text: element.alt_text.clone(),
+ title: element.title.clone(),
+ attributes: element
+ .attributes
+ .iter()
+ .map(|(name, value)| {
+ let value = match value {
+ liteparse::types::StructureAttributeValue::Boolean(v) => {
+ StructureAttributeValue::Boolean(*v)
+ }
+ liteparse::types::StructureAttributeValue::Number(v) => {
+ StructureAttributeValue::Number(*v)
+ }
+ liteparse::types::StructureAttributeValue::String(v) => {
+ StructureAttributeValue::String(v.clone())
+ }
+ };
+ (name.clone(), value)
+ })
+ .collect(),
+ marked_content_ids: element.marked_content_ids.clone(),
+ children: element
+ .children
+ .iter()
+ .map(StructureTreeElement::from_rust)
+ .collect(),
+ annotations: element
+ .annotations
+ .iter()
+ .map(DocumentAnnotation::from_rust)
+ .collect(),
+ }
+ }
+}
+
#[derive(Serialize, Tsify)]
#[tsify(into_wasm_abi)]
#[serde(rename_all = "camelCase")]
@@ -704,6 +834,7 @@ impl LiteParse {
.await
.map_err(|e| JsError::new(&format!("parse failed: {}", e)))?;
+ let extract_text_metadata = self.inner.config().extract_text_metadata;
let pages: Vec<ParsedPage> = result
.pages
.iter()
@@ -722,32 +853,51 @@ impl LiteParse {
text_items: p
.text_items
.iter()
- .map(|i| TextItem {
- text: i.text.clone(),
- x: i.x,
- y: i.y,
- width: i.width,
- height: i.height,
- font_name: i.font_name.clone(),
- font_size: i.font_size,
- confidence: i.confidence,
- rotation: i.rotation,
- words: if i.words.is_empty() {
- None
- } else {
- Some(
- i.words
- .iter()
- .map(|w| WordBox {
- text: w.text.clone(),
- x: w.x,
- y: w.y,
- width: w.width,
- height: w.height,
- })
- .collect(),
- )
- },
+ .map(|i| {
+ // Core-gated metadata view — the single source of
+ // truth for which fields `extractTextMetadata` covers.
+ let meta = i.text_metadata(extract_text_metadata);
+ TextItem {
+ text: i.text.clone(),
+ x: i.x,
+ y: i.y,
+ width: i.width,
+ height: i.height,
+ font_name: i.font_name.clone(),
+ font_size: i.font_size,
+ confidence: i.confidence,
+ rotation: i.rotation,
+ font_height: meta.font_height,
+ font_ascent: meta.font_ascent,
+ font_descent: meta.font_descent,
+ font_weight: meta.font_weight,
+ text_width: meta.text_width,
+ font_is_buggy: meta.font_is_buggy,
+ mcid: meta.mcid,
+ fill_color: meta.fill_color.map(str::to_owned),
+ stroke_color: meta.stroke_color.map(str::to_owned),
+ char_codes: meta
+ .char_codes
+ .filter(|codes| !codes.is_empty())
+ .map(<[u32]>::to_vec),
+ trailing_space_generated: meta.trailing_space_generated,
+ words: if i.words.is_empty() {
+ None
+ } else {
+ Some(
+ i.words
+ .iter()
+ .map(|w| WordBox {
+ text: w.text.clone(),
+ x: w.x,
+ y: w.y,
+ width: w.width,
+ height: w.height,
+ })
+ .collect(),
+ )
+ },
+ }
})
.collect(),
complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust),
@@ -795,6 +945,7 @@ impl LiteParse {
.form_fields
.as_ref()
.map(|fields| fields.iter().map(FormField::from_rust).collect()),
+ structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust),
})
.collect();
@@ -817,7 +968,7 @@ impl LiteParse {
rotation: img.rotation,
format: img.format.clone(),
duplicate_of: img.duplicate_of.clone(),
- bytes: img.bytes.clone(),
+ bytes: img.bytes.as_slice().to_vec(),
})
.collect();
@@ -1010,6 +1161,17 @@ pub fn search_items(items: Vec<SearchTextItem>, options: SearchOptions) -> Vec<T
font_size: i.font_size,
confidence: i.confidence,
rotation: i.rotation,
+ font_height: None,
+ font_ascent: None,
+ font_descent: None,
+ font_weight: None,
+ text_width: None,
+ font_is_buggy: None,
+ mcid: None,
+ fill_color: None,
+ stroke_color: None,
+ char_codes: None,
+ trailing_space_generated: None,
words: if i.words.is_empty() {
None
} else {
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs
index 688773e..3c11f98 100644
--- a/crates/liteparse/src/config.rs
+++ b/crates/liteparse/src/config.rs
@@ -35,7 +35,8 @@ pub struct LiteParseConfig {
/// This does not enable embedded-image extraction.
pub image_mode: ImageMode,
/// Extract embedded image bytes and metadata into `ParseResult.images`.
- /// Defaults to false and is the only switch that enables extraction.
+ /// Defaults to false. `ImageMode::Embed` also enables extraction for
+ /// backwards compatibility (see [`LiteParseConfig::effective_extract_images`]).
pub extract_images: bool,
/// Directory where extracted embedded images are written. Requires
/// `extract_images` to be true.
@@ -72,6 +73,13 @@ pub struct LiteParseConfig {
/// vector paths. Default `false` (adds a full-bitmap scan per page).
#[serde(default)]
pub detect_screenshot_rects: bool,
+ /// Draw AcroForm field appearances (filled values, checkbox states) into
+ /// rendered rasters — screenshots and OCR inputs. This initializes a
+ /// PDFium form-fill environment and runs the document's open/JS actions,
+ /// so it is off by default: plain parses should neither execute document
+ /// scripts nor change raster bytes for form-bearing PDFs.
+ #[serde(default)]
+ pub render_form_fields: bool,
/// Whether a systemic OCR failure (every OCR task failed *and* at least one
/// was a text-sparse page whose primary text source was OCR) aborts the
/// whole parse. Default `true`: surface the root cause instead of silently
@@ -138,12 +146,13 @@ pub struct CropBox {
/// Image handling for the markdown emitter.
///
/// * `Off` — strip image references entirely.
-/// * `Placeholder` (default) — emit `![](image_pN_K.png)` references in
+/// * `Placeholder` (default) — emit `![](img_pN_K.png)` references in
/// reading order at each image's y position, but do **not** extract or
/// return pixel bytes. Keeps response size small while letting the LLM see
/// where figures live in the document.
-/// * `Embed` — emit the same references as `Placeholder`. Embedded pixel bytes
-/// are controlled independently by `LiteParseConfig::extract_images`.
+/// * `Embed` — emit the same references as `Placeholder`, and extract the
+/// embedded pixel bytes into `ParseResult.images` (equivalent to setting
+/// `LiteParseConfig::extract_images`, which `Embed` predates).
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ImageMode {
@@ -161,6 +170,16 @@ pub enum OutputFormat {
Markdown,
}
+impl LiteParseConfig {
+ /// Whether embedded-image extraction should run. True when
+ /// `extract_images` is set, or when the legacy `ImageMode::Embed` is
+ /// configured — `Embed` implied byte extraction before the two switches
+ /// were decoupled, and existing callers rely on that.
+ pub fn effective_extract_images(&self) -> bool {
+ self.extract_images || self.image_mode == ImageMode::Embed
+ }
+}
+
impl Default for LiteParseConfig {
fn default() -> Self {
Self {
@@ -197,6 +216,7 @@ impl Default for LiteParseConfig {
extract_content_bounds: false,
extract_xfa_packets: false,
detect_screenshot_rects: false,
+ render_form_fields: false,
ocr_failure_fatal: true,
ocr_hedge_delays_ms: Vec::new(),
emit_word_boxes: false,
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 0cf8b0e..e099282 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -77,7 +77,7 @@ pub(crate) fn extract_pages_and_images(
let page_count = document.page_count();
let mut pages = Vec::new();
let mut images: Vec<ExtractedImage> = Vec::new();
- let mut image_cache: Vec<CachedImage> = Vec::new();
+ let mut image_cache = ImageCache::default();
let mut image_error_count = 0u32;
let form_environment = output_options
.extract_form_fields
@@ -522,13 +522,55 @@ struct CachedImage {
raw_bytes: Vec<u8>,
id: String,
format: String,
- bytes: Vec<u8>,
+ bytes: std::sync::Arc<Vec<u8>>,
+}
+
+/// Dedup key: hash of the source stream plus the metadata that affects
+/// decoding. The full raw bytes are still compared on lookup (hash prefilter,
+/// not trust-the-hash), but only within the matching bucket.
+#[derive(PartialEq, Eq, Hash)]
+struct CacheKey {
+ raw_hash: u64,
width: u32,
height: u32,
bits_per_pixel: u32,
colorspace: i32,
}
+#[derive(Default)]
+pub(crate) struct ImageCache {
+ entries: std::collections::HashMap<CacheKey, Vec<CachedImage>>,
+}
+
+impl ImageCache {
+ fn key(r: &ImageRef, raw_bytes: &[u8]) -> CacheKey {
+ use std::hash::{Hash, Hasher};
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ raw_bytes.hash(&mut hasher);
+ CacheKey {
+ raw_hash: hasher.finish(),
+ width: r.pixel_width,
+ height: r.pixel_height,
+ bits_per_pixel: r.bits_per_pixel,
+ colorspace: r.colorspace,
+ }
+ }
+
+ fn get(&self, r: &ImageRef, raw_bytes: &[u8]) -> Option<&CachedImage> {
+ self.entries
+ .get(&Self::key(r, raw_bytes))?
+ .iter()
+ .find(|entry| entry.raw_bytes == *raw_bytes)
+ }
+
+ fn insert(&mut self, r: &ImageRef, entry: CachedImage) {
+ self.entries
+ .entry(Self::key(r, &entry.raw_bytes))
+ .or_default()
+ .push(entry);
+ }
+}
+
pub(crate) struct RenderedImages {
pub images: Vec<ExtractedImage>,
pub error_count: u32,
@@ -538,23 +580,17 @@ fn render_page_images(
page: &Page,
page_number: u32,
refs: &[ImageRef],
- cache: &mut Vec<CachedImage>,
+ cache: &mut ImageCache,
) -> RenderedImages {
let mut out = Vec::with_capacity(refs.len());
let mut error_count = 0;
for r in refs {
if let Some(raw_bytes) = r.raw_bytes.as_ref()
- && let Some(cached) = cache.iter().find(|entry| {
- entry.raw_bytes == *raw_bytes
- && entry.width == r.pixel_width
- && entry.height == r.pixel_height
- && entry.bits_per_pixel == r.bits_per_pixel
- && entry.colorspace == r.colorspace
- })
+ && let Some(cached) = cache.get(r, raw_bytes)
{
out.push(ExtractedImage {
id: r.id.clone(),
- name: format!("image_{}.{}", r.id, cached.format),
+ name: format!("img_{}.{}", r.id, cached.format),
path: None,
page: page_number,
bbox: r.bbox.clone(),
@@ -563,7 +599,7 @@ fn render_page_images(
rotation: r.rotation,
format: cached.format.clone(),
duplicate_of: Some(cached.id.clone()),
- bytes: cached.bytes.clone(),
+ bytes: std::sync::Arc::clone(&cached.bytes),
});
continue;
}
@@ -594,9 +630,10 @@ fn render_page_images(
continue;
}
};
+ let bytes = std::sync::Arc::new(bytes);
out.push(ExtractedImage {
id: r.id.clone(),
- name: format!("image_{}.{}", r.id, format),
+ name: format!("img_{}.{}", r.id, format),
path: None,
page: page_number,
bbox: r.bbox.clone(),
@@ -605,19 +642,18 @@ fn render_page_images(
rotation: r.rotation,
format: format.clone(),
duplicate_of: None,
- bytes: bytes.clone(),
+ bytes: std::sync::Arc::clone(&bytes),
});
if let Some(raw_bytes) = r.raw_bytes.clone() {
- cache.push(CachedImage {
- raw_bytes,
- id: r.id.clone(),
- format,
- bytes,
- width: r.pixel_width,
- height: r.pixel_height,
- bits_per_pixel: r.bits_per_pixel,
- colorspace: r.colorspace,
- });
+ cache.insert(
+ r,
+ CachedImage {
+ raw_bytes,
+ id: r.id.clone(),
+ format,
+ bytes,
+ },
+ );
}
}
RenderedImages {
@@ -657,7 +693,9 @@ fn extract_page_image_refs(
.into_iter()
.enumerate()
.map(|(i, image)| ImageRef {
- id: format!("p{}_{}", page_number, i),
+ // 1-based image number to match the platform extractor's
+ // `img_p%d_%d` naming (C `imageNum` starts at 1).
+ id: format!("p{}_{}", page_number, i + 1),
bbox: Rect {
x: image.bounds.x,
y: image.bounds.y,
diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs
index 678be9e..3696544 100644
--- a/crates/liteparse/src/main.rs
+++ b/crates/liteparse/src/main.rs
@@ -100,7 +100,7 @@ struct ParseCommand {
num_workers: Option<usize>,
/// How to surface raster images in markdown output:
- /// `off` strips them, `placeholder` (default) emits `![](image_pN_K.png)`
+ /// `off` strips them, `placeholder` (default) emits `![](img_pN_K.png)`
/// references in reading order, and `embed` preserves the same presentation.
/// Use `--extract-images` to extract image bytes and metadata.
#[arg(long, default_value = "placeholder")]
diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rs
index f2a7505..be06fee 100644
--- a/crates/liteparse/src/markdown_layout/blocks.rs
+++ b/crates/liteparse/src/markdown_layout/blocks.rs
@@ -49,7 +49,7 @@ pub enum Block {
/// page's vector graphics (e.g. divider line between sections).
HorizontalRule,
/// Reference to a raster image on the page. Rendered as
- /// `![](image_{id}.{format})`. Suppressed entirely when `ImageMode::Off`.
+ /// `![](img_{id}.{format})`. Suppressed entirely when `ImageMode::Off`.
Figure {
id: String,
format: String,
@@ -240,7 +240,7 @@ pub fn render_blocks(blocks: &[Block]) -> String {
out.push_str("---");
}
Block::Figure { id, format } => {
- out.push_str("![](image_");
+ out.push_str("![](img_");
out.push_str(id);
out.push('.');
out.push_str(format);
@@ -280,10 +280,10 @@ mod tests {
fn render_figure_uses_extracted_format() {
assert_eq!(
render_blocks(&[Block::Figure {
- id: "p1_0".into(),
+ id: "p1_1".into(),
format: "jpg".into(),
}]),
- "![](image_p1_0.jpg)"
+ "![](img_p1_1.jpg)"
);
}
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index 866c640..79f9ec4 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -419,12 +419,16 @@ pub(crate) fn render_pages_for_ocr(
pages: &[Page],
dpi: f32,
grayscale: bool,
+ render_form_fields: bool,
) -> Result<Vec<RenderedPage>, LiteParseError> {
let mut rendered = Vec::new();
- // Draw form-field appearances into the OCR raster so filled-in form
- // values are visible to the OCR engine (matches the LlamaParse extract
- // binary's screenshot rendering).
- let form = document.form_environment();
+ // With `render_form_fields`, draw form-field appearances into the OCR
+ // raster so filled-in form values are visible to the OCR engine (matches
+ // the LlamaParse extract binary's screenshot rendering). Opt-in because
+ // the form environment runs the document's open/JS actions.
+ let form = render_form_fields
+ .then(|| document.form_environment())
+ .flatten();
if let Some(form) = form.as_ref() {
form.run_document_actions();
}
diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs
index a707ab3..34fe2bb 100644
--- a/crates/liteparse/src/output/json.rs
+++ b/crates/liteparse/src/output/json.rs
@@ -119,36 +119,35 @@ pub(crate) fn build_json(pages: &[ParsedPage], extract_text_metadata: bool) -> P
text_items: page
.text_items
.iter()
- .map(|item| JsonTextItem {
- text: item.text.clone(),
- x: item.x,
- y: item.y,
- width: item.width,
- height: item.height,
- rotation: Some(item.rotation),
- font_name: item.font_name.clone(),
- font_size: item.font_size,
- font_height: extract_text_metadata.then_some(item.font_height).flatten(),
- font_ascent: extract_text_metadata.then_some(item.font_ascent).flatten(),
- font_descent: extract_text_metadata.then_some(item.font_descent).flatten(),
- font_weight: extract_text_metadata.then_some(item.font_weight).flatten(),
- text_width: extract_text_metadata.then_some(item.text_width).flatten(),
- font_is_buggy: extract_text_metadata.then_some(item.font_is_buggy),
- mcid: extract_text_metadata.then_some(item.mcid).flatten(),
- fill_color: extract_text_metadata
- .then(|| item.fill_color.clone())
- .flatten(),
- stroke_color: extract_text_metadata
- .then(|| item.stroke_color.clone())
- .flatten(),
- char_codes: if extract_text_metadata {
- item.char_codes.clone()
- } else {
- Vec::new()
- },
- trailing_space_generated: extract_text_metadata
- && item.trailing_space_generated,
- confidence: item.confidence.or(Some(1.0)),
+ .map(|item| {
+ let meta = item.text_metadata(extract_text_metadata);
+ JsonTextItem {
+ text: item.text.clone(),
+ x: item.x,
+ y: item.y,
+ width: item.width,
+ height: item.height,
+ // Not part of TextMetadata: the API surfaces always
+ // expose rotation (it predates the metadata flag);
+ // only the CLI JSON gates it to stay byte-stable.
+ rotation: extract_text_metadata.then_some(item.rotation),
+ font_name: item.font_name.clone(),
+ font_size: item.font_size,
+ font_height: meta.font_height,
+ font_ascent: meta.font_ascent,
+ font_descent: meta.font_descent,
+ font_weight: meta.font_weight,
+ text_width: meta.text_width,
+ font_is_buggy: meta.font_is_buggy,
+ mcid: meta.mcid,
+ fill_color: meta.fill_color.map(str::to_owned),
+ stroke_color: meta.stroke_color.map(str::to_owned),
+ char_codes: meta.char_codes.map(<[u32]>::to_vec).unwrap_or_default(),
+ trailing_space_generated: meta
+ .trailing_space_generated
+ .unwrap_or(false),
+ confidence: item.confidence.or(Some(1.0)),
+ }
})
.collect(),
complexity: page.complexity.clone(),
@@ -359,7 +358,7 @@ mod tests {
let value: serde_json::Value =
serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();
let item = &value["pages"][0]["text_items"][0];
- assert_eq!(item["rotation"], 0.0);
+ assert!(item.get("rotation").is_none());
assert!(item.get("font_height").is_none());
assert!(item.get("font_is_buggy").is_none());
assert!(item.get("mcid").is_none());
@@ -371,8 +370,8 @@ mod tests {
fn test_format_json_result_includes_image_metadata_and_errors() {
let image = ExtractedImage {
id: "p2_0".into(),
- name: "image_p2_0.jpg".into(),
- path: Some("/tmp/images/image_p2_0.jpg".into()),
+ name: "img_p2_1.jpg".into(),
+ path: Some("/tmp/images/img_p2_1.jpg".into()),
page: 2,
bbox: Rect {
x: 10.0,
@@ -385,7 +384,7 @@ mod tests {
rotation: 90.0,
format: "jpg".into(),
duplicate_of: Some("p1_0".into()),
- bytes: vec![1, 2, 3],
+ bytes: std::sync::Arc::new(vec![1, 2, 3]),
};
let result = crate::parser::ParseResult {
pages: vec![],
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 5ef938b..c221f38 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -77,25 +77,74 @@ fn write_extracted_images(
use std::path::Path;
std::fs::create_dir_all(output_dir)?;
- let mut written: HashMap<String, (String, String)> = HashMap::new();
+ // Platform contract (mirrors the LlamaParse C extractor, which the worker
+ // pipeline is built around): only the canonical file is written; every
+ // duplicate placement keeps its own `name` but points `path` at the
+ // canonical file. Markdown figure references are rewritten to the
+ // canonical name (`rewrite_duplicate_image_refs`) so they only ever
+ // reference files that exist.
+ let mut written: HashMap<String, String> = HashMap::new();
for image in images {
if let Some(canonical) = image.duplicate_of.as_ref()
- && let Some((name, path)) = written.get(canonical)
+ && let Some(path) = written.get(canonical)
{
- image.name = name.clone();
image.path = Some(path.clone());
continue;
}
let path = Path::new(output_dir).join(&image.name);
- std::fs::write(&path, &image.bytes)?;
+ std::fs::write(&path, image.bytes.as_slice())?;
let path = path.to_string_lossy().into_owned();
image.path = Some(path.clone());
- written.insert(image.id.clone(), (image.name.clone(), path));
+ written.insert(image.id.clone(), path);
}
Ok(())
}
+/// Rewrite markdown figure references for deduplicated images to the
+/// canonical entry's file name. The markdown emitter references each figure
+/// by its own placement id (`![](img_p2_1.jpg)`), but only the canonical
+/// file is written to disk (see `write_extracted_images`), so duplicate
+/// placements must reference the canonical name — the same resolution the
+/// LlamaParse worker applies via `resolveOutputImageName`.
+fn rewrite_duplicate_image_refs(
+ pages: &mut [ParsedPage],
+ full_text: &mut String,
+ images: &[ExtractedImage],
+) {
+ use std::collections::HashMap;
+
+ let by_id: HashMap<&str, &ExtractedImage> = images
+ .iter()
+ .map(|image| (image.id.as_str(), image))
+ .collect();
+ let renames: Vec<(String, String)> = images
+ .iter()
+ .filter_map(|image| {
+ let canonical = by_id.get(image.duplicate_of.as_ref()?.as_str())?;
+ Some((
+ format!("![](img_{}.{})", image.id, image.format),
+ format!("![]({})", canonical.name),
+ ))
+ })
+ .collect();
+ if renames.is_empty() {
+ return;
+ }
+
+ for markdown in pages
+ .iter_mut()
+ .map(|page| &mut page.markdown)
+ .chain(std::iter::once(full_text))
+ {
+ for (from, to) in &renames {
+ if markdown.contains(from.as_str()) {
+ *markdown = markdown.replace(from.as_str(), to);
+ }
+ }
+ }
+}
+
/// Build the default glyph resolver from the environment, if configured.
#[cfg(not(target_arch = "wasm32"))]
fn default_glyph_resolver() -> Option<std::sync::Arc<dyn crate::GlyphResolver>> {
@@ -179,9 +228,10 @@ impl LiteParse {
}
fn validate_output_config(&self) -> Result<(), LiteParseError> {
- if self.config.image_output_dir.is_some() && !self.config.extract_images {
+ if self.config.image_output_dir.is_some() && !self.config.effective_extract_images() {
return Err(LiteParseError::Config(
- "image_output_dir requires extract_images = true".to_string(),
+ "image_output_dir requires extract_images = true (or image_mode = embed)"
+ .to_string(),
));
}
Ok(())
@@ -432,7 +482,7 @@ impl LiteParse {
self.glyph_resolver.as_deref(),
extract::ExtractionOutputOptions {
extract_content_bounds: self.config.extract_content_bounds,
- extract_images: self.config.extract_images,
+ extract_images: self.config.effective_extract_images(),
emit_word_boxes: self.config.emit_word_boxes,
extract_text_metadata: self.config.extract_text_metadata,
extract_vector_graphics: self.config.extract_vector_graphics,
@@ -453,6 +503,7 @@ impl LiteParse {
&pages,
self.config.dpi,
ocr_grayscale,
+ self.config.render_form_fields,
)?;
log(&format!(
"[liteparse] ocr render: {:.1}ms ({} pages)",
@@ -538,7 +589,7 @@ impl LiteParse {
t2.duration_since(t_ocr).as_secs_f64() * 1000.0
));
- let full_text = if self.config.output_format == crate::config::OutputFormat::Markdown {
+ let mut full_text = if self.config.output_format == crate::config::OutputFormat::Markdown {
let page_md =
markdown::format_markdown_pages(&parsed_pages, &outline, self.config.image_mode);
let md = page_md.join("\n\n-----\n\n");
@@ -558,12 +609,15 @@ impl LiteParse {
.collect::<Vec<_>>()
.join("\n\n")
};
+ if self.config.output_format == crate::config::OutputFormat::Markdown {
+ rewrite_duplicate_image_refs(&mut parsed_pages, &mut full_text, &images);
+ }
let total = web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;
log(&format!("[liteparse] total: {:.1}ms", total));
#[cfg(not(target_arch = "wasm32"))]
- if self.config.extract_images
+ if self.config.effective_extract_images()
&& let Some(output_dir) = self.config.image_output_dir.as_deref()
{
write_extracted_images(output_dir, &mut images)?;
@@ -665,6 +719,7 @@ impl LiteParse {
self.config.dpi,
self.config.password.as_deref(),
self.config.detect_screenshot_rects,
+ self.config.render_form_fields,
)?;
Ok(rendered
@@ -757,18 +812,23 @@ mod tests {
}
#[test]
- fn image_extraction_is_explicitly_opt_in() {
- let options = extract::ExtractionOutputOptions {
- extract_images: true,
- ..Default::default()
- };
- assert!(options.extract_images);
+ fn image_extraction_is_opt_in_but_embed_mode_implies_it() {
+ let default = LiteParseConfig::default();
+ assert!(!default.effective_extract_images());
+ // `image_mode = embed` predates `extract_images` and must keep
+ // extracting bytes for existing callers.
let embed = LiteParseConfig {
image_mode: crate::config::ImageMode::Embed,
..Default::default()
};
- assert!(!embed.extract_images);
+ assert!(embed.effective_extract_images());
+
+ let explicit = LiteParseConfig {
+ extract_images: true,
+ ..Default::default()
+ };
+ assert!(explicit.effective_extract_images());
}
#[test]
@@ -779,16 +839,23 @@ mod tests {
});
assert_eq!(
parser.validate_output_config().unwrap_err().to_string(),
- "invalid config: image_output_dir requires extract_images = true"
+ "invalid config: image_output_dir requires extract_images = true (or image_mode = embed)"
);
+
+ let embed = LiteParse::new(LiteParseConfig {
+ image_mode: crate::config::ImageMode::Embed,
+ image_output_dir: Some("images".into()),
+ ..Default::default()
+ });
+ assert!(embed.validate_output_config().is_ok());
}
#[test]
- fn image_output_reuses_canonical_file_for_duplicates() {
+ fn image_output_writes_duplicates_from_canonical_bytes() {
fn image(id: &str, duplicate_of: Option<&str>, bytes: &[u8]) -> ExtractedImage {
ExtractedImage {
id: id.into(),
- name: format!("image_{id}.png"),
+ name: format!("img_{id}.png"),
path: None,
page: 1,
bbox: crate::types::Rect::default(),
@@ -797,18 +864,21 @@ mod tests {
rotation: 0.0,
format: "png".into(),
duplicate_of: duplicate_of.map(str::to_owned),
- bytes: bytes.to_vec(),
+ bytes: std::sync::Arc::new(bytes.to_vec()),
}
}
let dir = tempfile::tempdir().unwrap();
let mut images = vec![
- image("p1_0", None, b"canonical"),
- image("p2_0", Some("p1_0"), b"canonical"),
+ image("p1_1", None, b"canonical"),
+ image("p2_1", Some("p1_1"), b"canonical"),
];
write_extracted_images(dir.path().to_str().unwrap(), &mut images).unwrap();
- assert_eq!(images[0].name, images[1].name);
+ // Platform contract: one file on disk; the duplicate keeps its own
+ // placement `name` but shares the canonical file's `path`.
+ assert_eq!(images[0].name, "img_p1_1.png");
+ assert_eq!(images[1].name, "img_p2_1.png");
assert_eq!(images[0].path, images[1].path);
assert_eq!(
std::fs::read(images[0].path.as_ref().unwrap()).unwrap(),
@@ -816,4 +886,56 @@ mod tests {
);
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);
}
+
+ #[test]
+ fn duplicate_image_markdown_refs_are_rewritten_to_canonical() {
+ fn image(id: &str, format: &str, duplicate_of: Option<&str>) -> ExtractedImage {
+ ExtractedImage {
+ id: id.into(),
+ name: format!("img_{id}.{format}"),
+ path: None,
+ page: 1,
+ bbox: crate::types::Rect::default(),
+ width: 2,
+ height: 2,
+ rotation: 0.0,
+ format: format.into(),
+ duplicate_of: duplicate_of.map(str::to_owned),
+ bytes: std::sync::Arc::new(Vec::new()),
+ }
+ }
+
+ let mut pages = vec![ParsedPage {
+ page_number: 1,
+ page_width: 612.0,
+ page_height: 792.0,
+ content_bounds: None,
+ text: String::new(),
+ markdown: "intro\n\n![](img_p2_1.jpg)\n\noutro".into(),
+ text_items: vec![],
+ projected_lines: vec![],
+ regions: crate::types::Region::default(),
+ graphics: vec![],
+ vector_graphics: None,
+ figures: vec![],
+ struct_nodes: vec![],
+ image_refs: vec![],
+ complexity: None,
+ annotations: None,
+ form_fields: None,
+ structure_tree: None,
+ }];
+ let mut full_text = pages[0].markdown.clone();
+ let images = vec![
+ image("p1_1", "jpg", None),
+ image("p2_1", "jpg", Some("p1_1")),
+ ];
+
+ rewrite_duplicate_image_refs(&mut pages, &mut full_text, &images);
+
+ // The duplicate's ref now points at the canonical file; canonical
+ // refs and surrounding text are untouched.
+ assert_eq!(pages[0].markdown, "intro\n\n![](img_p1_1.jpg)\n\noutro");
+ assert_eq!(full_text, pages[0].markdown);
+ }
}
diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rs
index 36df4a7..3f46860 100644
--- a/crates/liteparse/src/render.rs
+++ b/crates/liteparse/src/render.rs
@@ -21,9 +21,10 @@ pub struct RenderedPage {
/// Render selected pages from a PDF input to PNG bytes.
///
-/// Form-field appearances (filled values, checkbox states) are drawn on top
-/// of the page raster when the document has a form. With `detect_rects`,
-/// each page's raster is also scanned for solid rectangles and lines.
+/// With `render_form_fields`, form-field appearances (filled values, checkbox
+/// states) are drawn on top of the page raster; this runs the document's
+/// open/JS actions, so it is opt-in. With `detect_rects`, each page's raster
+/// is also scanned for solid rectangles and lines.
///
/// Acquires the process-global PDFium lock for the entire render. The lock
/// is held until this function returns — PNG encoding happens inside the
@@ -35,10 +36,17 @@ pub fn render_pages_to_png(
dpi: f32,
password: Option<&str>,
detect_rects: bool,
+ render_form_fields: bool,
) -> Result<Vec<RenderedPage>, LiteParseError> {
let lib = Library::init();
let document = load_document_from_input(&lib, input, password)?;
- render_document_pages(&document, page_numbers, dpi, detect_rects)
+ render_document_pages(
+ &document,
+ page_numbers,
+ dpi,
+ detect_rects,
+ render_form_fields,
+ )
}
fn render_document_pages(
@@ -46,6 +54,7 @@ fn render_document_pages(
page_numbers: Option<&[u32]>,
dpi: f32,
detect_rects: bool,
+ render_form_fields: bool,
) -> Result<Vec<RenderedPage>, LiteParseError> {
let page_count = document.page_count() as u32;
let pages: Vec<u32> = match page_numbers {
@@ -53,7 +62,9 @@ fn render_document_pages(
None => (1..=page_count).collect(),
};
- let form = document.form_environment();
+ let form = render_form_fields
+ .then(|| document.form_environment())
+ .flatten();
if let Some(form) = form.as_ref() {
form.run_document_actions();
}
@@ -236,7 +247,7 @@ pub fn screenshot(
password: Option<&str>,
) -> Result<(), LiteParseError> {
let input = PdfInput::Path(pdf_path.to_string());
- let pages = render_pages_to_png(&input, Some(&[page_num]), dpi, password, false)?;
+ let pages = render_pages_to_png(&input, Some(&[page_num]), dpi, password, false, false)?;
let page = pages
.into_iter()
.next()
diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs
index beb8496..a0c8a3c 100644
--- a/crates/liteparse/src/types.rs
+++ b/crates/liteparse/src/types.rs
@@ -87,6 +87,66 @@ pub struct TextItem {
pub words: Vec<WordBox>,
}
+/// The `TextItem` fields governed by `extract_text_metadata`, pre-gated by
+/// [`TextItem::text_metadata`]. This is the single source of truth for which
+/// fields count as rich text metadata: every output surface (CLI JSON, napi,
+/// python, wasm) builds its public items from this struct, so a new metadata
+/// field added here is a compile error in each surface until it is wired
+/// through — instead of a silent schema drift between bindings.
+///
+/// All fields are `Option`/slice so "extraction disabled" is representable
+/// even for the plain-`bool` fields on `TextItem`: `None`/empty means the
+/// caller did not opt in and the field must be omitted from output.
+pub struct TextMetadata<'a> {
+ pub font_height: Option<f32>,
+ pub font_ascent: Option<f32>,
+ pub font_descent: Option<f32>,
+ pub font_weight: Option<i32>,
+ pub text_width: Option<f32>,
+ pub font_is_buggy: Option<bool>,
+ pub mcid: Option<i32>,
+ pub fill_color: Option<&'a str>,
+ pub stroke_color: Option<&'a str>,
+ pub char_codes: Option<&'a [u32]>,
+ pub trailing_space_generated: Option<bool>,
+}
+
+impl TextItem {
+ /// The rich-metadata view of this item: real values when `enabled`, all
+ /// absent otherwise. See [`TextMetadata`].
+ pub fn text_metadata(&self, enabled: bool) -> TextMetadata<'_> {
+ if enabled {
+ TextMetadata {
+ font_height: self.font_height,
+ font_ascent: self.font_ascent,
+ font_descent: self.font_descent,
+ font_weight: self.font_weight,
+ text_width: self.text_width,
+ font_is_buggy: Some(self.font_is_buggy),
+ mcid: self.mcid,
+ fill_color: self.fill_color.as_deref(),
+ stroke_color: self.stroke_color.as_deref(),
+ char_codes: Some(&self.char_codes),
+ trailing_space_generated: Some(self.trailing_space_generated),
+ }
+ } else {
+ TextMetadata {
+ font_height: None,
+ font_ascent: None,
+ font_descent: None,
+ font_weight: None,
+ text_width: None,
+ font_is_buggy: None,
+ mcid: None,
+ fill_color: None,
+ stroke_color: None,
+ char_codes: None,
+ trailing_space_generated: None,
+ }
+ }
+ }
+}
+
/// One word's bounding box within a `TextItem`, in the same viewport space
/// (top-left origin, 72 DPI) as the parent item. `text` is the word's content
/// with inter-word spaces excluded.
@@ -358,7 +418,7 @@ pub struct ParsedPage {
}
/// One embedded raster image on a page. `id` is a stable, page-scoped slug
-/// used as the markdown link target (e.g. `image_p1_0.png`). `obj_index` is
+/// used as the markdown link target (e.g. `img_p1_1.png`). `obj_index` is
/// the image's position among image page-objects, so a later embed pass can
/// re-open the document and pull pixel bytes with `render_image_object`.
#[doc(hidden)]
@@ -397,8 +457,10 @@ pub struct ExtractedImage {
pub format: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub duplicate_of: Option<String>,
+ /// Encoded image bytes. Shared (`Arc`) so duplicate entries reference the
+ /// canonical image's buffer instead of copying it per occurrence.
#[serde(skip)]
- pub bytes: Vec<u8>,
+ pub bytes: std::sync::Arc<Vec<u8>>,
}
#[doc(hidden)]
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index 6bb9834..8e2a1c2 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -35,7 +35,7 @@ export interface JsLiteParseConfig {
numWorkers?: number
/**
* How to surface raster images in markdown output: "off", "placeholder"
- * (default — emits `![](image_pN_K.png)` references with no bytes), or
+ * (default — emits `![](img_pN_K.png)` references with no bytes), or
* "embed" (same presentation as placeholder; extraction is independent).
*/
imageMode?: string
@@ -72,6 +72,8 @@ export interface JsLiteParseConfig {
* them to each screenshot result. Default false.
*/
detectScreenshotRects?: boolean
+ /** Draw AcroForm field appearances into rendered rasters (runs document open/JS actions). Default false. */
+ renderFormFields?: boolean
/**
* Whether a systemic OCR failure aborts the whole parse (default true).
* Set false to keep already-recovered native text and return partial
diff --git a/packages/node/src/cli-json.ts b/packages/node/src/cli-json.ts
index 3ff7fa8..5ff590b 100644
--- a/packages/node/src/cli-json.ts
+++ b/packages/node/src/cli-json.ts
@@ -7,16 +7,19 @@ import type {
TextItem,
VectorGraphics,
StructureTreeElement,
+ XfaPacket,
} from "./lib.js";
-function textItemToCliJson(item: TextItem) {
+function textItemToCliJson(item: TextItem, extractTextMetadata: boolean) {
return {
text: item.text,
x: item.x,
y: item.y,
width: item.width,
height: item.height,
- ...(item.rotation !== undefined ? { rotation: item.rotation } : {}),
+ ...(extractTextMetadata && item.rotation !== undefined
+ ? { rotation: item.rotation }
+ : {}),
...(item.fontName !== undefined ? { font_name: item.fontName } : {}),
...(item.fontSize !== undefined ? { font_size: item.fontSize } : {}),
...(item.fontHeight !== undefined ? { font_height: item.fontHeight } : {}),
@@ -169,6 +172,15 @@ function formFieldToCliJson(field: FormField) {
};
}
+function xfaPacketToCliJson(packet: XfaPacket) {
+ return {
+ index: packet.index,
+ ...(packet.name !== undefined ? { name: packet.name } : {}),
+ content_length: packet.contentLength,
+ ...(packet.content !== undefined ? { content: packet.content } : {}),
+ };
+}
+
function imageToCliJson(image: ExtractedImage) {
return {
id: image.id,
@@ -192,14 +204,30 @@ function imageToCliJson(image: ExtractedImage) {
}
/** Project the camelCase Node API result into the Rust CLI JSON schema. */
-export function parseResultToCliJson(result: ParseResult) {
+export function parseResultToCliJson(
+ result: ParseResult,
+ options: { extractTextMetadata?: boolean } = {},
+) {
+ const extractTextMetadata = options.extractTextMetadata ?? false;
return {
pages: result.pages.map((page) => ({
page: page.pageNum,
width: page.width,
height: page.height,
+ ...(page.contentBounds
+ ? {
+ content_bounds: {
+ x: page.contentBounds.x,
+ y: page.contentBounds.y,
+ width: page.contentBounds.width,
+ height: page.contentBounds.height,
+ },
+ }
+ : {}),
text: page.text,
- text_items: page.textItems.map(textItemToCliJson),
+ text_items: page.textItems.map((item) =>
+ textItemToCliJson(item, extractTextMetadata),
+ ),
...(page.complexity
? { complexity: complexityToCliJson(page.complexity) }
: {}),
@@ -227,5 +255,8 @@ export function parseResultToCliJson(result: ParseResult) {
? { image_error_count: result.imageErrorCount }
: {}),
...(result.formType !== undefined ? { form_type: result.formType } : {}),
+ ...(result.xfaPackets !== undefined
+ ? { xfa_packets: result.xfaPackets.map(xfaPacketToCliJson) }
+ : {}),
};
}
diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts
index 146866c..b183564 100644
--- a/packages/node/src/cli.ts
+++ b/packages/node/src/cli.ts
@@ -158,7 +158,9 @@ program
const output =
config.outputFormat === "json"
? JSON.stringify(
- parseResultToCliJson(result),
+ parseResultToCliJson(result, {
+ extractTextMetadata: config.extractTextMetadata,
+ }),
null,
2,
)
@@ -425,7 +427,9 @@ program
const output =
format === "json"
? JSON.stringify(
- parseResultToCliJson(result),
+ parseResultToCliJson(result, {
+ extractTextMetadata: config.extractTextMetadata,
+ }),
null,
2,
)
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index ed6678b..8fd2c9a 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -50,6 +50,8 @@ export interface LiteParseConfig {
extractContentBounds: boolean;
/** Detect solid rectangles/lines in rendered page screenshots (default: false). */
detectScreenshotRects: boolean;
+ /** Draw AcroForm field appearances into rendered rasters — screenshots and OCR inputs (runs document open/JS actions; default: false). */
+ renderFormFields: boolean;
preserveVerySmallText: boolean;
password?: string;
quiet: boolean;
@@ -316,7 +318,7 @@ export interface FormField {
}
export interface ExtractedImage {
- /** Reference id used in the markdown output (e.g. `![](image_p1_0.png)` → `"p1_0"`). */
+ /** Reference id used in the markdown output (e.g. `![](img_p1_1.png)` → `"p1_1"`). */
id: string;
/** File name used when `imageOutputDir` is configured. */
name: string;
@@ -490,6 +492,7 @@ export class LiteParse {
extractXfaPackets: userConfig.extractXfaPackets,
extractContentBounds: userConfig.extractContentBounds,
detectScreenshotRects: userConfig.detectScreenshotRects,
+ renderFormFields: userConfig.renderFormFields,
preserveVerySmallText: userConfig.preserveVerySmallText,
password: userConfig.password,
quiet: userConfig.quiet,
@@ -528,6 +531,7 @@ export class LiteParse {
extractXfaPackets: resolved.extractXfaPackets ?? false,
extractContentBounds: resolved.extractContentBounds ?? false,
detectScreenshotRects: resolved.detectScreenshotRects ?? false,
+ renderFormFields: resolved.renderFormFields ?? false,
preserveVerySmallText: resolved.preserveVerySmallText ?? false,
password: resolved.password ?? undefined,
quiet: resolved.quiet ?? false,
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index 1d5a5e9..cb4d06f 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -39,6 +39,7 @@ export interface LiteParseNativeConfig {
extractXfaPackets?: boolean;
extractContentBounds?: boolean;
detectScreenshotRects?: boolean;
+ renderFormFields?: boolean;
preserveVerySmallText?: boolean;
password?: string;
quiet?: boolean;
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index 9cd68db..3b78f9a 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -371,6 +371,7 @@ class LiteParse:
extract_xfa_packets: Optional[bool] = None,
extract_content_bounds: Optional[bool] = None,
detect_screenshot_rects: Optional[bool] = None,
+ render_form_fields: Optional[bool] = None,
ocr_failure_fatal: Optional[bool] = None,
ocr_hedge_delays_ms: Optional[List[int]] = None,
emit_word_boxes: Optional[bool] = None,
@@ -494,6 +495,8 @@ class LiteParse:
kwargs["extract_content_bounds"] = extract_content_bounds
if detect_screenshot_rects is not None:
kwargs["detect_screenshot_rects"] = detect_screenshot_rects
+ if render_form_fields is not None:
+ kwargs["render_form_fields"] = render_form_fields
if ocr_failure_fatal is not None:
kwargs["ocr_failure_fatal"] = ocr_failure_fatal
if ocr_hedge_delays_ms is not None:
diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py
index fe0e2b8..247e8e1 100644
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@ -189,7 +189,7 @@ class ExtractedImage:
Populated only when ``extract_images=True``. ``image_mode`` controls
Markdown presentation independently.
The ``id`` matches the reference used in the markdown output
- (e.g. ``![](image_p1_0.png)`` → ``id="p1_0"``).
+ (e.g. ``![](img_p1_1.png)`` → ``id="p1_1"``).
"""
id: str
name: str