Skip to content

A fast, helpful, and open-source document parser

Converge extraction output options and schemas

Pierre-Loic doulcet8c31aa2df22b
diff --git a/README.md b/README.md
index beb2d96..3de0792 100644
--- a/README.md
+++ b/README.md
@@ -162,6 +162,9 @@ lit parse document.pdf --no-ocr
# Include page-scoped vector path data in JSON
lit parse document.pdf --format json --extract-vector-graphics
+# Include rich per-item PDF text metadata
+lit parse document.pdf --format json --extract-text-metadata
+
# Include page annotations in structured JSON
lit parse document.pdf --format json --extract-annotations
@@ -184,7 +187,7 @@ lit parse document.pdf --format markdown -o output.md
lit parse document.pdf --format markdown --image-mode off
# Extract embedded images to disk and reference them from the markdown
-lit parse document.pdf --format markdown --image-mode embed --image-output-dir ./images
+lit parse document.pdf --format markdown --image-mode embed --extract-images --image-output-dir ./images
# Extract image bytes and metadata without changing Markdown image handling
lit parse document.pdf --format json --extract-images
@@ -199,17 +202,17 @@ Image handling is controlled by `--image-mode`:
|------|----------|
| `placeholder` (default) | Emits `![](image_pN_K.png)` references in reading order |
| `off` | Strips images entirely |
-| `embed` | Returns image bytes and metadata; valid embedded JPEG streams remain JPEG |
+| `embed` | Emits the same image references as `placeholder` |
-`--image-output-dir` enables extraction independently of the Markdown mode. JSON output
+`--extract-images` is the only option that enables embedded-image extraction.
+`--image-output-dir` requires it and writes the extracted bytes to disk. JSON output
contains each image's `name`, `path`, page bbox, intrinsic pixel dimensions, rotation,
format, and duplicate relationship; pixel bytes are never embedded in JSON. Identical
image resources reuse the same output file.
Library callers can opt in with `extract_images: true` (Rust), `extractImages: true`
-(Node/WASM), or `extract_images=True` (Python). It defaults to false. For backward
-compatibility, `image_mode=embed` and a configured image output directory also imply
-extraction; the default placeholder mode alone only discovers lightweight page refs.
+(Node/WASM), or `extract_images=True` (Python). It defaults to false. Markdown image
+mode controls presentation only; placeholder refs are still discovered without bytes.
> Markdown reconstruction quality varies with document complexity. For the
> hardest documents (dense tables, multi-column layouts, scans),
@@ -298,7 +301,9 @@ Options:
--target-pages <pages> Pages to parse (e.g., "1-5,10,15-20")
--dpi <dpi> Rendering DPI [default: 150]
--image-mode <mode> Markdown image handling: off|placeholder|embed [default: placeholder]
- --image-output-dir <dir> Where to write images when --image-mode embed
+ --extract-images Extract embedded image bytes and metadata
+ --image-output-dir <dir> Write extracted images; requires --extract-images
+ --extract-text-metadata Include rich PDF text metadata in text items
--extract-vector-graphics Include page vector shapes and merged H/V lines
--no-links Emit link anchor text as plain text (no [text](url)) in markdown
--extract-annotations Include PDF annotations in page output
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs
index d8df0df..c6c28f2 100644
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@ -44,12 +44,12 @@ pub struct JsLiteParseConfig {
pub num_workers: Option<u32>,
/// How to surface raster images in markdown output: "off", "placeholder"
/// (default — emits `![](image_pN_K.png)` references with no bytes), or
- /// "embed" (also returns each image's bytes and metadata on `images`).
+ /// "embed" (same presentation as placeholder; extraction is independent).
pub image_mode: Option<String>,
/// Extract embedded image bytes and metadata (default false).
pub extract_images: Option<bool>,
- /// Directory where embedded image files are written. Also enables image
- /// extraction even when `imageMode` is `placeholder`.
+ /// Directory where embedded image files are written. Requires
+ /// `extractImages` to be true.
pub image_output_dir: Option<String>,
/// Render hyperlink annotations as `[text](url)` in markdown output
/// (default true). Set false for plain anchor text.
@@ -69,7 +69,7 @@ pub struct JsLiteParseConfig {
/// for word-level bbox attribution.
pub emit_word_boxes: Option<bool>,
/// Include rich PDF text metadata on returned text items. Default false.
- pub include_text_metadata: Option<bool>,
+ pub extract_text_metadata: Option<bool>,
/// Restrict output to a page sub-region. Each field is the fraction of the
/// page cropped from that side; a text item survives only if it lies
/// entirely inside the remaining rectangle. Unset keeps the whole page.
@@ -170,8 +170,8 @@ impl JsLiteParseConfig {
if let Some(v) = self.emit_word_boxes {
cfg.emit_word_boxes = v;
}
- if let Some(v) = self.include_text_metadata {
- cfg.include_text_metadata = v;
+ if let Some(v) = self.extract_text_metadata {
+ cfg.extract_text_metadata = v;
}
if let Some(v) = self.crop_box {
cfg.crop_box = Some(CropBox {
@@ -233,7 +233,7 @@ impl JsLiteParseConfig {
.collect(),
),
emit_word_boxes: Some(cfg.emit_word_boxes),
- include_text_metadata: Some(cfg.include_text_metadata),
+ extract_text_metadata: Some(cfg.extract_text_metadata),
crop_box: cfg.crop_box.map(|c| JsCropBox {
top: c.top as f64,
right: c.right as f64,
@@ -298,7 +298,7 @@ pub struct JsTextItem {
/// Raw PDF content-stream character codes for the source glyphs.
pub char_codes: Option<Vec<u32>>,
/// True when the trailing source space was synthesized by PDFium.
- pub tsg: Option<bool>,
+ pub trailing_space_generated: Option<bool>,
pub confidence: Option<f64>,
/// Rotation in degrees (viewport space). Defaults to 0 when omitted.
pub rotation: Option<f64>,
@@ -328,7 +328,7 @@ impl JsTextItem {
fill_color: self.fill_color.clone(),
stroke_color: self.stroke_color.clone(),
char_codes: self.char_codes.clone().unwrap_or_default(),
- tsg: self.tsg.unwrap_or(false),
+ trailing_space_generated: self.trailing_space_generated.unwrap_or(false),
confidence: self.confidence.map(|v| v as f32),
..Default::default()
}
@@ -354,15 +354,15 @@ impl JsTextItem {
fill_color: item.fill_color.clone(),
stroke_color: item.stroke_color.clone(),
char_codes: Some(item.char_codes.clone()),
- tsg: Some(item.tsg),
+ trailing_space_generated: Some(item.trailing_space_generated),
confidence: item.confidence.map(|v| v as f64).or(Some(1.0)),
words: item.words.iter().map(JsWordBox::from_rust).collect(),
}
}
- fn from_rust_for_output(item: &TextItem, include_text_metadata: bool) -> Self {
+ fn from_rust_for_output(item: &TextItem, extract_text_metadata: bool) -> Self {
let mut output = Self::from_rust(item);
- if !include_text_metadata {
+ if !extract_text_metadata {
output.font_height = None;
output.font_ascent = None;
output.font_descent = None;
@@ -373,7 +373,7 @@ impl JsTextItem {
output.fill_color = None;
output.stroke_color = None;
output.char_codes = None;
- output.tsg = None;
+ output.trailing_space_generated = None;
}
output
}
@@ -639,7 +639,7 @@ impl JsDocumentAnnotation {
}
impl JsParsedPage {
- pub fn from_rust(page: &ParsedPage, include_text_metadata: bool) -> Self {
+ pub fn from_rust(page: &ParsedPage, extract_text_metadata: bool) -> Self {
Self {
page_num: page.page_number as u32,
width: page.page_width as f64,
@@ -649,7 +649,7 @@ impl JsParsedPage {
text_items: page
.text_items
.iter()
- .map(|item| JsTextItem::from_rust_for_output(item, include_text_metadata))
+ .map(|item| JsTextItem::from_rust_for_output(item, extract_text_metadata))
.collect(),
complexity: page
.complexity
@@ -805,7 +805,7 @@ impl JsParseResult {
pages: result
.pages
.iter()
- .map(|page| JsParsedPage::from_rust(page, config.include_text_metadata))
+ .map(|page| JsParsedPage::from_rust(page, config.extract_text_metadata))
.collect(),
text: result.text.clone(),
image_error_count: result.image_error_count,
@@ -853,20 +853,20 @@ mod tests {
fill_color: Some("ff112233".into()),
stroke_color: Some("ff445566".into()),
char_codes: vec![65, 32],
- tsg: true,
+ trailing_space_generated: true,
..Default::default()
};
let js = JsTextItem::from_rust(&item);
assert_eq!(js.char_codes, Some(vec![65, 32]));
- assert_eq!(js.tsg, Some(true));
+ assert_eq!(js.trailing_space_generated, Some(true));
assert_eq!(js.fill_color.as_deref(), Some("ff112233"));
let lightweight = JsTextItem::from_rust_for_output(&item, false);
assert_eq!(lightweight.font_height, None);
assert_eq!(lightweight.font_is_buggy, None);
assert_eq!(lightweight.char_codes, None);
- assert_eq!(lightweight.tsg, None);
+ assert_eq!(lightweight.trailing_space_generated, None);
let round_trip = js.to_rust();
assert_eq!(round_trip.font_height, Some(12.0));
@@ -878,15 +878,15 @@ mod tests {
assert_eq!(round_trip.mcid, Some(2));
assert_eq!(round_trip.stroke_color.as_deref(), Some("ff445566"));
assert_eq!(round_trip.char_codes, vec![65, 32]);
- assert!(round_trip.tsg);
+ assert!(round_trip.trailing_space_generated);
}
#[test]
fn text_metadata_config_defaults_off_and_round_trips() {
let mut js = JsLiteParseConfig::from_rust(&LiteParseConfig::default());
- assert_eq!(js.include_text_metadata, Some(false));
- js.include_text_metadata = Some(true);
- assert!(js.into_rust().include_text_metadata);
+ assert_eq!(js.extract_text_metadata, Some(false));
+ js.extract_text_metadata = Some(true);
+ assert!(js.into_rust().extract_text_metadata);
}
#[test]
diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rs
index df90656..0918b2e 100644
--- a/crates/liteparse-python/src/cli.rs
+++ b/crates/liteparse-python/src/cli.rs
@@ -62,14 +62,14 @@ struct ParseCommand {
#[arg(long)]
num_workers: Option<usize>,
/// How to surface raster images in markdown output: `off`, `placeholder`
- /// (default), or `embed` (extracts image bytes and metadata).
+ /// (default), or `embed`. Use `--extract-images` for bytes and metadata.
#[arg(long, default_value = "placeholder")]
image_mode: String,
#[arg(long)]
extract_images: bool,
/// Directory to write embedded images to. Valid source JPEGs keep their
- /// format; other images are PNG. Setting this enables extraction
- /// independently of image mode. Created if missing.
+ /// format; other images are PNG. Requires `--extract-images`. Created if
+ /// missing.
#[arg(long)]
image_output_dir: Option<String>,
/// Disable hyperlink extraction. By default URI link annotations render as
@@ -85,7 +85,7 @@ struct ParseCommand {
complexity: bool,
/// Include rich PDF text metadata in text items and JSON output.
#[arg(long)]
- include_text_metadata: bool,
+ extract_text_metadata: bool,
/// Include page-scoped vector shapes and merged horizontal/vertical lines.
#[arg(long)]
extract_vector_graphics: bool,
@@ -162,7 +162,7 @@ struct BatchParseCommand {
complexity: bool,
/// Include rich PDF text metadata in text items and JSON output.
#[arg(long)]
- include_text_metadata: bool,
+ extract_text_metadata: bool,
#[arg(long)]
extract_images: bool,
/// Include page-scoped vector shapes and merged horizontal/vertical lines.
@@ -249,7 +249,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
extract_links: !cmd.no_links,
extract_annotations: cmd.extract_annotations,
include_complexity: cmd.complexity,
- include_text_metadata: cmd.include_text_metadata,
+ extract_text_metadata: cmd.extract_text_metadata,
extract_vector_graphics: cmd.extract_vector_graphics,
..Default::default()
};
@@ -267,7 +267,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
&result.pages,
&result.images,
result.image_error_count,
- lp.config().include_text_metadata,
+ lp.config().extract_text_metadata,
)?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
@@ -340,7 +340,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
include_complexity: cmd.complexity,
- include_text_metadata: cmd.include_text_metadata,
+ extract_text_metadata: cmd.extract_text_metadata,
extract_images: cmd.extract_images,
extract_vector_graphics: cmd.extract_vector_graphics,
..Default::default()
@@ -387,7 +387,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
&result.pages,
&result.images,
result.image_error_count,
- lp.config().include_text_metadata,
+ lp.config().extract_text_metadata,
)
.map_err(|e| e.into()),
OutputFormat::Text => Ok(text::format_text(&result.pages)),
@@ -559,13 +559,13 @@ mod tests {
}
#[test]
- fn include_text_metadata_flag_is_available_for_parse_and_batch() {
- let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--include-text-metadata"])
+ fn extract_text_metadata_flag_is_available_for_parse_and_batch() {
+ let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--extract-text-metadata"])
.unwrap();
assert!(matches!(
cli.command,
Commands::Parse(ParseCommand {
- include_text_metadata: true,
+ extract_text_metadata: true,
..
})
));
@@ -575,13 +575,13 @@ mod tests {
"batch-parse",
"input",
"output",
- "--include-text-metadata",
+ "--extract-text-metadata",
])
.unwrap();
assert!(matches!(
cli.command,
Commands::BatchParse(BatchParseCommand {
- include_text_metadata: true,
+ extract_text_metadata: true,
..
})
));
diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs
index 8ee44bc..b1f2764 100644
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@ -91,7 +91,7 @@ struct PyTextItem {
char_codes: Vec<u32>,
/// True when the trailing source space was synthesized by PDFium.
#[pyo3(get)]
- tsg: bool,
+ trailing_space_generated: bool,
#[pyo3(get)]
confidence: Option<f64>,
/// Rotation in degrees (viewport space). Defaults to 0.
@@ -198,7 +198,7 @@ impl PyTextItem {
fill_color: self.fill_color.clone(),
stroke_color: self.stroke_color.clone(),
char_codes: self.char_codes.clone(),
- tsg: self.tsg,
+ trailing_space_generated: self.trailing_space_generated,
confidence: self.confidence.map(|v| v as f32),
..Default::default()
}
@@ -223,12 +223,32 @@ impl PyTextItem {
fill_color: item.fill_color,
stroke_color: item.stroke_color,
char_codes: item.char_codes,
- tsg: item.tsg,
+ trailing_space_generated: item.trailing_space_generated,
confidence: item.confidence.map(|v| v as f64).or(Some(1.0)),
rotation: item.rotation as f64,
words: item.words.into_iter().map(PyWordBox::from_rust).collect(),
}
}
+
+ fn from_rust_for_output(
+ mut item: liteparse::types::TextItem,
+ extract_text_metadata: bool,
+ ) -> Self {
+ if !extract_text_metadata {
+ item.font_height = None;
+ item.font_ascent = None;
+ item.font_descent = None;
+ item.font_weight = None;
+ item.text_width = None;
+ item.font_is_buggy = false;
+ item.mcid = None;
+ item.fill_color = None;
+ item.stroke_color = None;
+ item.char_codes.clear();
+ item.trailing_space_generated = false;
+ }
+ Self::from_rust(item)
+ }
}
#[pyclass(frozen, from_py_object)]
@@ -369,7 +389,7 @@ impl PyParsedPage {
}
impl PyParsedPage {
- fn from_rust(page: liteparse::types::ParsedPage) -> Self {
+ fn from_rust(page: liteparse::types::ParsedPage, extract_text_metadata: bool) -> Self {
Self {
page_num: page.page_number as u32,
width: page.page_width as f64,
@@ -379,7 +399,7 @@ impl PyParsedPage {
text_items: page
.text_items
.into_iter()
- .map(PyTextItem::from_rust)
+ .map(|item| PyTextItem::from_rust_for_output(item, extract_text_metadata))
.collect(),
complexity: page
.complexity
@@ -431,12 +451,12 @@ impl PyParseResult {
}
impl PyParseResult {
- fn from_rust(result: liteparse::parser::ParseResult) -> Self {
+ fn from_rust(result: liteparse::parser::ParseResult, extract_text_metadata: bool) -> Self {
Self {
pages: result
.pages
.into_iter()
- .map(PyParsedPage::from_rust)
+ .map(|page| PyParsedPage::from_rust(page, extract_text_metadata))
.collect(),
text: result.text,
images: result
@@ -729,7 +749,7 @@ struct PyLiteParseConfig {
#[pyo3(get)]
include_complexity: bool,
#[pyo3(get)]
- include_text_metadata: bool,
+ extract_text_metadata: bool,
#[pyo3(get)]
image_output_dir: Option<String>,
#[pyo3(get)]
@@ -788,7 +808,7 @@ impl PyLiteParseConfig {
.map(|c| (c.top, c.right, c.bottom, c.left)),
skip_diagonal_text: cfg.skip_diagonal_text,
include_complexity: cfg.include_complexity,
- include_text_metadata: cfg.include_text_metadata,
+ extract_text_metadata: cfg.extract_text_metadata,
image_output_dir: cfg.image_output_dir.clone(),
extract_images: cfg.extract_images,
extract_vector_graphics: cfg.extract_vector_graphics,
@@ -833,7 +853,7 @@ impl LiteParse {
ocr_failure_fatal = None,
ocr_hedge_delays_ms = None,
emit_word_boxes = None,
- include_text_metadata = None,
+ extract_text_metadata = None,
crop_box = None,
skip_diagonal_text = None,
include_complexity = None,
@@ -861,7 +881,7 @@ impl LiteParse {
ocr_failure_fatal: Option<bool>,
ocr_hedge_delays_ms: Option<Vec<u64>>,
emit_word_boxes: Option<bool>,
- include_text_metadata: Option<bool>,
+ extract_text_metadata: Option<bool>,
crop_box: Option<(f32, f32, f32, f32)>,
skip_diagonal_text: Option<bool>,
include_complexity: Option<bool>,
@@ -939,8 +959,8 @@ impl LiteParse {
if let Some(v) = emit_word_boxes {
cfg.emit_word_boxes = v;
}
- if let Some(v) = include_text_metadata {
- cfg.include_text_metadata = v;
+ if let Some(v) = extract_text_metadata {
+ cfg.extract_text_metadata = v;
}
if let Some((top, right, bottom, left)) = crop_box {
cfg.crop_box = Some(CropBox {
@@ -977,7 +997,10 @@ impl LiteParse {
let result = py
.detach(|| self.runtime.block_on(self.inner.parse_input(pdf_input)))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
- Ok(PyParseResult::from_rust(result))
+ Ok(PyParseResult::from_rust(
+ result,
+ self.config.extract_text_metadata,
+ ))
}
/// Parse a document from raw bytes.
@@ -986,7 +1009,10 @@ impl LiteParse {
let result = py
.detach(|| self.runtime.block_on(self.inner.parse_input(pdf_input)))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
- Ok(PyParseResult::from_rust(result))
+ Ok(PyParseResult::from_rust(
+ result,
+ self.config.extract_text_metadata,
+ ))
}
/// Determine per-page complexity for a document at the given path. Returns
@@ -1165,13 +1191,13 @@ mod tests {
fill_color: Some("ff112233".into()),
stroke_color: Some("ff445566".into()),
char_codes: vec![65, 32],
- tsg: true,
+ trailing_space_generated: true,
..Default::default()
};
let py = PyTextItem::from_rust(item);
assert_eq!(py.char_codes, vec![65, 32]);
- assert!(py.tsg);
+ assert!(py.trailing_space_generated);
assert_eq!(py.fill_color.as_deref(), Some("ff112233"));
let round_trip = py.to_rust();
@@ -1184,20 +1210,20 @@ mod tests {
assert_eq!(round_trip.mcid, Some(2));
assert_eq!(round_trip.stroke_color.as_deref(), Some("ff445566"));
assert_eq!(round_trip.char_codes, vec![65, 32]);
- assert!(round_trip.tsg);
+ assert!(round_trip.trailing_space_generated);
}
#[test]
fn text_metadata_config_defaults_off_and_can_be_enabled() {
let py = PyLiteParseConfig::from_rust(&LiteParseConfig::default());
- assert!(!py.include_text_metadata);
+ assert!(!py.extract_text_metadata);
let config = LiteParseConfig {
- include_text_metadata: true,
+ extract_text_metadata: true,
..Default::default()
};
let py = PyLiteParseConfig::from_rust(&config);
- assert!(py.include_text_metadata);
+ assert!(py.extract_text_metadata);
}
#[test]
diff --git a/crates/liteparse/README.md b/crates/liteparse/README.md
index 8016450..6f286dc 100644
--- a/crates/liteparse/README.md
+++ b/crates/liteparse/README.md
@@ -58,7 +58,7 @@ let config = LiteParseConfig {
output_format: OutputFormat::Json, // Json | Text | Markdown
extract_annotations: false, // Include page annotations in output
preserve_very_small_text: false, // Keep tiny text
- include_text_metadata: false, // Opt in to rich PDF text metadata
+ extract_text_metadata: false, // Opt in to rich PDF text metadata
password: None, // Password for protected documents
quiet: false, // Suppress progress output
..Default::default()
@@ -83,11 +83,10 @@ images, and links reconstructed from the spatial layout. Set
- `image_mode` (`ImageMode::Placeholder` default | `Off` | `Embed`) — how raster
images are surfaced in the output.
- `extract_images` (default `false`) — return embedded image bytes and metadata
- without changing Markdown image handling. `ImageMode::Embed` remains an
- extraction opt-in for backward compatibility.
+ without changing Markdown image handling. This is the only option that enables
+ extraction.
- `image_output_dir` — write extracted image files and return their names/paths;
- setting it enables extraction even without `ImageMode::Embed`. Duplicate image
- resources reuse the same file.
+ requires `extract_images: true`. Duplicate image resources reuse the same file.
- `extract_links` (default `true`) — render hyperlink annotations as
`[text](url)`; set `false` for plain anchor text.
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs
index 1cbabbe..488534d 100644
--- a/crates/liteparse/src/config.rs
+++ b/crates/liteparse/src/config.rs
@@ -31,15 +31,14 @@ pub struct LiteParseConfig {
pub quiet: bool,
/// Number of concurrent OCR workers. Defaults to (number of CPU cores - 1), minimum 1.
pub num_workers: usize,
- /// Controls how raster images are surfaced in markdown output. `Embed`
- /// also exposes image bytes and metadata to callers.
+ /// Controls how raster image references are surfaced in markdown output.
+ /// This does not enable embedded-image extraction.
pub image_mode: ImageMode,
/// Extract embedded image bytes and metadata into `ParseResult.images`.
- /// Defaults to false. `ImageMode::Embed` and `image_output_dir` also imply
- /// extraction for backward compatibility.
+ /// Defaults to false and is the only switch that enables extraction.
pub extract_images: bool,
- /// Directory where extracted embedded images are written. Setting this
- /// also enables image extraction, independently of `image_mode`.
+ /// Directory where extracted embedded images are written. Requires
+ /// `extract_images` to be true.
pub image_output_dir: Option<String>,
/// Extract hyperlink annotations and render them as `[text](url)` in
/// markdown output. Default on. Disable for benchmark parity with
@@ -74,10 +73,11 @@ pub struct LiteParseConfig {
/// Include rich PDF text metadata on public text items: MCID, glyph width,
/// font metrics/weight/buggy state, fill/stroke colors, raw character
/// codes, and generated-trailing-space state. Default `false` to preserve
- /// LiteParse's lightweight response shape. Extraction still computes the
- /// metadata for internal layout; it is stripped only before returning.
+ /// LiteParse's lightweight response shape. Layout-required metrics remain
+ /// available internally; source character codes and generated-space state
+ /// are not computed unless this is enabled.
#[serde(default)]
- pub include_text_metadata: bool,
+ pub extract_text_metadata: bool,
/// Restrict output to a sub-region of every page. Each field is the
/// fraction of the page to crop away from that side (e.g. `left = 0.5`
/// discards the left half). A text item is kept only when it lies
@@ -118,9 +118,8 @@ pub struct CropBox {
/// reading order at each image's y position, but do **not** extract or
/// return pixel bytes. Keeps response size small while letting the LLM see
/// where figures live in the document.
-/// * `Embed` — same references, plus bytes and metadata returned via
-/// `ParseResult.images`. Opt-in because pixel bytes can dwarf the text
-/// payload on image-heavy PDFs.
+/// * `Embed` — emit the same references as `Placeholder`. Embedded pixel bytes
+/// are controlled independently by `LiteParseConfig::extract_images`.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ImageMode {
@@ -172,7 +171,7 @@ impl Default for LiteParseConfig {
ocr_failure_fatal: true,
ocr_hedge_delays_ms: Vec::new(),
emit_word_boxes: false,
- include_text_metadata: false,
+ extract_text_metadata: false,
crop_box: None,
skip_diagonal_text: false,
include_complexity: false,
@@ -294,7 +293,7 @@ mod tests {
assert_eq!(c.output_format, OutputFormat::Json);
assert!(!c.preserve_very_small_text);
assert!(!c.quiet);
- assert!(!c.include_text_metadata);
+ assert!(!c.extract_text_metadata);
assert!(c.password.is_none());
assert!(!c.extract_images);
}
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index f9591dd..771ce9b 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -55,7 +55,6 @@ pub(crate) fn extract_pages_from_document(
target_pages,
max_pages,
false,
- false,
None,
ExtractionOutputOptions::default(),
)?
@@ -63,15 +62,14 @@ pub(crate) fn extract_pages_from_document(
}
/// Same as `extract_pages_from_document` but optionally also renders every
-/// raster image object to bytes (when `render_images = true`). Returned
+/// raster image object to bytes (when `output_options.extract_images` is true). Returned
/// `ExtractedImage`s carry the same ids the markdown emitter will reference,
-/// so callers can match them up by id. When `render_images = false` the
+/// so callers can match them up by id. When image extraction is disabled the
/// returned image vec is always empty.
pub(crate) fn extract_pages_and_images(
document: &Document,
target_pages: Option<&[u32]>,
max_pages: usize,
- render_images: bool,
extract_links: bool,
glyph_resolver: Option<&dyn crate::GlyphResolver>,
output_options: ExtractionOutputOptions,
@@ -109,6 +107,7 @@ pub(crate) fn extract_pages_and_images(
&view_box,
glyph_resolver,
output_options.emit_word_boxes,
+ output_options.extract_text_metadata,
)?;
if extract_links {
assign_links(&mut text_items, &page.links(&view_box));
@@ -120,7 +119,8 @@ pub(crate) fn extract_pages_and_images(
.then(|| build_vector_graphics(&paths));
assign_strikethrough(&mut text_items, &graphics);
let struct_nodes = extract_page_struct_nodes(&page, &view_box);
- let extracted_refs = extract_page_image_refs(&page, page_number, render_images);
+ let extracted_refs =
+ extract_page_image_refs(&page, page_number, output_options.extract_images);
let mut image_refs = extracted_refs.refs;
image_error_count += extracted_refs.error_count;
let annotations = output_options.extract_annotations.then(|| {
@@ -143,7 +143,7 @@ pub(crate) fn extract_pages_and_images(
.collect()
});
- if render_images && !image_refs.is_empty() {
+ if output_options.extract_images && !image_refs.is_empty() {
let rendered = render_page_images(&page, page_number, &image_refs, &mut image_cache);
image_error_count += rendered.error_count;
images.extend(rendered.images);
@@ -171,9 +171,11 @@ pub(crate) fn extract_pages_and_images(
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct ExtractionOutputOptions {
- pub emit_word_boxes: bool,
+ pub extract_text_metadata: bool,
+ pub extract_images: bool,
pub extract_vector_graphics: bool,
pub extract_annotations: bool,
+ pub emit_word_boxes: bool,
}
fn rect_from_pdfium(rect: RectF) -> Rect {
@@ -408,7 +410,7 @@ const IMAGE_MAX_COVERAGE: f32 = 0.9;
/// Extract every image referenced in `refs`, preserving valid JPEG streams and
/// rendering other images to PNG. Returns one `ExtractedImage` per ref. Used
-/// when `ImageMode::Embed` or an output directory is configured. Failures for
+/// when embedded-image extraction is explicitly enabled. Failures for
/// individual images are counted but do not fail the whole parse.
struct CachedImage {
raw_bytes: Vec<u8>,
@@ -950,6 +952,7 @@ fn extract_page_text_items(
view_box: &RectF,
glyph_resolver: Option<&dyn crate::GlyphResolver>,
emit_word_boxes: bool,
+ extract_text_metadata: bool,
) -> Result<Vec<TextItem>, LiteParseError> {
let char_count = text_page.char_count();
if char_count <= 0 {
@@ -980,7 +983,7 @@ fn extract_page_text_items(
let page_rotation = page.rotation();
let vp_xform = page.viewport_transform(view_box);
let mut items: Vec<TextItem> = Vec::new();
- let mut seg = SegmentBuilder::new(emit_word_boxes);
+ let mut seg = SegmentBuilder::new(emit_word_boxes, extract_text_metadata);
let garbage_fonts = detect_garbage_unicode_fonts(text_page, char_count);
let mut glyph_decoder = GlyphDecoder::new(
std::env::var("LITEPARSE_DEBUG_GLYPH").is_ok(),
@@ -1071,7 +1074,7 @@ fn extract_page_text_items(
// Keep PDFium-generated gaps as item boundaries so the emitted
// item can retain the same trailing-space-generated distinction
// as the source extractor. The visible text remains trimmed.
- if is_generated {
+ if is_generated && extract_text_metadata {
seg.flush(&mut items);
}
continue;
@@ -1933,10 +1936,12 @@ struct SegmentBuilder {
word_has: bool,
// When false, per-word tracking is skipped entirely and `words` stays empty.
emit_words: bool,
+ // When false, source-code/trailing-space metadata is not accumulated.
+ extract_text_metadata: bool,
}
impl SegmentBuilder {
- fn new(emit_words: bool) -> Self {
+ fn new(emit_words: bool, extract_text_metadata: bool) -> Self {
Self {
text: String::new(),
vp_left: f32::MAX,
@@ -1976,6 +1981,7 @@ impl SegmentBuilder {
word_bottom: f32::MIN,
word_has: false,
emit_words,
+ extract_text_metadata,
}
}
@@ -2063,14 +2069,18 @@ impl SegmentBuilder {
};
self.has_content = true;
self.pending_space = false;
- self.pending_space_char_codes.clear();
+ if self.extract_text_metadata {
+ self.pending_space_char_codes.clear();
+ }
self.pending_space_generated = false;
self.words.clear();
self.word_has = false;
self.add_word_char(c, vp_loose);
self.text_width = 0.0;
- self.char_codes.clear();
- self.char_codes.push(ch.char_code());
+ if self.extract_text_metadata {
+ self.char_codes.clear();
+ self.char_codes.push(ch.char_code());
+ }
self.font_is_buggy = false;
self.font_is_embedded = false;
self.font = None;
@@ -2170,7 +2180,9 @@ impl SegmentBuilder {
self.last_char_loose_right = vp_loose.right;
self.last_char_bottom = vp_strict.bottom;
self.char_count += 1;
- self.char_codes.push(ch.char_code());
+ if self.extract_text_metadata {
+ self.char_codes.push(ch.char_code());
+ }
if counts_as_unmapped(recovered, ch.has_unicode_map_error()) {
self.unmapped_char_count += 1;
}
@@ -2216,8 +2228,10 @@ impl SegmentBuilder {
fn mark_pending_space(&mut self, is_generated: bool, char_code: u32) {
if self.has_content {
self.pending_space = true;
- self.pending_space_generated = is_generated;
- self.pending_space_char_codes.push(char_code);
+ if self.extract_text_metadata {
+ self.pending_space_generated = is_generated;
+ self.pending_space_char_codes.push(char_code);
+ }
}
}
@@ -2226,7 +2240,9 @@ impl SegmentBuilder {
if self.pending_space {
self.break_word();
self.text.push(' ');
- self.char_codes.append(&mut self.pending_space_char_codes);
+ if self.extract_text_metadata {
+ self.char_codes.append(&mut self.pending_space_char_codes);
+ }
self.pending_space = false;
self.pending_space_generated = false;
}
@@ -2239,7 +2255,9 @@ impl SegmentBuilder {
}
self.break_word();
- self.char_codes.append(&mut self.pending_space_char_codes);
+ if self.extract_text_metadata {
+ self.char_codes.append(&mut self.pending_space_char_codes);
+ }
let trimmed = self.text.trim();
if !trimmed.is_empty() {
let width = self.vp_right - self.vp_left;
@@ -2276,7 +2294,7 @@ impl SegmentBuilder {
fill_color: self.fill_color.clone(),
stroke_color: self.stroke_color.clone(),
char_codes: std::mem::take(&mut self.char_codes),
- tsg: self.pending_space_generated,
+ trailing_space_generated: self.pending_space_generated,
confidence: None,
link: None,
strike: false,
@@ -2284,7 +2302,7 @@ impl SegmentBuilder {
});
}
- *self = Self::new(self.emit_words);
+ *self = Self::new(self.emit_words, self.extract_text_metadata);
}
}
@@ -2317,7 +2335,7 @@ mod tests {
}
fn segment_with_one_char() -> SegmentBuilder {
- let mut segment = SegmentBuilder::new(false);
+ let mut segment = SegmentBuilder::new(false, true);
segment.text = "A".into();
segment.vp_left = 1.0;
segment.vp_right = 7.0;
@@ -2339,7 +2357,7 @@ mod tests {
assert_eq!(items.len(), 1);
assert_eq!(items[0].text, "A");
assert_eq!(items[0].char_codes, vec![65, 32]);
- assert!(items[0].tsg);
+ assert!(items[0].trailing_space_generated);
}
#[test]
@@ -2350,7 +2368,25 @@ mod tests {
segment.flush(&mut items);
assert_eq!(items[0].char_codes, vec![65, 32]);
- assert!(!items[0].tsg);
+ assert!(!items[0].trailing_space_generated);
+ }
+
+ #[test]
+ fn disabled_text_metadata_does_not_accumulate_source_codes() {
+ let mut segment = SegmentBuilder::new(false, false);
+ segment.text = "A".into();
+ segment.vp_left = 1.0;
+ segment.vp_right = 7.0;
+ segment.vp_top = 2.0;
+ segment.vp_bottom = 12.0;
+ segment.char_count = 1;
+ segment.has_content = true;
+ segment.mark_pending_space(true, 32);
+ let mut items = Vec::new();
+ segment.flush(&mut items);
+
+ assert!(items[0].char_codes.is_empty());
+ assert!(!items[0].trailing_space_generated);
}
fn strike_item() -> TextItem {
diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs
index f82c294..f9df078 100644
--- a/crates/liteparse/src/main.rs
+++ b/crates/liteparse/src/main.rs
@@ -101,8 +101,8 @@ struct ParseCommand {
/// How to surface raster images in markdown output:
/// `off` strips them, `placeholder` (default) emits `![](image_pN_K.png)`
- /// references in reading order, and `embed` extracts image bytes and
- /// metadata.
+ /// references in reading order, and `embed` preserves the same presentation.
+ /// Use `--extract-images` to extract image bytes and metadata.
#[arg(long, default_value = "placeholder")]
image_mode: String,
@@ -111,8 +111,8 @@ struct ParseCommand {
extract_images: bool,
/// Directory to write embedded images to. Valid source JPEGs keep their
- /// format; other images are PNG. Setting this enables image extraction
- /// independently of `--image-mode`. Created if missing.
+ /// format; other images are PNG. Requires `--extract-images`. Created if
+ /// missing.
#[arg(long)]
image_output_dir: Option<String>,
@@ -134,7 +134,7 @@ struct ParseCommand {
complexity: bool,
/// Include rich PDF text metadata in text items and JSON output.
#[arg(long)]
- include_text_metadata: bool,
+ extract_text_metadata: bool,
/// Include page-scoped vector shapes and merged horizontal/vertical lines.
#[arg(long)]
extract_vector_graphics: bool,
@@ -233,7 +233,7 @@ struct BatchParseCommand {
complexity: bool,
/// Include rich PDF text metadata in text items and JSON output.
#[arg(long)]
- include_text_metadata: bool,
+ extract_text_metadata: bool,
/// Extract embedded image bytes and metadata.
#[arg(long)]
extract_images: bool,
@@ -365,7 +365,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
extract_links: !cmd.no_links,
extract_annotations: cmd.extract_annotations,
include_complexity: cmd.complexity,
- include_text_metadata: cmd.include_text_metadata,
+ extract_text_metadata: cmd.extract_text_metadata,
extract_vector_graphics: cmd.extract_vector_graphics,
..Default::default()
};
@@ -384,7 +384,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
&result.pages,
&result.images,
result.image_error_count,
- lp.config().include_text_metadata,
+ lp.config().extract_text_metadata,
)?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
@@ -461,7 +461,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
include_complexity: cmd.complexity,
- include_text_metadata: cmd.include_text_metadata,
+ extract_text_metadata: cmd.extract_text_metadata,
extract_images: cmd.extract_images,
extract_vector_graphics: cmd.extract_vector_graphics,
extract_annotations: cmd.extract_annotations,
@@ -512,7 +512,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
&result.pages,
&result.images,
result.image_error_count,
- lp.config().include_text_metadata,
+ lp.config().extract_text_metadata,
)
.map_err(|e| e.into()),
OutputFormat::Text => Ok(text::format_text(&result.pages)),
@@ -712,13 +712,13 @@ mod tests {
}
#[test]
- fn include_text_metadata_flag_is_available_for_parse_and_batch() {
- let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--include-text-metadata"])
+ fn extract_text_metadata_flag_is_available_for_parse_and_batch() {
+ let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--extract-text-metadata"])
.unwrap();
assert!(matches!(
cli.command,
Commands::Parse(ParseCommand {
- include_text_metadata: true,
+ extract_text_metadata: true,
..
})
));
@@ -728,13 +728,13 @@ mod tests {
"batch-parse",
"input",
"output",
- "--include-text-metadata",
+ "--extract-text-metadata",
])
.unwrap();
assert!(matches!(
cli.command,
Commands::BatchParse(BatchParseCommand {
- include_text_metadata: true,
+ extract_text_metadata: true,
..
})
));
diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs
index 6ecc458..88645e0 100644
--- a/crates/liteparse/src/output/json.rs
+++ b/crates/liteparse/src/output/json.rs
@@ -36,7 +36,7 @@ pub(crate) struct JsonTextItem {
#[serde(skip_serializing_if = "Vec::is_empty")]
pub char_codes: Vec<u32>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
- pub tsg: bool,
+ pub trailing_space_generated: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub confidence: Option<f32>,
}
@@ -86,7 +86,7 @@ pub(crate) struct JsonImage {
}
/// Build structured JSON output from parsed pages.
-pub(crate) fn build_json(pages: &[ParsedPage], include_text_metadata: bool) -> ParseResultJson {
+pub(crate) fn build_json(pages: &[ParsedPage], extract_text_metadata: bool) -> ParseResultJson {
ParseResultJson {
images: Vec::new(),
image_error_count: 0,
@@ -106,28 +106,29 @@ pub(crate) fn build_json(pages: &[ParsedPage], include_text_metadata: bool) -> P
y: item.y,
width: item.width,
height: item.height,
- rotation: include_text_metadata.then_some(item.rotation),
+ rotation: Some(item.rotation),
font_name: item.font_name.clone(),
font_size: item.font_size,
- font_height: include_text_metadata.then_some(item.font_height).flatten(),
- font_ascent: include_text_metadata.then_some(item.font_ascent).flatten(),
- font_descent: include_text_metadata.then_some(item.font_descent).flatten(),
- font_weight: include_text_metadata.then_some(item.font_weight).flatten(),
- text_width: include_text_metadata.then_some(item.text_width).flatten(),
- font_is_buggy: include_text_metadata.then_some(item.font_is_buggy),
- mcid: include_text_metadata.then_some(item.mcid).flatten(),
- fill_color: include_text_metadata
+ font_height: extract_text_metadata.then_some(item.font_height).flatten(),
+ font_ascent: extract_text_metadata.then_some(item.font_ascent).flatten(),
+ font_descent: extract_text_metadata.then_some(item.font_descent).flatten(),
+ font_weight: extract_text_metadata.then_some(item.font_weight).flatten(),
+ text_width: extract_text_metadata.then_some(item.text_width).flatten(),
+ font_is_buggy: extract_text_metadata.then_some(item.font_is_buggy),
+ mcid: extract_text_metadata.then_some(item.mcid).flatten(),
+ fill_color: extract_text_metadata
.then(|| item.fill_color.clone())
.flatten(),
- stroke_color: include_text_metadata
+ stroke_color: extract_text_metadata
.then(|| item.stroke_color.clone())
.flatten(),
- char_codes: if include_text_metadata {
+ char_codes: if extract_text_metadata {
item.char_codes.clone()
} else {
Vec::new()
},
- tsg: include_text_metadata && item.tsg,
+ trailing_space_generated: extract_text_metadata
+ && item.trailing_space_generated,
confidence: item.confidence.or(Some(1.0)),
})
.collect(),
@@ -145,9 +146,9 @@ pub fn format_json_result(
pages: &[ParsedPage],
images: &[ExtractedImage],
image_error_count: u32,
- include_text_metadata: bool,
+ extract_text_metadata: bool,
) -> Result<String, serde_json::Error> {
- let mut result = build_json(pages, include_text_metadata);
+ let mut result = build_json(pages, extract_text_metadata);
result.images = images
.iter()
.map(|image| JsonImage {
@@ -175,9 +176,9 @@ pub fn format_json(pages: &[ParsedPage]) -> Result<String, serde_json::Error> {
/// Format parsed pages as JSON, optionally including rich PDF text metadata.
pub fn format_json_with_text_metadata(
pages: &[ParsedPage],
- include_text_metadata: bool,
+ extract_text_metadata: bool,
) -> Result<String, serde_json::Error> {
- let result = build_json(pages, include_text_metadata);
+ let result = build_json(pages, extract_text_metadata);
serde_json::to_string_pretty(&result)
}
@@ -256,7 +257,7 @@ mod tests {
text_item.fill_color = Some("ff112233".into());
text_item.stroke_color = Some("ff445566".into());
text_item.char_codes = vec![104, 105, 32];
- text_item.tsg = true;
+ text_item.trailing_space_generated = true;
let value: serde_json::Value = serde_json::from_str(
&format_json_with_text_metadata(&[page(vec![text_item])], true).unwrap(),
@@ -273,7 +274,7 @@ mod tests {
assert_eq!(item["fill_color"], "ff112233");
assert_eq!(item["stroke_color"], "ff445566");
assert_eq!(item["char_codes"], serde_json::json!([104, 105, 32]));
- assert_eq!(item["tsg"], true);
+ assert_eq!(item["trailing_space_generated"], true);
assert_eq!(item["rotation"], 0.0);
}
@@ -290,17 +291,17 @@ mod tests {
text_item.font_is_buggy = true;
text_item.mcid = Some(4);
text_item.char_codes = vec![104, 105];
- text_item.tsg = true;
+ text_item.trailing_space_generated = true;
let value: serde_json::Value =
serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();
let item = &value["pages"][0]["text_items"][0];
- assert!(item.get("rotation").is_none());
+ assert_eq!(item["rotation"], 0.0);
assert!(item.get("font_height").is_none());
assert!(item.get("font_is_buggy").is_none());
assert!(item.get("mcid").is_none());
assert!(item.get("char_codes").is_none());
- assert!(item.get("tsg").is_none());
+ assert!(item.get("trailing_space_generated").is_none());
}
#[test]
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 5366eaf..cc9f938 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -26,7 +26,7 @@ pub struct ParseResult {
/// emitter as a high-priority heading source on untagged PDFs.
pub outline: Vec<OutlineTarget>,
/// Raster images extracted from the document. Empty unless the parser
- /// was configured with `ImageMode::Embed`. Each entry carries the same
+ /// was configured with `extract_images`. Each entry carries the same
/// `id` and `format` the markdown emitter referenced, so the caller can
/// match them up without parsing markdown.
pub images: Vec<ExtractedImage>,
@@ -51,12 +51,6 @@ pub struct ScreenshotResult {
#[cfg(not(target_arch = "wasm32"))]
const FONT_DB_DIR_ENV: &str = "LITEPARSE_FONT_DB_DIR";
-fn should_extract_images(config: &LiteParseConfig) -> bool {
- config.extract_images
- || matches!(config.image_mode, crate::config::ImageMode::Embed)
- || config.image_output_dir.is_some()
-}
-
#[cfg(not(target_arch = "wasm32"))]
fn write_extracted_images(
output_dir: &str,
@@ -167,6 +161,15 @@ impl LiteParse {
.map_err(|e| format!("invalid --target-pages: {}", e).into())
}
+ fn validate_output_config(&self) -> Result<(), LiteParseError> {
+ if self.config.image_output_dir.is_some() && !self.config.extract_images {
+ return Err(LiteParseError::Config(
+ "image_output_dir requires extract_images = true".to_string(),
+ ));
+ }
+ Ok(())
+ }
+
/// Determine the complexity of each page in a document, returning a vector
/// of `PageComplexityStats` for each page. This is useful for deciding
/// whether to enable OCR on a per-page basis, or for other heuristics.
@@ -211,7 +214,6 @@ impl LiteParse {
&document,
target_pages.as_deref(),
self.config.max_pages,
- false, // render_images: image rasters not needed for complexity
false, // extract_links: irrelevant for complexity stats
self.glyph_resolver.as_deref(),
extract::ExtractionOutputOptions::default(),
@@ -285,6 +287,8 @@ impl LiteParse {
let t0 = web_time::Instant::now();
+ self.validate_output_config()?;
+
#[cfg(not(target_arch = "wasm32"))]
let (validated_input, _guard) =
conversion::resolve_pdf_input(input, self.config.password.as_deref(), false).await?;
@@ -301,8 +305,6 @@ impl LiteParse {
// projection (pure Rust) do not touch PDFium, so they can run
// concurrently with other `LiteParse` calls.
let password = self.config.password.as_deref();
- let render_images = should_extract_images(&self.config);
-
// Build the OCR engine up front so the renderer knows whether to emit a
// grayscale buffer (cheaper, for engines that binarize internally) or RGB.
let ocr_engine: Option<std::sync::Arc<dyn OcrEngine>> = if self.config.ocr_enabled {
@@ -349,6 +351,7 @@ impl LiteParse {
};
let ocr_grayscale = ocr_engine.as_ref().is_some_and(|e| e.prefers_grayscale());
+ #[allow(unused_mut)] // mutated only by the native image-output writer
let (pages, ocr_rendered, outline, mut images, image_error_count, complexity) = {
let lib = Library::init();
let document = extract::load_document_from_input(&lib, &validated_input, password)?;
@@ -357,12 +360,13 @@ impl LiteParse {
&document,
target_pages.as_deref(),
self.config.max_pages,
- render_images,
self.config.extract_links
&& self.config.output_format == crate::config::OutputFormat::Markdown,
self.glyph_resolver.as_deref(),
extract::ExtractionOutputOptions {
+ extract_images: self.config.extract_images,
emit_word_boxes: self.config.emit_word_boxes,
+ extract_text_metadata: self.config.extract_text_metadata,
extract_vector_graphics: self.config.extract_vector_graphics,
extract_annotations: self.config.extract_annotations,
},
@@ -481,15 +485,13 @@ impl LiteParse {
.join("\n\n")
};
- if !self.config.include_text_metadata {
- strip_text_metadata(&mut parsed_pages);
- }
-
let total = web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;
log(&format!("[liteparse] total: {:.1}ms", total));
#[cfg(not(target_arch = "wasm32"))]
- if let Some(output_dir) = self.config.image_output_dir.as_deref() {
+ if self.config.extract_images
+ && let Some(output_dir) = self.config.image_output_dir.as_deref()
+ {
write_extracted_images(output_dir, &mut images)?;
}
@@ -529,10 +531,6 @@ impl LiteParse {
.join("\n\n")
};
- if !self.config.include_text_metadata {
- strip_text_metadata(&mut parsed_pages);
- }
-
ParseResult {
pages: parsed_pages,
text: full_text,
@@ -602,26 +600,6 @@ impl LiteParse {
}
}
-/// Remove opt-in extraction metadata only after projection/Markdown have had
-/// access to it. Longstanding lightweight fields (text, bbox, rotation,
-/// font name/size, confidence, links, strike state, and optional word boxes)
-/// remain unchanged.
-fn strip_text_metadata(pages: &mut [crate::types::ParsedPage]) {
- for item in pages.iter_mut().flat_map(|page| &mut page.text_items) {
- item.font_height = None;
- item.font_ascent = None;
- item.font_descent = None;
- item.font_weight = None;
- item.text_width = None;
- item.font_is_buggy = false;
- item.mcid = None;
- item.fill_color = None;
- item.stroke_color = None;
- item.char_codes.clear();
- item.tsg = false;
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -648,12 +626,14 @@ mod tests {
fill_color: Some("ff112233".into()),
stroke_color: Some("ff445566".into()),
char_codes: vec![104, 101, 108, 108, 111],
- tsg: true,
+ trailing_space_generated: true,
..Default::default()
}],
graphics: vec![],
+ vector_graphics: None,
struct_nodes: vec![],
image_refs: vec![],
+ annotations: None,
}
}
@@ -669,29 +649,12 @@ mod tests {
}
#[test]
- fn parse_result_strips_text_metadata_by_default() {
+ fn parse_from_pages_preserves_internal_text_metadata() {
let result = LiteParse::new(LiteParseConfig::default())
.parse_from_pages(vec![page_with_text_metadata()], vec![]);
let item = &result.pages[0].text_items[0];
assert_eq!(item.font_name.as_deref(), Some("Helvetica"));
assert_eq!(item.font_size, Some(10.0));
- assert_eq!(item.font_height, None);
- assert_eq!(item.font_weight, None);
- assert_eq!(item.mcid, None);
- assert!(item.char_codes.is_empty());
- assert!(!item.font_is_buggy);
- assert!(!item.tsg);
- }
-
- #[test]
- fn parse_result_preserves_text_metadata_when_enabled() {
- let config = LiteParseConfig {
- include_text_metadata: true,
- ..Default::default()
- };
- let result =
- LiteParse::new(config).parse_from_pages(vec![page_with_text_metadata()], vec![]);
- let item = &result.pages[0].text_items[0];
assert_eq!(item.font_height, Some(10.0));
assert_eq!(item.font_ascent, Some(8.0));
assert_eq!(item.font_descent, Some(-2.0));
@@ -702,31 +665,34 @@ mod tests {
assert_eq!(item.fill_color.as_deref(), Some("ff112233"));
assert_eq!(item.stroke_color.as_deref(), Some("ff445566"));
assert_eq!(item.char_codes, vec![104, 101, 108, 108, 111]);
- assert!(item.tsg);
+ assert!(item.trailing_space_generated);
}
#[test]
- fn image_extraction_is_opt_in_with_compatibility_implications() {
- let default = LiteParseConfig::default();
- assert!(!should_extract_images(&default));
-
- let explicit = LiteParseConfig {
+ fn image_extraction_is_explicitly_opt_in() {
+ let options = extract::ExtractionOutputOptions {
extract_images: true,
..Default::default()
};
- assert!(should_extract_images(&explicit));
+ assert!(options.extract_images);
let embed = LiteParseConfig {
image_mode: crate::config::ImageMode::Embed,
..Default::default()
};
- assert!(should_extract_images(&embed));
+ assert!(!embed.extract_images);
+ }
- let output_dir = LiteParseConfig {
+ #[test]
+ fn image_output_dir_requires_image_extraction() {
+ let parser = LiteParse::new(LiteParseConfig {
image_output_dir: Some("images".into()),
..Default::default()
- };
- assert!(should_extract_images(&output_dir));
+ });
+ assert_eq!(
+ parser.validate_output_config().unwrap_err().to_string(),
+ "invalid config: image_output_dir requires extract_images = true"
+ );
}
#[test]
diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs
index 354c2c1..5c2a522 100644
--- a/crates/liteparse/src/types.rs
+++ b/crates/liteparse/src/types.rs
@@ -63,7 +63,7 @@ pub struct TextItem {
/// Whether the trailing source space was synthesized by PDFium rather than
/// represented by a real space glyph in the PDF content stream.
#[serde(skip_serializing_if = "std::ops::Not::not")]
- pub tsg: bool,
+ pub trailing_space_generated: bool,
/// OCR confidence score (0.0–1.0). None for native PDF text.
#[serde(skip_serializing_if = "Option::is_none")]
pub confidence: Option<f32>,
@@ -254,8 +254,8 @@ pub struct ImageRef {
}
/// A raster image extracted from a page along with its pixel bytes. Surfaced
-/// on `ParseResult.images` only when `ImageMode::Embed` is configured —
-/// otherwise the extraction step skips the render and only `ImageRef`s are
+/// on `ParseResult.images` only when `extract_images` is enabled — otherwise
+/// the extraction step skips the render and only `ImageRef`s are
/// produced. Other images are encoded as PNG from PDFium's rendered bitmap.
/// JPEG streams are preserved without re-encoding when PDFium
/// exposes a valid directly decoded DCT stream.
diff --git a/packages/node/README.md b/packages/node/README.md
index db5dd9f..711ee27 100644
--- a/packages/node/README.md
+++ b/packages/node/README.md
@@ -64,20 +64,24 @@ const parser = new LiteParse({
extractVectorGraphics: false, // Opt-in shapes + merged H/V lines per page
extractAnnotations: false, // Include page annotations in structured output
preserveVerySmallText: false, // Keep tiny text
- includeTextMetadata: false, // Opt in to MCID, font metrics, colors, char codes, and tsg
+ extractTextMetadata: false, // Opt in to MCID, font metrics, colors, char codes, and trailingSpaceGenerated
password: undefined, // Password for protected documents
quiet: false, // Suppress progress output
numWorkers: 4, // Concurrent OCR workers
});
```
-When `imageOutputDir` is set, image extraction is enabled automatically. Each
+When `extractImages` is true, image extraction is enabled. `imageOutputDir` requires
+that explicit opt-in and writes the extracted bytes to disk. Each
`result.images` entry includes its page bbox, intrinsic pixel dimensions, rotation,
format, `name`, and `path`. Valid source JPEGs are preserved, exact duplicates reuse
one file, and JSON CLI output contains metadata only—never base64 image data.
-`imageMode: "embed"` also continues to imply extraction. With the defaults
-(`imageMode: "placeholder"`, `extractImages: false`), only lightweight Markdown
-placement refs are collected and `result.images` stays empty.
+`imageMode` controls Markdown presentation only and does not imply extraction. With
+`extractImages: false`, lightweight Markdown placement refs are still collected and
+`result.images` stays empty.
+
+The Node CLI uses the same snake_case JSON schema as the Rust CLI; camelCase is
+reserved for the programmatic Node API.
When `extractAnnotations` is enabled, each parsed page has an `annotations`
array containing the subtype, contents, author/title, PDF date strings,
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index cb0bcd2..2da6a88 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -36,10 +36,15 @@ export interface JsLiteParseConfig {
/**
* How to surface raster images in markdown output: "off", "placeholder"
* (default — emits `![](image_pN_K.png)` references with no bytes), or
- * "embed" (also returns each image's bytes and metadata on `images`).
+ * "embed" (same presentation as placeholder; extraction is independent).
*/
imageMode?: string
+ /** Extract embedded image bytes and metadata (default false). */
extractImages?: boolean
+ /**
+ * Directory where embedded image files are written. Requires
+ * `extractImages` to be true.
+ */
imageOutputDir?: string
/**
* Render hyperlink annotations as `[text](url)` in markdown output
@@ -67,7 +72,7 @@ export interface JsLiteParseConfig {
*/
emitWordBoxes?: boolean
/** Include rich PDF text metadata on returned text items. Default false. */
- includeTextMetadata?: boolean
+ extractTextMetadata?: boolean
/**
* Restrict output to a page sub-region. Each field is the fraction of the
* page cropped from that side; a text item survives only if it lies
@@ -85,6 +90,8 @@ export interface JsLiteParseConfig {
* Default false; enabling it runs an extra vector-text detection pass.
*/
includeComplexity?: boolean
+ /** Expose page-scoped vector path extraction. Default false. */
+ extractVectorGraphics?: boolean
}
/**
* A page sub-region as the fraction cropped from each side (top-left origin,
@@ -126,7 +133,7 @@ export interface JsTextItem {
/** Raw PDF content-stream character codes for the source glyphs. */
charCodes?: Array<number>
/** True when the trailing source space was synthesized by PDFium. */
- tsg?: boolean
+ trailingSpaceGenerated?: boolean
confidence?: number
/** Rotation in degrees (viewport space). Defaults to 0 when omitted. */
rotation?: number
@@ -187,14 +194,44 @@ export interface JsParsedPage {
markdown: string
textItems: Array<JsTextItem>
complexity?: JsPageComplexityStats
+ vectorGraphics?: JsVectorGraphics
annotations?: Array<JsDocumentAnnotation>
}
+export interface JsVectorShape {
+ bbox: JsRect
+ stroke: boolean
+ strokeColor?: string
+ fill: boolean
+ fillColor?: string
+ hasCurve: boolean
+}
+export interface JsRect {
+ x: number
+ y: number
+ width: number
+ height: number
+}
export interface JsAnnotationRect {
x: number
y: number
width: number
height: number
}
+export interface JsVectorLine {
+ x1: number
+ y1: number
+ x2: number
+ y2: number
+ stroke: boolean
+ strokeWidth?: number
+ strokeColor?: string
+ fill: boolean
+ fillColor?: string
+}
+export interface JsVectorGraphics {
+ shapes: Array<JsVectorShape>
+ lines: Array<JsVectorLine>
+}
export interface JsDocumentAnnotation {
subtype: string
contents?: string
diff --git a/packages/node/src/cli-json.ts b/packages/node/src/cli-json.ts
new file mode 100644
index 0000000..a9df8c4
--- /dev/null
+++ b/packages/node/src/cli-json.ts
@@ -0,0 +1,182 @@
+import type {
+ DocumentAnnotation,
+ ExtractedImage,
+ PageComplexityStats,
+ ParseResult,
+ TextItem,
+ VectorGraphics,
+} from "./lib.js";
+
+function textItemToCliJson(item: TextItem) {
+ return {
+ text: item.text,
+ x: item.x,
+ y: item.y,
+ width: item.width,
+ height: item.height,
+ ...(item.rotation !== undefined ? { rotation: item.rotation } : {}),
+ ...(item.fontName !== undefined ? { font_name: item.fontName } : {}),
+ ...(item.fontSize !== undefined ? { font_size: item.fontSize } : {}),
+ ...(item.fontHeight !== undefined ? { font_height: item.fontHeight } : {}),
+ ...(item.fontAscent !== undefined ? { font_ascent: item.fontAscent } : {}),
+ ...(item.fontDescent !== undefined ? { font_descent: item.fontDescent } : {}),
+ ...(item.fontWeight !== undefined ? { font_weight: item.fontWeight } : {}),
+ ...(item.textWidth !== undefined ? { text_width: item.textWidth } : {}),
+ ...(item.fontIsBuggy !== undefined ? { font_is_buggy: item.fontIsBuggy } : {}),
+ ...(item.mcid !== undefined ? { mcid: item.mcid } : {}),
+ ...(item.fillColor !== undefined ? { fill_color: item.fillColor } : {}),
+ ...(item.strokeColor !== undefined ? { stroke_color: item.strokeColor } : {}),
+ ...(item.charCodes?.length ? { char_codes: item.charCodes } : {}),
+ ...(item.trailingSpaceGenerated
+ ? { trailing_space_generated: true }
+ : {}),
+ ...(item.confidence !== undefined ? { confidence: item.confidence } : {}),
+ };
+}
+
+function complexityToCliJson(complexity: PageComplexityStats) {
+ return {
+ page_number: complexity.pageNumber,
+ text_length: complexity.textLength,
+ text_coverage: complexity.textCoverage,
+ has_substantial_images: complexity.hasSubstantialImages,
+ image_block_count: complexity.imageBlockCount,
+ image_coverage: complexity.imageCoverage,
+ largest_image_coverage: complexity.largestImageCoverage,
+ full_page_image: complexity.fullPageImage,
+ ...(complexity.uncoveredVectorArea !== undefined
+ ? { uncovered_vector_area: complexity.uncoveredVectorArea }
+ : {}),
+ is_garbled: complexity.isGarbled,
+ page_area: complexity.pageArea,
+ needs_ocr: complexity.needsOcr,
+ reasons: complexity.reasons,
+ ...(complexity.layout
+ ? {
+ layout: {
+ column_count: complexity.layout.columnCount,
+ ruled_table_count: complexity.layout.ruledTableCount,
+ ruled_table_coverage: complexity.layout.ruledTableCoverage,
+ text_table_run_count: complexity.layout.textTableRunCount,
+ figure_count: complexity.layout.figureCount,
+ figure_coverage: complexity.layout.figureCoverage,
+ is_complex: complexity.layout.isComplex,
+ reasons: complexity.layout.reasons,
+ },
+ }
+ : {}),
+ };
+}
+
+function vectorGraphicsToCliJson(vectorGraphics: VectorGraphics) {
+ return {
+ shapes: vectorGraphics.shapes.map((shape) => ({
+ bbox: {
+ x: shape.bbox.x,
+ y: shape.bbox.y,
+ width: shape.bbox.width,
+ height: shape.bbox.height,
+ },
+ stroke: shape.stroke,
+ ...(shape.strokeColor !== undefined
+ ? { stroke_color: shape.strokeColor }
+ : {}),
+ fill: shape.fill,
+ ...(shape.fillColor !== undefined
+ ? { fill_color: shape.fillColor }
+ : {}),
+ has_curve: shape.hasCurve,
+ })),
+ lines: vectorGraphics.lines.map((line) => ({
+ x1: line.x1,
+ y1: line.y1,
+ x2: line.x2,
+ y2: line.y2,
+ stroke: line.stroke,
+ ...(line.strokeWidth !== undefined
+ ? { stroke_width: line.strokeWidth }
+ : {}),
+ ...(line.strokeColor !== undefined
+ ? { stroke_color: line.strokeColor }
+ : {}),
+ fill: line.fill,
+ ...(line.fillColor !== undefined ? { fill_color: line.fillColor } : {}),
+ })),
+ };
+}
+
+function annotationToCliJson(annotation: DocumentAnnotation) {
+ return {
+ subtype: annotation.subtype,
+ ...(annotation.contents !== undefined
+ ? { contents: annotation.contents }
+ : {}),
+ ...(annotation.created !== undefined ? { created: annotation.created } : {}),
+ ...(annotation.modified !== undefined
+ ? { modified: annotation.modified }
+ : {}),
+ ...(annotation.title !== undefined ? { title: annotation.title } : {}),
+ ...(annotation.rect !== undefined ? { rect: annotation.rect } : {}),
+ ...(annotation.quadpointRects.length
+ ? {
+ quadpoint_rects: annotation.quadpointRects.map((rect) => ({
+ x: rect.x,
+ y: rect.y,
+ width: rect.width,
+ height: rect.height,
+ })),
+ }
+ : {}),
+ ...(annotation.uri !== undefined ? { uri: annotation.uri } : {}),
+ };
+}
+
+function imageToCliJson(image: ExtractedImage) {
+ return {
+ id: image.id,
+ name: image.name,
+ ...(image.path !== undefined ? { path: image.path } : {}),
+ page: image.page,
+ bbox: {
+ x: image.bbox.x,
+ y: image.bbox.y,
+ width: image.bbox.width,
+ height: image.bbox.height,
+ },
+ width: image.width,
+ height: image.height,
+ rotation: image.rotation,
+ format: image.format,
+ ...(image.duplicateOf !== undefined
+ ? { duplicate_of: image.duplicateOf }
+ : {}),
+ };
+}
+
+/** Project the camelCase Node API result into the Rust CLI JSON schema. */
+export function parseResultToCliJson(result: ParseResult) {
+ return {
+ pages: result.pages.map((page) => ({
+ page: page.pageNum,
+ width: page.width,
+ height: page.height,
+ text: page.text,
+ text_items: page.textItems.map(textItemToCliJson),
+ ...(page.complexity
+ ? { complexity: complexityToCliJson(page.complexity) }
+ : {}),
+ ...(page.vectorGraphics
+ ? { vector_graphics: vectorGraphicsToCliJson(page.vectorGraphics) }
+ : {}),
+ ...(page.annotations !== undefined
+ ? { annotations: page.annotations.map(annotationToCliJson) }
+ : {}),
+ })),
+ ...(result.images.length
+ ? { images: result.images.map(imageToCliJson) }
+ : {}),
+ ...(result.imageErrorCount
+ ? { image_error_count: result.imageErrorCount }
+ : {}),
+ };
+}
diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts
index 80bdff5..403a5c0 100644
--- a/packages/node/src/cli.ts
+++ b/packages/node/src/cli.ts
@@ -2,6 +2,7 @@
import { program } from "commander";
import { LiteParse, type LiteParseConfig } from "./lib.js";
+import { parseResultToCliJson } from "./cli-json.js";
import { readFileSync, writeFileSync, mkdirSync, readdirSync, statSync } from "node:fs";
import { join, relative, parse as parsePath } from "node:path";
@@ -81,7 +82,7 @@ program
.option("--dpi <dpi>", "Rendering DPI", parseFloat)
.option("--preserve-small-text", "Keep very small text")
.option(
- "--include-text-metadata",
+ "--extract-text-metadata",
"Include rich PDF text metadata in text items and JSON output",
)
.option("--password <password>", "Password for encrypted documents")
@@ -127,7 +128,7 @@ program
if (opts.targetPages) config.targetPages = opts.targetPages as string;
if (opts.dpi) config.dpi = opts.dpi as number;
if (opts.preserveSmallText) config.preserveVerySmallText = true;
- if (opts.includeTextMetadata) config.includeTextMetadata = true;
+ if (opts.extractTextMetadata) config.extractTextMetadata = true;
if (opts.password) config.password = opts.password as string;
if (opts.quiet) config.quiet = true;
if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;
@@ -143,19 +144,7 @@ program
const output =
config.outputFormat === "json"
? JSON.stringify(
- {
- pages: result.pages.map((p) => ({
- page: p.pageNum,
- width: p.width,
- height: p.height,
- text: p.text,
- textItems: p.textItems,
- ...(p.vectorGraphics ? { vectorGraphics: p.vectorGraphics } : {}),
- ...(p.annotations ? { annotations: p.annotations } : {}),
- })),
- images: result.images.map(({ bytes: _bytes, ...image }) => image),
- imageErrorCount: result.imageErrorCount,
- },
+ parseResultToCliJson(result),
null,
2,
)
@@ -322,10 +311,11 @@ program
.option("-q, --quiet", "Suppress progress output")
.option("--num-workers <n>", "Number of concurrent OCR workers", parseInt)
.option(
- "--include-text-metadata",
+ "--extract-text-metadata",
"Include rich PDF text metadata in text items and JSON output",
)
.option("--extract-images", "Extract embedded image bytes and metadata")
+ .option("--extract-annotations", "Include all PDF annotations in page output")
.option(
"--extract-vector-graphics",
"Include page-scoped vector shapes and merged horizontal/vertical lines",
@@ -351,8 +341,9 @@ program
if (opts.password) config.password = opts.password as string;
if (opts.quiet) config.quiet = true;
if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;
- if (opts.includeTextMetadata) config.includeTextMetadata = true;
+ if (opts.extractTextMetadata) config.extractTextMetadata = true;
if (opts.extractImages) config.extractImages = true;
+ if (opts.extractAnnotations) config.extractAnnotations = true;
if (opts.extractVectorGraphics) config.extractVectorGraphics = true;
const parser = new LiteParse(config);
@@ -403,18 +394,7 @@ program
const output =
format === "json"
? JSON.stringify(
- {
- pages: result.pages.map((p) => ({
- page: p.pageNum,
- width: p.width,
- height: p.height,
- text: p.text,
- textItems: p.textItems,
- ...(p.vectorGraphics ? { vectorGraphics: p.vectorGraphics } : {}),
- })),
- images: result.images.map(({ bytes: _bytes, ...image }) => image),
- imageErrorCount: result.imageErrorCount,
- },
+ parseResultToCliJson(result),
null,
2,
)
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index 0449f6a..1a3ee1b 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -33,7 +33,7 @@ export interface LiteParseConfig {
imageMode: ImageMode;
/** Extract embedded image bytes and metadata (default: false). */
extractImages: boolean;
- /** Directory where extracted embedded image files are written. */
+ /** Directory where extracted embedded image files are written. Requires `extractImages`. */
imageOutputDir?: string;
/** Render hyperlink annotations as `[text](url)` in markdown output (default: true). */
extractLinks: boolean;
@@ -64,7 +64,7 @@ export interface LiteParseConfig {
*/
emitWordBoxes: boolean;
/** Include rich PDF text metadata on returned text items. Default false. */
- includeTextMetadata?: boolean;
+ extractTextMetadata?: boolean;
/**
* Restrict output to a page sub-region. Each field is the fraction of the
* page cropped away from that side (top-left origin), so `{ left: 0.5 }`
@@ -138,7 +138,7 @@ export interface TextItem {
/** Raw PDF content-stream character codes for the source glyphs. */
charCodes?: number[];
/** True when the trailing source space was synthesized by PDFium. */
- tsg?: boolean;
+ trailingSpaceGenerated?: boolean;
confidence?: number;
/** Rotation in degrees (viewport space). Defaults to 0 when omitted. */
rotation?: number;
@@ -273,7 +273,7 @@ export interface ExtractedImage {
export interface ParseResult {
pages: ParsedPage[];
text: string;
- /** Populated when image extraction is explicitly or implicitly enabled. */
+ /** Populated only when `extractImages` is true. */
images: ExtractedImage[];
/** Embedded image objects that PDFium could not render or encode. */
imageErrorCount: number;
@@ -393,7 +393,7 @@ export class LiteParse {
ocrFailureFatal: userConfig.ocrFailureFatal,
ocrHedgeDelaysMs: userConfig.ocrHedgeDelaysMs,
emitWordBoxes: userConfig.emitWordBoxes,
- includeTextMetadata: userConfig.includeTextMetadata,
+ extractTextMetadata: userConfig.extractTextMetadata,
cropBox: userConfig.cropBox,
skipDiagonalText: userConfig.skipDiagonalText,
includeComplexity: userConfig.includeComplexity,
@@ -426,7 +426,7 @@ export class LiteParse {
ocrFailureFatal: resolved.ocrFailureFatal ?? true,
ocrHedgeDelaysMs: resolved.ocrHedgeDelaysMs ?? [],
emitWordBoxes: resolved.emitWordBoxes ?? false,
- includeTextMetadata: resolved.includeTextMetadata ?? false,
+ extractTextMetadata: resolved.extractTextMetadata ?? false,
cropBox: resolved.cropBox ?? undefined,
skipDiagonalText: resolved.skipDiagonalText ?? false,
includeComplexity: resolved.includeComplexity ?? false,
@@ -585,7 +585,7 @@ function toTextItem(item: NativeTextItem): TextItem {
fillColor: item.fillColor,
strokeColor: item.strokeColor,
charCodes: item.charCodes,
- tsg: item.tsg,
+ trailingSpaceGenerated: item.trailingSpaceGenerated,
confidence: item.confidence,
rotation: item.rotation,
words: item.words,
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index da44ef1..4f66c19 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -41,7 +41,7 @@ export interface LiteParseNativeConfig {
ocrFailureFatal?: boolean;
ocrHedgeDelaysMs?: number[];
emitWordBoxes?: boolean;
- includeTextMetadata?: boolean;
+ extractTextMetadata?: boolean;
cropBox?: NativeCropBox;
skipDiagonalText?: boolean;
includeComplexity?: boolean;
@@ -81,7 +81,7 @@ export interface NativeTextItem {
fillColor?: string;
strokeColor?: string;
charCodes?: number[];
- tsg?: boolean;
+ trailingSpaceGenerated?: boolean;
confidence?: number;
rotation?: number;
words?: NativeWordBox[];
diff --git a/packages/python/README.md b/packages/python/README.md
index a4c9103..40787bc 100644
--- a/packages/python/README.md
+++ b/packages/python/README.md
@@ -57,26 +57,27 @@ parser = LiteParse(
dpi=150, # Rendering DPI
output_format="json", # "json" | "text" | "markdown"
image_mode="placeholder", # Markdown image handling: "placeholder" | "off" | "embed"
- extract_images=True, # Extract image bytes + metadata (default: False)
+ extract_images=True, # Extract image bytes + metadata (default: False)
image_output_dir="./images", # Write images and return name/path metadata (optional)
extract_links=True, # Render [text](url) links in markdown output
extract_vector_graphics=False, # Opt-in shapes + merged H/V lines per page
extract_annotations=False, # Include page annotations in structured output
preserve_very_small_text=False, # Keep tiny text
- include_text_metadata=False, # Opt in to MCID, font metrics, colors, char codes, and tsg
+ extract_text_metadata=False, # Opt in to MCID, font metrics, colors, char codes, and trailing_space_generated
password=None, # Password for protected documents
quiet=False, # Suppress progress output
num_workers=4, # Concurrent OCR workers
)
```
-When ``image_output_dir`` is set, image extraction is enabled automatically. Each
+When ``extract_images=True``, image extraction is enabled. ``image_output_dir``
+requires that explicit opt-in and writes the extracted bytes to disk. Each
``result.images`` entry includes its page bbox, intrinsic pixel dimensions, rotation,
format, ``name``, and ``path``. Valid source JPEGs are preserved, exact duplicates
reuse one file, and JSON CLI output contains metadata only—never base64 image data.
-``image_mode="embed"`` also continues to imply extraction. With the defaults
-(``image_mode="placeholder"``, ``extract_images=False``), only lightweight Markdown
-placement refs are collected and ``result.images`` stays empty.
+``image_mode`` controls Markdown presentation only and does not imply extraction.
+With ``extract_images=False``, lightweight Markdown placement refs are still collected
+and ``result.images`` stays empty.
When `extract_annotations` is enabled, each parsed page has an `annotations`
list containing the subtype, contents, author/title, PDF date strings,
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index 5fd9c97..a4cbee8 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -82,7 +82,7 @@ def _convert_native_result(native_result: Any) -> ParseResult:
fill_color=getattr(item, "fill_color", None),
stroke_color=getattr(item, "stroke_color", None),
char_codes=list(getattr(item, "char_codes", [])),
- tsg=getattr(item, "tsg", False),
+ trailing_space_generated=getattr(item, "trailing_space_generated", False),
confidence=item.confidence,
rotation=getattr(item, "rotation", 0.0),
words=[
@@ -247,7 +247,7 @@ class LiteParse:
ocr_failure_fatal: Optional[bool] = None,
ocr_hedge_delays_ms: Optional[List[int]] = None,
emit_word_boxes: Optional[bool] = None,
- include_text_metadata: Optional[bool] = None,
+ extract_text_metadata: Optional[bool] = None,
crop_box: Optional[Tuple[float, float, float, float]] = None,
skip_diagonal_text: Optional[bool] = None,
include_complexity: Optional[bool] = None,
@@ -274,11 +274,11 @@ class LiteParse:
extract_links: Render hyperlink annotations as ``[text](url)`` in
markdown output (default: True). Set False for plain anchor text.
image_output_dir: Directory where extracted embedded images are
- written. Setting it enables image extraction and returns file
+ written. Requires ``extract_images=True`` and returns file
names/paths in each ``ExtractedImage``.
extract_images: Extract embedded image bytes and metadata. Defaults
- to False; ``image_mode="embed"`` and ``image_output_dir`` also
- imply extraction for compatibility.
+ to False. This is the only option that enables extraction;
+ ``image_mode`` controls Markdown presentation independently.
extract_annotations: Include all PDF annotations as page-scoped
structured data (default: False).
ocr_failure_fatal: Whether a systemic OCR failure (every OCR task
@@ -296,9 +296,9 @@ class LiteParse:
(``TextItem.words``). Default False. Word boxes roughly double
the text-item payload, so enable only for word-level bbox
attribution.
- include_text_metadata: Include rich PDF text metadata on returned
+ extract_text_metadata: Include rich PDF text metadata on returned
text items (MCID, glyph width, font metrics/weight/buggy state,
- colors, raw character codes, and ``tsg``). Default False.
+ colors, raw character codes, and ``trailing_space_generated``). Default False.
crop_box: Restrict output to a page sub-region, as a
``(top, right, bottom, left)`` tuple where each value is the
fraction cropped from that side (top-left origin, each in
@@ -359,8 +359,8 @@ class LiteParse:
kwargs["ocr_hedge_delays_ms"] = ocr_hedge_delays_ms
if emit_word_boxes is not None:
kwargs["emit_word_boxes"] = emit_word_boxes
- if include_text_metadata is not None:
- kwargs["include_text_metadata"] = include_text_metadata
+ if extract_text_metadata is not None:
+ kwargs["extract_text_metadata"] = extract_text_metadata
if crop_box is not None:
kwargs["crop_box"] = crop_box
if skip_diagonal_text is not None:
@@ -511,7 +511,7 @@ class LiteParse:
crop_box=cfg.crop_box,
skip_diagonal_text=cfg.skip_diagonal_text,
include_complexity=cfg.include_complexity,
- include_text_metadata=cfg.include_text_metadata,
+ extract_text_metadata=cfg.extract_text_metadata,
extract_images=cfg.extract_images,
extract_vector_graphics=cfg.extract_vector_graphics,
)
diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py
index c6822a3..159955c 100644
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@ -41,7 +41,7 @@ class TextItem:
#: Raw PDF content-stream character codes for the source glyphs.
char_codes: List[int] = field(default_factory=list)
#: True when the trailing source space was synthesized by PDFium.
- tsg: bool = False
+ trailing_space_generated: bool = False
confidence: Optional[float] = None
rotation: float = 0.0
#: Per-word sub-boxes. Empty unless the parser was configured with
@@ -132,8 +132,8 @@ class ImageRect:
class ExtractedImage:
"""An embedded raster image extracted from a page.
- Populated when ``extract_images=True``, ``image_mode="embed"``, or an image
- output directory is configured.
+ Populated only when ``extract_images=True``. ``image_mode`` controls
+ Markdown presentation independently.
The ``id`` matches the reference used in the markdown output
(e.g. ``![](image_p1_0.png)`` → ``id="p1_0"``).
"""
@@ -254,7 +254,7 @@ class LiteParseConfig:
crop_box: Optional[Tuple[float, float, float, float]]
skip_diagonal_text: bool
include_complexity: bool
- include_text_metadata: bool = False
+ extract_text_metadata: bool = False
extract_images: bool = False
extract_vector_graphics: bool = False