Skip to content

A fast, helpful, and open-source document parser

Make rich text metadata opt-in

Pierre-Loic doulcet5bbe1230dfa3
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs
index c31e71d..9606a99 100644
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@ -59,6 +59,8 @@ pub struct JsLiteParseConfig {
/// false. Word boxes roughly double the text-item payload, so enable only
/// for word-level bbox attribution.
pub emit_word_boxes: Option<bool>,
+ /// Include rich PDF text metadata on returned text items. Default false.
+ pub include_text_metadata: Option<bool>,
/// Restrict output to a page sub-region. Each field is the fraction of the
/// page cropped from that side; a text item survives only if it lies
/// entirely inside the remaining rectangle. Unset keeps the whole page.
@@ -148,6 +150,9 @@ impl JsLiteParseConfig {
if let Some(v) = self.emit_word_boxes {
cfg.emit_word_boxes = v;
}
+ if let Some(v) = self.include_text_metadata {
+ cfg.include_text_metadata = v;
+ }
if let Some(v) = self.crop_box {
cfg.crop_box = Some(CropBox {
top: v.top as f32,
@@ -202,6 +207,7 @@ impl JsLiteParseConfig {
.collect(),
),
emit_word_boxes: Some(cfg.emit_word_boxes),
+ include_text_metadata: Some(cfg.include_text_metadata),
crop_box: cfg.crop_box.map(|c| JsCropBox {
top: c.top as f64,
right: c.right as f64,
@@ -326,6 +332,24 @@ impl JsTextItem {
words: item.words.iter().map(JsWordBox::from_rust).collect(),
}
}
+
+ fn from_rust_for_output(item: &TextItem, include_text_metadata: bool) -> Self {
+ let mut output = Self::from_rust(item);
+ if !include_text_metadata {
+ output.font_height = None;
+ output.font_ascent = None;
+ output.font_descent = None;
+ output.font_weight = None;
+ output.text_width = None;
+ output.font_is_buggy = None;
+ output.mcid = None;
+ output.fill_color = None;
+ output.stroke_color = None;
+ output.char_codes = None;
+ output.tsg = None;
+ }
+ output
+ }
}
// ---------------------------------------------------------------------------
@@ -452,14 +476,18 @@ pub struct JsParsedPage {
}
impl JsParsedPage {
- pub fn from_rust(page: &ParsedPage) -> Self {
+ pub fn from_rust(page: &ParsedPage, include_text_metadata: bool) -> Self {
Self {
page_num: page.page_number as u32,
width: page.page_width as f64,
height: page.page_height as f64,
text: page.text.clone(),
markdown: page.markdown.clone(),
- text_items: page.text_items.iter().map(JsTextItem::from_rust).collect(),
+ text_items: page
+ .text_items
+ .iter()
+ .map(|item| JsTextItem::from_rust_for_output(item, include_text_metadata))
+ .collect(),
complexity: page
.complexity
.as_ref()
@@ -582,9 +610,13 @@ impl JsPageComplexityStats {
}
impl JsParseResult {
- pub fn from_rust(result: &ParseResult, _config: &LiteParseConfig) -> Self {
+ pub fn from_rust(result: &ParseResult, config: &LiteParseConfig) -> Self {
Self {
- pages: result.pages.iter().map(JsParsedPage::from_rust).collect(),
+ pages: result
+ .pages
+ .iter()
+ .map(|page| JsParsedPage::from_rust(page, config.include_text_metadata))
+ .collect(),
text: result.text.clone(),
images: result
.images
@@ -627,6 +659,12 @@ mod tests {
assert_eq!(js.tsg, Some(true));
assert_eq!(js.fill_color.as_deref(), Some("ff112233"));
+ let lightweight = JsTextItem::from_rust_for_output(&item, false);
+ assert_eq!(lightweight.font_height, None);
+ assert_eq!(lightweight.font_is_buggy, None);
+ assert_eq!(lightweight.char_codes, None);
+ assert_eq!(lightweight.tsg, None);
+
let round_trip = js.to_rust();
assert_eq!(round_trip.font_height, Some(12.0));
assert_eq!(round_trip.font_ascent, Some(9.0));
@@ -639,4 +677,12 @@ mod tests {
assert_eq!(round_trip.char_codes, vec![65, 32]);
assert!(round_trip.tsg);
}
+
+ #[test]
+ fn text_metadata_config_defaults_off_and_round_trips() {
+ let mut js = JsLiteParseConfig::from_rust(&LiteParseConfig::default());
+ assert_eq!(js.include_text_metadata, Some(false));
+ js.include_text_metadata = Some(true);
+ assert!(js.into_rust().include_text_metadata);
+ }
}
diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rs
index 2d41bb1..df46d62 100644
--- a/crates/liteparse-python/src/cli.rs
+++ b/crates/liteparse-python/src/cli.rs
@@ -79,6 +79,10 @@ struct ParseCommand {
/// page of JSON output. Off by default.
#[arg(long)]
complexity: bool,
+
+ /// Include rich PDF text metadata in text items and JSON output.
+ #[arg(long)]
+ include_text_metadata: bool,
}
#[derive(Args, Debug)]
@@ -150,6 +154,10 @@ struct BatchParseCommand {
/// page of JSON output. Off by default.
#[arg(long)]
complexity: bool,
+
+ /// Include rich PDF text metadata in text items and JSON output.
+ #[arg(long)]
+ include_text_metadata: bool,
}
/// Parse a `Name: Value` header string into a `(name, value)` pair.
@@ -228,6 +236,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
image_mode,
extract_links: !cmd.no_links,
include_complexity: cmd.complexity,
+ include_text_metadata: cmd.include_text_metadata,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -240,7 +249,10 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
rt.block_on(lp.parse(&cmd.file))?
};
let formatted = match lp.config().output_format {
- OutputFormat::Json => json::format_json(&result.pages)?,
+ OutputFormat::Json => json::format_json_with_text_metadata(
+ &result.pages,
+ lp.config().include_text_metadata,
+ )?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
};
@@ -328,6 +340,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
include_complexity: cmd.complexity,
+ include_text_metadata: cmd.include_text_metadata,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -368,9 +381,11 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
Ok(result) => {
let fmt_result: Result<String, Box<dyn std::error::Error>> =
match lp.config().output_format {
- OutputFormat::Json => {
- json::format_json(&result.pages).map_err(|e| e.into())
- }
+ OutputFormat::Json => json::format_json_with_text_metadata(
+ &result.pages,
+ lp.config().include_text_metadata,
+ )
+ .map_err(|e| e.into()),
OutputFormat::Text => Ok(text::format_text(&result.pages)),
OutputFormat::Markdown => Ok(result.text.clone()),
};
@@ -522,7 +537,7 @@ fn collect_files_inner(
#[cfg(test)]
mod tests {
- use super::batch_output_path;
+ use super::*;
use std::path::Path;
#[test]
@@ -538,4 +553,33 @@ mod tests {
assert_eq!(out_path, Path::new("out/nested/report.md"));
}
+
+ #[test]
+ fn include_text_metadata_flag_is_available_for_parse_and_batch() {
+ let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--include-text-metadata"])
+ .unwrap();
+ assert!(matches!(
+ cli.command,
+ Commands::Parse(ParseCommand {
+ include_text_metadata: true,
+ ..
+ })
+ ));
+
+ let cli = Cli::try_parse_from([
+ "lit",
+ "batch-parse",
+ "input",
+ "output",
+ "--include-text-metadata",
+ ])
+ .unwrap();
+ assert!(matches!(
+ cli.command,
+ Commands::BatchParse(BatchParseCommand {
+ include_text_metadata: true,
+ ..
+ })
+ ));
+ }
}
diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs
index 56e4cda..8f8a8d5 100644
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@ -508,6 +508,8 @@ struct PyLiteParseConfig {
skip_diagonal_text: bool,
#[pyo3(get)]
include_complexity: bool,
+ #[pyo3(get)]
+ include_text_metadata: bool,
}
#[pymethods]
@@ -559,6 +561,7 @@ impl PyLiteParseConfig {
.map(|c| (c.top, c.right, c.bottom, c.left)),
skip_diagonal_text: cfg.skip_diagonal_text,
include_complexity: cfg.include_complexity,
+ include_text_metadata: cfg.include_text_metadata,
}
}
}
@@ -597,6 +600,7 @@ impl LiteParse {
ocr_failure_fatal = None,
ocr_hedge_delays_ms = None,
emit_word_boxes = None,
+ include_text_metadata = None,
crop_box = None,
skip_diagonal_text = None,
include_complexity = None,
@@ -620,6 +624,7 @@ impl LiteParse {
ocr_failure_fatal: Option<bool>,
ocr_hedge_delays_ms: Option<Vec<u64>>,
emit_word_boxes: Option<bool>,
+ include_text_metadata: Option<bool>,
crop_box: Option<(f32, f32, f32, f32)>,
skip_diagonal_text: Option<bool>,
include_complexity: Option<bool>,
@@ -687,6 +692,9 @@ impl LiteParse {
if let Some(v) = emit_word_boxes {
cfg.emit_word_boxes = v;
}
+ if let Some(v) = include_text_metadata {
+ cfg.include_text_metadata = v;
+ }
if let Some((top, right, bottom, left)) = crop_box {
cfg.crop_box = Some(CropBox {
top,
@@ -921,4 +929,17 @@ mod tests {
assert_eq!(round_trip.char_codes, vec![65, 32]);
assert!(round_trip.tsg);
}
+
+ #[test]
+ fn text_metadata_config_defaults_off_and_can_be_enabled() {
+ let py = PyLiteParseConfig::from_rust(&LiteParseConfig::default());
+ assert!(!py.include_text_metadata);
+
+ let config = LiteParseConfig {
+ include_text_metadata: true,
+ ..Default::default()
+ };
+ let py = PyLiteParseConfig::from_rust(&config);
+ assert!(py.include_text_metadata);
+ }
}
diff --git a/crates/liteparse/README.md b/crates/liteparse/README.md
index 0d6f168..8a54672 100644
--- a/crates/liteparse/README.md
+++ b/crates/liteparse/README.md
@@ -57,6 +57,7 @@ let config = LiteParseConfig {
dpi: 150.0, // Rendering DPI
output_format: OutputFormat::Json, // Json | Text | Markdown
preserve_very_small_text: false, // Keep tiny text
+ include_text_metadata: false, // Opt in to rich PDF text metadata
password: None, // Password for protected documents
quiet: false, // Suppress progress output
..Default::default()
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs
index 75ec5c3..0b5c60f 100644
--- a/crates/liteparse/src/config.rs
+++ b/crates/liteparse/src/config.rs
@@ -60,6 +60,13 @@ pub struct LiteParseConfig {
/// `false`, `TextItem.words` is always empty and the per-word tracking is
/// skipped entirely (zero allocation).
pub emit_word_boxes: bool,
+ /// Include rich PDF text metadata on public text items: MCID, glyph width,
+ /// font metrics/weight/buggy state, fill/stroke colors, raw character
+ /// codes, and generated-trailing-space state. Default `false` to preserve
+ /// LiteParse's lightweight response shape. Extraction still computes the
+ /// metadata for internal layout; it is stripped only before returning.
+ #[serde(default)]
+ pub include_text_metadata: bool,
/// Restrict output to a sub-region of every page. Each field is the
/// fraction of the page to crop away from that side (e.g. `left = 0.5`
/// discards the left half). A text item is kept only when it lies
@@ -148,6 +155,7 @@ impl Default for LiteParseConfig {
ocr_failure_fatal: true,
ocr_hedge_delays_ms: Vec::new(),
emit_word_boxes: false,
+ include_text_metadata: false,
crop_box: None,
skip_diagonal_text: false,
include_complexity: false,
@@ -268,6 +276,7 @@ mod tests {
assert_eq!(c.output_format, OutputFormat::Json);
assert!(!c.preserve_very_small_text);
assert!(!c.quiet);
+ assert!(!c.include_text_metadata);
assert!(c.password.is_none());
}
diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs
index e095571..198c026 100644
--- a/crates/liteparse/src/main.rs
+++ b/crates/liteparse/src/main.rs
@@ -126,6 +126,9 @@ struct ParseCommand {
/// it runs the extra vector-text detection pass.
#[arg(long)]
complexity: bool,
+ /// Include rich PDF text metadata in text items and JSON output.
+ #[arg(long)]
+ include_text_metadata: bool,
}
#[derive(Args, Debug)]
@@ -219,6 +222,9 @@ struct BatchParseCommand {
/// page of JSON output. Off by default.
#[arg(long)]
complexity: bool,
+ /// Include rich PDF text metadata in text items and JSON output.
+ #[arg(long)]
+ include_text_metadata: bool,
}
#[derive(Args, Debug)]
@@ -337,6 +343,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
image_mode,
extract_links: !cmd.no_links,
include_complexity: cmd.complexity,
+ include_text_metadata: cmd.include_text_metadata,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -350,7 +357,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
lp.parse(&cmd.file).await?
};
let formatted = match lp.config().output_format {
- OutputFormat::Json => json::format_json(&result.pages)?,
+ OutputFormat::Json => json::format_json_with_text_metadata(
+ &result.pages,
+ lp.config().include_text_metadata,
+ )?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
};
@@ -442,6 +452,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
include_complexity: cmd.complexity,
+ include_text_metadata: cmd.include_text_metadata,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -485,9 +496,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(result) => {
let fmt_result: Result<String, Box<dyn std::error::Error>> =
match lp.config().output_format {
- OutputFormat::Json => {
- json::format_json(&result.pages).map_err(|e| e.into())
- }
+ OutputFormat::Json => json::format_json_with_text_metadata(
+ &result.pages,
+ lp.config().include_text_metadata,
+ )
+ .map_err(|e| e.into()),
OutputFormat::Text => Ok(text::format_text(&result.pages)),
OutputFormat::Markdown => Ok(result.text.clone()),
};
@@ -667,7 +680,7 @@ fn collect_files_inner(
#[cfg(test)]
mod tests {
- use super::batch_output_path;
+ use super::*;
use std::path::Path;
#[test]
@@ -683,4 +696,33 @@ mod tests {
assert_eq!(out_path, Path::new("out/nested/report.md"));
}
+
+ #[test]
+ fn include_text_metadata_flag_is_available_for_parse_and_batch() {
+ let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--include-text-metadata"])
+ .unwrap();
+ assert!(matches!(
+ cli.command,
+ Commands::Parse(ParseCommand {
+ include_text_metadata: true,
+ ..
+ })
+ ));
+
+ let cli = Cli::try_parse_from([
+ "lit",
+ "batch-parse",
+ "input",
+ "output",
+ "--include-text-metadata",
+ ])
+ .unwrap();
+ assert!(matches!(
+ cli.command,
+ Commands::BatchParse(BatchParseCommand {
+ include_text_metadata: true,
+ ..
+ })
+ ));
+ }
}
diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs
index 6ebe7a2..dbba09d 100644
--- a/crates/liteparse/src/output/json.rs
+++ b/crates/liteparse/src/output/json.rs
@@ -9,7 +9,8 @@ pub(crate) struct JsonTextItem {
pub y: f32,
pub width: f32,
pub height: f32,
- pub rotation: f32,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub rotation: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub font_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -24,7 +25,8 @@ pub(crate) struct JsonTextItem {
pub font_weight: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub text_width: Option<f32>,
- pub font_is_buggy: bool,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub font_is_buggy: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mcid: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -56,7 +58,7 @@ pub(crate) struct ParseResultJson {
}
/// Build structured JSON output from parsed pages.
-pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {
+pub(crate) fn build_json(pages: &[ParsedPage], include_text_metadata: bool) -> ParseResultJson {
ParseResultJson {
pages: pages
.iter()
@@ -74,20 +76,28 @@ pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {
y: item.y,
width: item.width,
height: item.height,
- rotation: item.rotation,
+ rotation: include_text_metadata.then_some(item.rotation),
font_name: item.font_name.clone(),
font_size: item.font_size,
- font_height: item.font_height,
- font_ascent: item.font_ascent,
- font_descent: item.font_descent,
- font_weight: item.font_weight,
- text_width: item.text_width,
- font_is_buggy: item.font_is_buggy,
- mcid: item.mcid,
- fill_color: item.fill_color.clone(),
- stroke_color: item.stroke_color.clone(),
- char_codes: item.char_codes.clone(),
- tsg: item.tsg,
+ font_height: include_text_metadata.then_some(item.font_height).flatten(),
+ font_ascent: include_text_metadata.then_some(item.font_ascent).flatten(),
+ font_descent: include_text_metadata.then_some(item.font_descent).flatten(),
+ font_weight: include_text_metadata.then_some(item.font_weight).flatten(),
+ text_width: include_text_metadata.then_some(item.text_width).flatten(),
+ font_is_buggy: include_text_metadata.then_some(item.font_is_buggy),
+ mcid: include_text_metadata.then_some(item.mcid).flatten(),
+ fill_color: include_text_metadata
+ .then(|| item.fill_color.clone())
+ .flatten(),
+ stroke_color: include_text_metadata
+ .then(|| item.stroke_color.clone())
+ .flatten(),
+ char_codes: if include_text_metadata {
+ item.char_codes.clone()
+ } else {
+ Vec::new()
+ },
+ tsg: include_text_metadata && item.tsg,
confidence: item.confidence.or(Some(1.0)),
})
.collect(),
@@ -99,7 +109,15 @@ pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {
/// Format parsed pages as pretty-printed JSON string.
pub fn format_json(pages: &[ParsedPage]) -> Result<String, serde_json::Error> {
- let result = build_json(pages);
+ format_json_with_text_metadata(pages, false)
+}
+
+/// Format parsed pages as JSON, optionally including rich PDF text metadata.
+pub fn format_json_with_text_metadata(
+ pages: &[ParsedPage],
+ include_text_metadata: bool,
+) -> Result<String, serde_json::Error> {
+ let result = build_json(pages, include_text_metadata);
serde_json::to_string_pretty(&result)
}
@@ -142,7 +160,7 @@ mod tests {
#[test]
fn test_build_json_native_text_defaults_confidence_to_one() {
- let j = build_json(&[page(vec![item("hi", None)])]);
+ let j = build_json(&[page(vec![item("hi", None)])], false);
assert_eq!(j.pages.len(), 1);
assert_eq!(j.pages[0].page, 1);
assert_eq!(j.pages[0].text_items[0].confidence, Some(1.0));
@@ -151,7 +169,7 @@ mod tests {
#[test]
fn test_build_json_preserves_ocr_confidence() {
- let j = build_json(&[page(vec![item("hi", Some(0.42))])]);
+ let j = build_json(&[page(vec![item("hi", Some(0.42))])], false);
assert_eq!(j.pages[0].text_items[0].confidence, Some(0.42));
}
@@ -178,8 +196,10 @@ mod tests {
text_item.char_codes = vec![104, 105, 32];
text_item.tsg = true;
- let value: serde_json::Value =
- serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();
+ let value: serde_json::Value = serde_json::from_str(
+ &format_json_with_text_metadata(&[page(vec![text_item])], true).unwrap(),
+ )
+ .unwrap();
let item = &value["pages"][0]["text_items"][0];
assert_eq!(item["font_height"], 11.0);
assert_eq!(item["font_ascent"], 8.0);
@@ -197,7 +217,27 @@ mod tests {
#[test]
fn test_build_json_empty() {
- let j = build_json(&[]);
+ let j = build_json(&[], false);
assert!(j.pages.is_empty());
}
+
+ #[test]
+ fn test_text_metadata_is_omitted_by_default() {
+ let mut text_item = item("hi", None);
+ text_item.font_height = Some(11.0);
+ text_item.font_is_buggy = true;
+ text_item.mcid = Some(4);
+ text_item.char_codes = vec![104, 105];
+ text_item.tsg = true;
+
+ let value: serde_json::Value =
+ serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();
+ let item = &value["pages"][0]["text_items"][0];
+ assert!(item.get("rotation").is_none());
+ assert!(item.get("font_height").is_none());
+ assert!(item.get("font_is_buggy").is_none());
+ assert!(item.get("mcid").is_none());
+ assert!(item.get("char_codes").is_none());
+ assert!(item.get("tsg").is_none());
+ }
}
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 3ce72a1..d6cd56a 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -433,6 +433,10 @@ impl LiteParse {
.join("\n\n")
};
+ if !self.config.include_text_metadata {
+ strip_text_metadata(&mut parsed_pages);
+ }
+
let total = web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;
log(&format!("[liteparse] total: {:.1}ms", total));
@@ -471,6 +475,10 @@ impl LiteParse {
.join("\n\n")
};
+ if !self.config.include_text_metadata {
+ strip_text_metadata(&mut parsed_pages);
+ }
+
ParseResult {
pages: parsed_pages,
text: full_text,
@@ -539,9 +547,60 @@ impl LiteParse {
}
}
+/// Remove opt-in extraction metadata only after projection/Markdown have had
+/// access to it. Longstanding lightweight fields (text, bbox, rotation,
+/// font name/size, confidence, links, strike state, and optional word boxes)
+/// remain unchanged.
+fn strip_text_metadata(pages: &mut [crate::types::ParsedPage]) {
+ for item in pages.iter_mut().flat_map(|page| &mut page.text_items) {
+ item.font_height = None;
+ item.font_ascent = None;
+ item.font_descent = None;
+ item.font_weight = None;
+ item.text_width = None;
+ item.font_is_buggy = false;
+ item.mcid = None;
+ item.fill_color = None;
+ item.stroke_color = None;
+ item.char_codes.clear();
+ item.tsg = false;
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
+ use crate::types::{Page, TextItem};
+
+ fn page_with_text_metadata() -> Page {
+ Page {
+ page_number: 1,
+ page_width: 100.0,
+ page_height: 100.0,
+ text_items: vec![TextItem {
+ text: "hello".into(),
+ width: 20.0,
+ height: 10.0,
+ font_name: Some("Helvetica".into()),
+ font_size: Some(10.0),
+ font_height: Some(10.0),
+ font_ascent: Some(8.0),
+ font_descent: Some(-2.0),
+ font_weight: Some(700),
+ text_width: Some(19.0),
+ font_is_buggy: true,
+ mcid: Some(3),
+ fill_color: Some("ff112233".into()),
+ stroke_color: Some("ff445566".into()),
+ char_codes: vec![104, 101, 108, 108, 111],
+ tsg: true,
+ ..Default::default()
+ }],
+ graphics: vec![],
+ struct_nodes: vec![],
+ image_refs: vec![],
+ }
+ }
#[test]
#[allow(clippy::field_reassign_with_default)]
@@ -553,4 +612,41 @@ mod tests {
assert!(!lp.config().ocr_enabled);
assert_eq!(lp.config().max_pages, 7);
}
+
+ #[test]
+ fn parse_result_strips_text_metadata_by_default() {
+ let result = LiteParse::new(LiteParseConfig::default())
+ .parse_from_pages(vec![page_with_text_metadata()], vec![]);
+ let item = &result.pages[0].text_items[0];
+ assert_eq!(item.font_name.as_deref(), Some("Helvetica"));
+ assert_eq!(item.font_size, Some(10.0));
+ assert_eq!(item.font_height, None);
+ assert_eq!(item.font_weight, None);
+ assert_eq!(item.mcid, None);
+ assert!(item.char_codes.is_empty());
+ assert!(!item.font_is_buggy);
+ assert!(!item.tsg);
+ }
+
+ #[test]
+ fn parse_result_preserves_text_metadata_when_enabled() {
+ let config = LiteParseConfig {
+ include_text_metadata: true,
+ ..Default::default()
+ };
+ let result =
+ LiteParse::new(config).parse_from_pages(vec![page_with_text_metadata()], vec![]);
+ let item = &result.pages[0].text_items[0];
+ assert_eq!(item.font_height, Some(10.0));
+ assert_eq!(item.font_ascent, Some(8.0));
+ assert_eq!(item.font_descent, Some(-2.0));
+ assert_eq!(item.font_weight, Some(700));
+ assert_eq!(item.text_width, Some(19.0));
+ assert!(item.font_is_buggy);
+ assert_eq!(item.mcid, Some(3));
+ assert_eq!(item.fill_color.as_deref(), Some("ff112233"));
+ assert_eq!(item.stroke_color.as_deref(), Some("ff445566"));
+ assert_eq!(item.char_codes, vec![104, 101, 108, 108, 111]);
+ assert!(item.tsg);
+ }
}
diff --git a/packages/node/README.md b/packages/node/README.md
index c30a02f..f3898ba 100644
--- a/packages/node/README.md
+++ b/packages/node/README.md
@@ -60,6 +60,7 @@ const parser = new LiteParse({
imageMode: 'placeholder', // Markdown image handling: "placeholder" | "off" | "embed"
extractLinks: true, // Render [text](url) links in markdown output
preserveVerySmallText: false, // Keep tiny text
+ includeTextMetadata: false, // Opt in to MCID, font metrics, colors, char codes, and tsg
password: undefined, // Password for protected documents
quiet: false, // Suppress progress output
numWorkers: 4, // Concurrent OCR workers
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index 5e8dfd7..37d2dd5 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -62,6 +62,8 @@ export interface JsLiteParseConfig {
* for word-level bbox attribution.
*/
emitWordBoxes?: boolean
+ /** Include rich PDF text metadata on returned text items. Default false. */
+ includeTextMetadata?: boolean
/**
* Restrict output to a page sub-region. Each field is the fraction of the
* page cropped from that side; a text item survives only if it lies
diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts
index 9be4554..c82e528 100644
--- a/packages/node/src/cli.ts
+++ b/packages/node/src/cli.ts
@@ -78,6 +78,10 @@ program
)
.option("--dpi <dpi>", "Rendering DPI", parseFloat)
.option("--preserve-small-text", "Keep very small text")
+ .option(
+ "--include-text-metadata",
+ "Include rich PDF text metadata in text items and JSON output",
+ )
.option("--password <password>", "Password for encrypted documents")
.option("--config <file>", "JSON config file path")
.option("-q, --quiet", "Suppress progress output")
@@ -113,6 +117,7 @@ program
if (opts.targetPages) config.targetPages = opts.targetPages as string;
if (opts.dpi) config.dpi = opts.dpi as number;
if (opts.preserveSmallText) config.preserveVerySmallText = true;
+ if (opts.includeTextMetadata) config.includeTextMetadata = true;
if (opts.password) config.password = opts.password as string;
if (opts.quiet) config.quiet = true;
if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;
@@ -313,6 +318,10 @@ program
.option("--password <password>", "Password for encrypted documents")
.option("-q, --quiet", "Suppress progress output")
.option("--num-workers <n>", "Number of concurrent OCR workers", parseInt)
+ .option(
+ "--include-text-metadata",
+ "Include rich PDF text metadata in text items and JSON output",
+ )
.action(
async (
inputDir: string,
@@ -334,6 +343,7 @@ program
if (opts.password) config.password = opts.password as string;
if (opts.quiet) config.quiet = true;
if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;
+ if (opts.includeTextMetadata) config.includeTextMetadata = true;
const parser = new LiteParse(config);
const outExt = format === "json" ? ".json" : format === "markdown" ? ".md" : ".txt";
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index 926fa39..c4fbc1d 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -57,6 +57,8 @@ export interface LiteParseConfig {
* marshalling), so enable only when doing word-level bbox attribution.
*/
emitWordBoxes: boolean;
+ /** Include rich PDF text metadata on returned text items. Default false. */
+ includeTextMetadata?: boolean;
/**
* Restrict output to a page sub-region. Each field is the fraction of the
* page cropped away from that side (top-left origin), so `{ left: 0.5 }`
@@ -320,6 +322,7 @@ export class LiteParse {
ocrFailureFatal: userConfig.ocrFailureFatal,
ocrHedgeDelaysMs: userConfig.ocrHedgeDelaysMs,
emitWordBoxes: userConfig.emitWordBoxes,
+ includeTextMetadata: userConfig.includeTextMetadata,
cropBox: userConfig.cropBox,
skipDiagonalText: userConfig.skipDiagonalText,
includeComplexity: userConfig.includeComplexity,
@@ -348,6 +351,7 @@ export class LiteParse {
ocrFailureFatal: resolved.ocrFailureFatal ?? true,
ocrHedgeDelaysMs: resolved.ocrHedgeDelaysMs ?? [],
emitWordBoxes: resolved.emitWordBoxes ?? false,
+ includeTextMetadata: resolved.includeTextMetadata ?? false,
cropBox: resolved.cropBox ?? undefined,
skipDiagonalText: resolved.skipDiagonalText ?? false,
includeComplexity: resolved.includeComplexity ?? false,
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index 29d8b2a..a512109 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -38,6 +38,7 @@ export interface LiteParseNativeConfig {
ocrFailureFatal?: boolean;
ocrHedgeDelaysMs?: number[];
emitWordBoxes?: boolean;
+ includeTextMetadata?: boolean;
cropBox?: NativeCropBox;
skipDiagonalText?: boolean;
includeComplexity?: boolean;
diff --git a/packages/python/README.md b/packages/python/README.md
index a624501..0454b2b 100644
--- a/packages/python/README.md
+++ b/packages/python/README.md
@@ -59,6 +59,7 @@ parser = LiteParse(
image_mode="placeholder", # Markdown image handling: "placeholder" | "off" | "embed"
extract_links=True, # Render [text](url) links in markdown output
preserve_very_small_text=False, # Keep tiny text
+ include_text_metadata=False, # Opt in to MCID, font metrics, colors, char codes, and tsg
password=None, # Password for protected documents
quiet=False, # Suppress progress output
num_workers=4, # Concurrent OCR workers
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index e5f76fc..1e04abf 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -158,6 +158,7 @@ class LiteParse:
ocr_failure_fatal: Optional[bool] = None,
ocr_hedge_delays_ms: Optional[List[int]] = None,
emit_word_boxes: Optional[bool] = None,
+ include_text_metadata: Optional[bool] = None,
crop_box: Optional[Tuple[float, float, float, float]] = None,
skip_diagonal_text: Optional[bool] = None,
include_complexity: Optional[bool] = None,
@@ -197,6 +198,9 @@ class LiteParse:
(``TextItem.words``). Default False. Word boxes roughly double
the text-item payload, so enable only for word-level bbox
attribution.
+ include_text_metadata: Include rich PDF text metadata on returned
+ text items (MCID, glyph width, font metrics/weight/buggy state,
+ colors, raw character codes, and ``tsg``). Default False.
crop_box: Restrict output to a page sub-region, as a
``(top, right, bottom, left)`` tuple where each value is the
fraction cropped from that side (top-left origin, each in
@@ -249,6 +253,8 @@ class LiteParse:
kwargs["ocr_hedge_delays_ms"] = ocr_hedge_delays_ms
if emit_word_boxes is not None:
kwargs["emit_word_boxes"] = emit_word_boxes
+ if include_text_metadata is not None:
+ kwargs["include_text_metadata"] = include_text_metadata
if crop_box is not None:
kwargs["crop_box"] = crop_box
if skip_diagonal_text is not None:
@@ -395,6 +401,7 @@ class LiteParse:
crop_box=cfg.crop_box,
skip_diagonal_text=cfg.skip_diagonal_text,
include_complexity=cfg.include_complexity,
+ include_text_metadata=cfg.include_text_metadata,
)
def __repr__(self) -> str:
diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py
index 989d8b6..3951bbc 100644
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@ -179,6 +179,7 @@ class LiteParseConfig:
crop_box: Optional[Tuple[float, float, float, float]]
skip_diagonal_text: bool
include_complexity: bool
+ include_text_metadata: bool = False
class ParseError(Exception):