A fast, helpful, and open-source document parser
Make rich text metadata opt-in
Pierre-Loic doulcet5bbe1230dfa3
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rsindex c31e71d..9606a99 100644--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@ -59,6 +59,8 @@ pub struct JsLiteParseConfig {/// false. Word boxes roughly double the text-item payload, so enable only/// for word-level bbox attribution.pub emit_word_boxes: Option<bool>,+ /// Include rich PDF text metadata on returned text items. Default false.+ pub include_text_metadata: Option<bool>,/// Restrict output to a page sub-region. Each field is the fraction of the/// page cropped from that side; a text item survives only if it lies/// entirely inside the remaining rectangle. Unset keeps the whole page.@@ -148,6 +150,9 @@ impl JsLiteParseConfig {if let Some(v) = self.emit_word_boxes {cfg.emit_word_boxes = v;}+ if let Some(v) = self.include_text_metadata {+ cfg.include_text_metadata = v;+ }if let Some(v) = self.crop_box {cfg.crop_box = Some(CropBox {top: v.top as f32,@@ -202,6 +207,7 @@ impl JsLiteParseConfig {.collect(),),emit_word_boxes: Some(cfg.emit_word_boxes),+ include_text_metadata: Some(cfg.include_text_metadata),crop_box: cfg.crop_box.map(|c| JsCropBox {top: c.top as f64,right: c.right as f64,@@ -326,6 +332,24 @@ impl JsTextItem {words: item.words.iter().map(JsWordBox::from_rust).collect(),}}++ fn from_rust_for_output(item: &TextItem, include_text_metadata: bool) -> Self {+ let mut output = Self::from_rust(item);+ if !include_text_metadata {+ output.font_height = None;+ output.font_ascent = None;+ output.font_descent = None;+ output.font_weight = None;+ output.text_width = None;+ output.font_is_buggy = None;+ output.mcid = None;+ output.fill_color = None;+ output.stroke_color = None;+ output.char_codes = None;+ output.tsg = None;+ }+ output+ }}// ---------------------------------------------------------------------------@@ -452,14 +476,18 @@ pub struct JsParsedPage {}impl JsParsedPage {- pub fn from_rust(page: &ParsedPage) -> Self {+ pub fn from_rust(page: &ParsedPage, include_text_metadata: bool) -> Self {Self {page_num: page.page_number as u32,width: page.page_width as f64,height: page.page_height as f64,text: page.text.clone(),markdown: page.markdown.clone(),- text_items: page.text_items.iter().map(JsTextItem::from_rust).collect(),+ text_items: page+ .text_items+ .iter()+ .map(|item| JsTextItem::from_rust_for_output(item, include_text_metadata))+ .collect(),complexity: page.complexity.as_ref()@@ -582,9 +610,13 @@ impl JsPageComplexityStats {}impl JsParseResult {- pub fn from_rust(result: &ParseResult, _config: &LiteParseConfig) -> Self {+ pub fn from_rust(result: &ParseResult, config: &LiteParseConfig) -> Self {Self {- pages: result.pages.iter().map(JsParsedPage::from_rust).collect(),+ pages: result+ .pages+ .iter()+ .map(|page| JsParsedPage::from_rust(page, config.include_text_metadata))+ .collect(),text: result.text.clone(),images: result.images@@ -627,6 +659,12 @@ mod tests {assert_eq!(js.tsg, Some(true));assert_eq!(js.fill_color.as_deref(), Some("ff112233"));+ let lightweight = JsTextItem::from_rust_for_output(&item, false);+ assert_eq!(lightweight.font_height, None);+ assert_eq!(lightweight.font_is_buggy, None);+ assert_eq!(lightweight.char_codes, None);+ assert_eq!(lightweight.tsg, None);+let round_trip = js.to_rust();assert_eq!(round_trip.font_height, Some(12.0));assert_eq!(round_trip.font_ascent, Some(9.0));@@ -639,4 +677,12 @@ mod tests {assert_eq!(round_trip.char_codes, vec![65, 32]);assert!(round_trip.tsg);}++ #[test]+ fn text_metadata_config_defaults_off_and_round_trips() {+ let mut js = JsLiteParseConfig::from_rust(&LiteParseConfig::default());+ assert_eq!(js.include_text_metadata, Some(false));+ js.include_text_metadata = Some(true);+ assert!(js.into_rust().include_text_metadata);+ }}diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rsindex 2d41bb1..df46d62 100644--- a/crates/liteparse-python/src/cli.rs+++ b/crates/liteparse-python/src/cli.rs@@ -79,6 +79,10 @@ struct ParseCommand {/// page of JSON output. Off by default.#[arg(long)]complexity: bool,++ /// Include rich PDF text metadata in text items and JSON output.+ #[arg(long)]+ include_text_metadata: bool,}#[derive(Args, Debug)]@@ -150,6 +154,10 @@ struct BatchParseCommand {/// page of JSON output. Off by default.#[arg(long)]complexity: bool,++ /// Include rich PDF text metadata in text items and JSON output.+ #[arg(long)]+ include_text_metadata: bool,}/// Parse a `Name: Value` header string into a `(name, value)` pair.@@ -228,6 +236,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {image_mode,extract_links: !cmd.no_links,include_complexity: cmd.complexity,+ include_text_metadata: cmd.include_text_metadata,..Default::default()};if let Some(n) = cmd.num_workers {@@ -240,7 +249,10 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {rt.block_on(lp.parse(&cmd.file))?};let formatted = match lp.config().output_format {- OutputFormat::Json => json::format_json(&result.pages)?,+ OutputFormat::Json => json::format_json_with_text_metadata(+ &result.pages,+ lp.config().include_text_metadata,+ )?,OutputFormat::Text => text::format_text(&result.pages),OutputFormat::Markdown => result.text.clone(),};@@ -328,6 +340,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {ocr_server_url: cmd.ocr_server_url,ocr_server_headers: cmd.ocr_server_headers,include_complexity: cmd.complexity,+ include_text_metadata: cmd.include_text_metadata,..Default::default()};if let Some(n) = cmd.num_workers {@@ -368,9 +381,11 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {Ok(result) => {let fmt_result: Result<String, Box<dyn std::error::Error>> =match lp.config().output_format {- OutputFormat::Json => {- json::format_json(&result.pages).map_err(|e| e.into())- }+ OutputFormat::Json => json::format_json_with_text_metadata(+ &result.pages,+ lp.config().include_text_metadata,+ )+ .map_err(|e| e.into()),OutputFormat::Text => Ok(text::format_text(&result.pages)),OutputFormat::Markdown => Ok(result.text.clone()),};@@ -522,7 +537,7 @@ fn collect_files_inner(#[cfg(test)]mod tests {- use super::batch_output_path;+ use super::*;use std::path::Path;#[test]@@ -538,4 +553,33 @@ mod tests {assert_eq!(out_path, Path::new("out/nested/report.md"));}++ #[test]+ fn include_text_metadata_flag_is_available_for_parse_and_batch() {+ let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--include-text-metadata"])+ .unwrap();+ assert!(matches!(+ cli.command,+ Commands::Parse(ParseCommand {+ include_text_metadata: true,+ ..+ })+ ));++ let cli = Cli::try_parse_from([+ "lit",+ "batch-parse",+ "input",+ "output",+ "--include-text-metadata",+ ])+ .unwrap();+ assert!(matches!(+ cli.command,+ Commands::BatchParse(BatchParseCommand {+ include_text_metadata: true,+ ..+ })+ ));+ }}diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rsindex 56e4cda..8f8a8d5 100644--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@ -508,6 +508,8 @@ struct PyLiteParseConfig {skip_diagonal_text: bool,#[pyo3(get)]include_complexity: bool,+ #[pyo3(get)]+ include_text_metadata: bool,}#[pymethods]@@ -559,6 +561,7 @@ impl PyLiteParseConfig {.map(|c| (c.top, c.right, c.bottom, c.left)),skip_diagonal_text: cfg.skip_diagonal_text,include_complexity: cfg.include_complexity,+ include_text_metadata: cfg.include_text_metadata,}}}@@ -597,6 +600,7 @@ impl LiteParse {ocr_failure_fatal = None,ocr_hedge_delays_ms = None,emit_word_boxes = None,+ include_text_metadata = None,crop_box = None,skip_diagonal_text = None,include_complexity = None,@@ -620,6 +624,7 @@ impl LiteParse {ocr_failure_fatal: Option<bool>,ocr_hedge_delays_ms: Option<Vec<u64>>,emit_word_boxes: Option<bool>,+ include_text_metadata: Option<bool>,crop_box: Option<(f32, f32, f32, f32)>,skip_diagonal_text: Option<bool>,include_complexity: Option<bool>,@@ -687,6 +692,9 @@ impl LiteParse {if let Some(v) = emit_word_boxes {cfg.emit_word_boxes = v;}+ if let Some(v) = include_text_metadata {+ cfg.include_text_metadata = v;+ }if let Some((top, right, bottom, left)) = crop_box {cfg.crop_box = Some(CropBox {top,@@ -921,4 +929,17 @@ mod tests {assert_eq!(round_trip.char_codes, vec![65, 32]);assert!(round_trip.tsg);}++ #[test]+ fn text_metadata_config_defaults_off_and_can_be_enabled() {+ let py = PyLiteParseConfig::from_rust(&LiteParseConfig::default());+ assert!(!py.include_text_metadata);++ let config = LiteParseConfig {+ include_text_metadata: true,+ ..Default::default()+ };+ let py = PyLiteParseConfig::from_rust(&config);+ assert!(py.include_text_metadata);+ }}diff --git a/crates/liteparse/README.md b/crates/liteparse/README.mdindex 0d6f168..8a54672 100644--- a/crates/liteparse/README.md+++ b/crates/liteparse/README.md@@ -57,6 +57,7 @@ let config = LiteParseConfig {dpi: 150.0, // Rendering DPIoutput_format: OutputFormat::Json, // Json | Text | Markdownpreserve_very_small_text: false, // Keep tiny text+ include_text_metadata: false, // Opt in to rich PDF text metadatapassword: None, // Password for protected documentsquiet: false, // Suppress progress output..Default::default()diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex 75ec5c3..0b5c60f 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -60,6 +60,13 @@ pub struct LiteParseConfig {/// `false`, `TextItem.words` is always empty and the per-word tracking is/// skipped entirely (zero allocation).pub emit_word_boxes: bool,+ /// Include rich PDF text metadata on public text items: MCID, glyph width,+ /// font metrics/weight/buggy state, fill/stroke colors, raw character+ /// codes, and generated-trailing-space state. Default `false` to preserve+ /// LiteParse's lightweight response shape. Extraction still computes the+ /// metadata for internal layout; it is stripped only before returning.+ #[serde(default)]+ pub include_text_metadata: bool,/// Restrict output to a sub-region of every page. Each field is the/// fraction of the page to crop away from that side (e.g. `left = 0.5`/// discards the left half). A text item is kept only when it lies@@ -148,6 +155,7 @@ impl Default for LiteParseConfig {ocr_failure_fatal: true,ocr_hedge_delays_ms: Vec::new(),emit_word_boxes: false,+ include_text_metadata: false,crop_box: None,skip_diagonal_text: false,include_complexity: false,@@ -268,6 +276,7 @@ mod tests {assert_eq!(c.output_format, OutputFormat::Json);assert!(!c.preserve_very_small_text);assert!(!c.quiet);+ assert!(!c.include_text_metadata);assert!(c.password.is_none());}diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rsindex e095571..198c026 100644--- a/crates/liteparse/src/main.rs+++ b/crates/liteparse/src/main.rs@@ -126,6 +126,9 @@ struct ParseCommand {/// it runs the extra vector-text detection pass.#[arg(long)]complexity: bool,+ /// Include rich PDF text metadata in text items and JSON output.+ #[arg(long)]+ include_text_metadata: bool,}#[derive(Args, Debug)]@@ -219,6 +222,9 @@ struct BatchParseCommand {/// page of JSON output. Off by default.#[arg(long)]complexity: bool,+ /// Include rich PDF text metadata in text items and JSON output.+ #[arg(long)]+ include_text_metadata: bool,}#[derive(Args, Debug)]@@ -337,6 +343,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {image_mode,extract_links: !cmd.no_links,include_complexity: cmd.complexity,+ include_text_metadata: cmd.include_text_metadata,..Default::default()};if let Some(n) = cmd.num_workers {@@ -350,7 +357,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {lp.parse(&cmd.file).await?};let formatted = match lp.config().output_format {- OutputFormat::Json => json::format_json(&result.pages)?,+ OutputFormat::Json => json::format_json_with_text_metadata(+ &result.pages,+ lp.config().include_text_metadata,+ )?,OutputFormat::Text => text::format_text(&result.pages),OutputFormat::Markdown => result.text.clone(),};@@ -442,6 +452,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {ocr_server_url: cmd.ocr_server_url,ocr_server_headers: cmd.ocr_server_headers,include_complexity: cmd.complexity,+ include_text_metadata: cmd.include_text_metadata,..Default::default()};if let Some(n) = cmd.num_workers {@@ -485,9 +496,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {Ok(result) => {let fmt_result: Result<String, Box<dyn std::error::Error>> =match lp.config().output_format {- OutputFormat::Json => {- json::format_json(&result.pages).map_err(|e| e.into())- }+ OutputFormat::Json => json::format_json_with_text_metadata(+ &result.pages,+ lp.config().include_text_metadata,+ )+ .map_err(|e| e.into()),OutputFormat::Text => Ok(text::format_text(&result.pages)),OutputFormat::Markdown => Ok(result.text.clone()),};@@ -667,7 +680,7 @@ fn collect_files_inner(#[cfg(test)]mod tests {- use super::batch_output_path;+ use super::*;use std::path::Path;#[test]@@ -683,4 +696,33 @@ mod tests {assert_eq!(out_path, Path::new("out/nested/report.md"));}++ #[test]+ fn include_text_metadata_flag_is_available_for_parse_and_batch() {+ let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--include-text-metadata"])+ .unwrap();+ assert!(matches!(+ cli.command,+ Commands::Parse(ParseCommand {+ include_text_metadata: true,+ ..+ })+ ));++ let cli = Cli::try_parse_from([+ "lit",+ "batch-parse",+ "input",+ "output",+ "--include-text-metadata",+ ])+ .unwrap();+ assert!(matches!(+ cli.command,+ Commands::BatchParse(BatchParseCommand {+ include_text_metadata: true,+ ..+ })+ ));+ }}diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rsindex 6ebe7a2..dbba09d 100644--- a/crates/liteparse/src/output/json.rs+++ b/crates/liteparse/src/output/json.rs@@ -9,7 +9,8 @@ pub(crate) struct JsonTextItem {pub y: f32,pub width: f32,pub height: f32,- pub rotation: f32,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub rotation: Option<f32>,#[serde(skip_serializing_if = "Option::is_none")]pub font_name: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]@@ -24,7 +25,8 @@ pub(crate) struct JsonTextItem {pub font_weight: Option<i32>,#[serde(skip_serializing_if = "Option::is_none")]pub text_width: Option<f32>,- pub font_is_buggy: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub font_is_buggy: Option<bool>,#[serde(skip_serializing_if = "Option::is_none")]pub mcid: Option<i32>,#[serde(skip_serializing_if = "Option::is_none")]@@ -56,7 +58,7 @@ pub(crate) struct ParseResultJson {}/// Build structured JSON output from parsed pages.-pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {+pub(crate) fn build_json(pages: &[ParsedPage], include_text_metadata: bool) -> ParseResultJson {ParseResultJson {pages: pages.iter()@@ -74,20 +76,28 @@ pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {y: item.y,width: item.width,height: item.height,- rotation: item.rotation,+ rotation: include_text_metadata.then_some(item.rotation),font_name: item.font_name.clone(),font_size: item.font_size,- font_height: item.font_height,- font_ascent: item.font_ascent,- font_descent: item.font_descent,- font_weight: item.font_weight,- text_width: item.text_width,- font_is_buggy: item.font_is_buggy,- mcid: item.mcid,- fill_color: item.fill_color.clone(),- stroke_color: item.stroke_color.clone(),- char_codes: item.char_codes.clone(),- tsg: item.tsg,+ font_height: include_text_metadata.then_some(item.font_height).flatten(),+ font_ascent: include_text_metadata.then_some(item.font_ascent).flatten(),+ font_descent: include_text_metadata.then_some(item.font_descent).flatten(),+ font_weight: include_text_metadata.then_some(item.font_weight).flatten(),+ text_width: include_text_metadata.then_some(item.text_width).flatten(),+ font_is_buggy: include_text_metadata.then_some(item.font_is_buggy),+ mcid: include_text_metadata.then_some(item.mcid).flatten(),+ fill_color: include_text_metadata+ .then(|| item.fill_color.clone())+ .flatten(),+ stroke_color: include_text_metadata+ .then(|| item.stroke_color.clone())+ .flatten(),+ char_codes: if include_text_metadata {+ item.char_codes.clone()+ } else {+ Vec::new()+ },+ tsg: include_text_metadata && item.tsg,confidence: item.confidence.or(Some(1.0)),}).collect(),@@ -99,7 +109,15 @@ pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {/// Format parsed pages as pretty-printed JSON string.pub fn format_json(pages: &[ParsedPage]) -> Result<String, serde_json::Error> {- let result = build_json(pages);+ format_json_with_text_metadata(pages, false)+}++/// Format parsed pages as JSON, optionally including rich PDF text metadata.+pub fn format_json_with_text_metadata(+ pages: &[ParsedPage],+ include_text_metadata: bool,+) -> Result<String, serde_json::Error> {+ let result = build_json(pages, include_text_metadata);serde_json::to_string_pretty(&result)}@@ -142,7 +160,7 @@ mod tests {#[test]fn test_build_json_native_text_defaults_confidence_to_one() {- let j = build_json(&[page(vec![item("hi", None)])]);+ let j = build_json(&[page(vec![item("hi", None)])], false);assert_eq!(j.pages.len(), 1);assert_eq!(j.pages[0].page, 1);assert_eq!(j.pages[0].text_items[0].confidence, Some(1.0));@@ -151,7 +169,7 @@ mod tests {#[test]fn test_build_json_preserves_ocr_confidence() {- let j = build_json(&[page(vec![item("hi", Some(0.42))])]);+ let j = build_json(&[page(vec![item("hi", Some(0.42))])], false);assert_eq!(j.pages[0].text_items[0].confidence, Some(0.42));}@@ -178,8 +196,10 @@ mod tests {text_item.char_codes = vec![104, 105, 32];text_item.tsg = true;- let value: serde_json::Value =- serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();+ let value: serde_json::Value = serde_json::from_str(+ &format_json_with_text_metadata(&[page(vec![text_item])], true).unwrap(),+ )+ .unwrap();let item = &value["pages"][0]["text_items"][0];assert_eq!(item["font_height"], 11.0);assert_eq!(item["font_ascent"], 8.0);@@ -197,7 +217,27 @@ mod tests {#[test]fn test_build_json_empty() {- let j = build_json(&[]);+ let j = build_json(&[], false);assert!(j.pages.is_empty());}++ #[test]+ fn test_text_metadata_is_omitted_by_default() {+ let mut text_item = item("hi", None);+ text_item.font_height = Some(11.0);+ text_item.font_is_buggy = true;+ text_item.mcid = Some(4);+ text_item.char_codes = vec![104, 105];+ text_item.tsg = true;++ let value: serde_json::Value =+ serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();+ let item = &value["pages"][0]["text_items"][0];+ assert!(item.get("rotation").is_none());+ assert!(item.get("font_height").is_none());+ assert!(item.get("font_is_buggy").is_none());+ assert!(item.get("mcid").is_none());+ assert!(item.get("char_codes").is_none());+ assert!(item.get("tsg").is_none());+ }}diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 3ce72a1..d6cd56a 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -433,6 +433,10 @@ impl LiteParse {.join("\n\n")};+ if !self.config.include_text_metadata {+ strip_text_metadata(&mut parsed_pages);+ }+let total = web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;log(&format!("[liteparse] total: {:.1}ms", total));@@ -471,6 +475,10 @@ impl LiteParse {.join("\n\n")};+ if !self.config.include_text_metadata {+ strip_text_metadata(&mut parsed_pages);+ }+ParseResult {pages: parsed_pages,text: full_text,@@ -539,9 +547,60 @@ impl LiteParse {}}+/// Remove opt-in extraction metadata only after projection/Markdown have had+/// access to it. Longstanding lightweight fields (text, bbox, rotation,+/// font name/size, confidence, links, strike state, and optional word boxes)+/// remain unchanged.+fn strip_text_metadata(pages: &mut [crate::types::ParsedPage]) {+ for item in pages.iter_mut().flat_map(|page| &mut page.text_items) {+ item.font_height = None;+ item.font_ascent = None;+ item.font_descent = None;+ item.font_weight = None;+ item.text_width = None;+ item.font_is_buggy = false;+ item.mcid = None;+ item.fill_color = None;+ item.stroke_color = None;+ item.char_codes.clear();+ item.tsg = false;+ }+}+#[cfg(test)]mod tests {use super::*;+ use crate::types::{Page, TextItem};++ fn page_with_text_metadata() -> Page {+ Page {+ page_number: 1,+ page_width: 100.0,+ page_height: 100.0,+ text_items: vec![TextItem {+ text: "hello".into(),+ width: 20.0,+ height: 10.0,+ font_name: Some("Helvetica".into()),+ font_size: Some(10.0),+ font_height: Some(10.0),+ font_ascent: Some(8.0),+ font_descent: Some(-2.0),+ font_weight: Some(700),+ text_width: Some(19.0),+ font_is_buggy: true,+ mcid: Some(3),+ fill_color: Some("ff112233".into()),+ stroke_color: Some("ff445566".into()),+ char_codes: vec![104, 101, 108, 108, 111],+ tsg: true,+ ..Default::default()+ }],+ graphics: vec![],+ struct_nodes: vec![],+ image_refs: vec![],+ }+ }#[test]#[allow(clippy::field_reassign_with_default)]@@ -553,4 +612,41 @@ mod tests {assert!(!lp.config().ocr_enabled);assert_eq!(lp.config().max_pages, 7);}++ #[test]+ fn parse_result_strips_text_metadata_by_default() {+ let result = LiteParse::new(LiteParseConfig::default())+ .parse_from_pages(vec![page_with_text_metadata()], vec![]);+ let item = &result.pages[0].text_items[0];+ assert_eq!(item.font_name.as_deref(), Some("Helvetica"));+ assert_eq!(item.font_size, Some(10.0));+ assert_eq!(item.font_height, None);+ assert_eq!(item.font_weight, None);+ assert_eq!(item.mcid, None);+ assert!(item.char_codes.is_empty());+ assert!(!item.font_is_buggy);+ assert!(!item.tsg);+ }++ #[test]+ fn parse_result_preserves_text_metadata_when_enabled() {+ let config = LiteParseConfig {+ include_text_metadata: true,+ ..Default::default()+ };+ let result =+ LiteParse::new(config).parse_from_pages(vec![page_with_text_metadata()], vec![]);+ let item = &result.pages[0].text_items[0];+ assert_eq!(item.font_height, Some(10.0));+ assert_eq!(item.font_ascent, Some(8.0));+ assert_eq!(item.font_descent, Some(-2.0));+ assert_eq!(item.font_weight, Some(700));+ assert_eq!(item.text_width, Some(19.0));+ assert!(item.font_is_buggy);+ assert_eq!(item.mcid, Some(3));+ assert_eq!(item.fill_color.as_deref(), Some("ff112233"));+ assert_eq!(item.stroke_color.as_deref(), Some("ff445566"));+ assert_eq!(item.char_codes, vec![104, 101, 108, 108, 111]);+ assert!(item.tsg);+ }}diff --git a/packages/node/README.md b/packages/node/README.mdindex c30a02f..f3898ba 100644--- a/packages/node/README.md+++ b/packages/node/README.md@@ -60,6 +60,7 @@ const parser = new LiteParse({imageMode: 'placeholder', // Markdown image handling: "placeholder" | "off" | "embed"extractLinks: true, // Render [text](url) links in markdown outputpreserveVerySmallText: false, // Keep tiny text+ includeTextMetadata: false, // Opt in to MCID, font metrics, colors, char codes, and tsgpassword: undefined, // Password for protected documentsquiet: false, // Suppress progress outputnumWorkers: 4, // Concurrent OCR workersdiff --git a/packages/node/native.d.ts b/packages/node/native.d.tsindex 5e8dfd7..37d2dd5 100644--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@ -62,6 +62,8 @@ export interface JsLiteParseConfig {* for word-level bbox attribution.*/emitWordBoxes?: boolean+ /** Include rich PDF text metadata on returned text items. Default false. */+ includeTextMetadata?: boolean/*** Restrict output to a page sub-region. Each field is the fraction of the* page cropped from that side; a text item survives only if it liesdiff --git a/packages/node/src/cli.ts b/packages/node/src/cli.tsindex 9be4554..c82e528 100644--- a/packages/node/src/cli.ts+++ b/packages/node/src/cli.ts@@ -78,6 +78,10 @@ program).option("--dpi <dpi>", "Rendering DPI", parseFloat).option("--preserve-small-text", "Keep very small text")+ .option(+ "--include-text-metadata",+ "Include rich PDF text metadata in text items and JSON output",+ ).option("--password <password>", "Password for encrypted documents").option("--config <file>", "JSON config file path").option("-q, --quiet", "Suppress progress output")@@ -113,6 +117,7 @@ programif (opts.targetPages) config.targetPages = opts.targetPages as string;if (opts.dpi) config.dpi = opts.dpi as number;if (opts.preserveSmallText) config.preserveVerySmallText = true;+ if (opts.includeTextMetadata) config.includeTextMetadata = true;if (opts.password) config.password = opts.password as string;if (opts.quiet) config.quiet = true;if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;@@ -313,6 +318,10 @@ program.option("--password <password>", "Password for encrypted documents").option("-q, --quiet", "Suppress progress output").option("--num-workers <n>", "Number of concurrent OCR workers", parseInt)+ .option(+ "--include-text-metadata",+ "Include rich PDF text metadata in text items and JSON output",+ ).action(async (inputDir: string,@@ -334,6 +343,7 @@ programif (opts.password) config.password = opts.password as string;if (opts.quiet) config.quiet = true;if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;+ if (opts.includeTextMetadata) config.includeTextMetadata = true;const parser = new LiteParse(config);const outExt = format === "json" ? ".json" : format === "markdown" ? ".md" : ".txt";diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex 926fa39..c4fbc1d 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -57,6 +57,8 @@ export interface LiteParseConfig {* marshalling), so enable only when doing word-level bbox attribution.*/emitWordBoxes: boolean;+ /** Include rich PDF text metadata on returned text items. Default false. */+ includeTextMetadata?: boolean;/*** Restrict output to a page sub-region. Each field is the fraction of the* page cropped away from that side (top-left origin), so `{ left: 0.5 }`@@ -320,6 +322,7 @@ export class LiteParse {ocrFailureFatal: userConfig.ocrFailureFatal,ocrHedgeDelaysMs: userConfig.ocrHedgeDelaysMs,emitWordBoxes: userConfig.emitWordBoxes,+ includeTextMetadata: userConfig.includeTextMetadata,cropBox: userConfig.cropBox,skipDiagonalText: userConfig.skipDiagonalText,includeComplexity: userConfig.includeComplexity,@@ -348,6 +351,7 @@ export class LiteParse {ocrFailureFatal: resolved.ocrFailureFatal ?? true,ocrHedgeDelaysMs: resolved.ocrHedgeDelaysMs ?? [],emitWordBoxes: resolved.emitWordBoxes ?? false,+ includeTextMetadata: resolved.includeTextMetadata ?? false,cropBox: resolved.cropBox ?? undefined,skipDiagonalText: resolved.skipDiagonalText ?? false,includeComplexity: resolved.includeComplexity ?? false,diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex 29d8b2a..a512109 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -38,6 +38,7 @@ export interface LiteParseNativeConfig {ocrFailureFatal?: boolean;ocrHedgeDelaysMs?: number[];emitWordBoxes?: boolean;+ includeTextMetadata?: boolean;cropBox?: NativeCropBox;skipDiagonalText?: boolean;includeComplexity?: boolean;diff --git a/packages/python/README.md b/packages/python/README.mdindex a624501..0454b2b 100644--- a/packages/python/README.md+++ b/packages/python/README.md@@ -59,6 +59,7 @@ parser = LiteParse(image_mode="placeholder", # Markdown image handling: "placeholder" | "off" | "embed"extract_links=True, # Render [text](url) links in markdown outputpreserve_very_small_text=False, # Keep tiny text+ include_text_metadata=False, # Opt in to MCID, font metrics, colors, char codes, and tsgpassword=None, # Password for protected documentsquiet=False, # Suppress progress outputnum_workers=4, # Concurrent OCR workersdiff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex e5f76fc..1e04abf 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -158,6 +158,7 @@ class LiteParse:ocr_failure_fatal: Optional[bool] = None,ocr_hedge_delays_ms: Optional[List[int]] = None,emit_word_boxes: Optional[bool] = None,+ include_text_metadata: Optional[bool] = None,crop_box: Optional[Tuple[float, float, float, float]] = None,skip_diagonal_text: Optional[bool] = None,include_complexity: Optional[bool] = None,@@ -197,6 +198,9 @@ class LiteParse:(``TextItem.words``). Default False. Word boxes roughly doublethe text-item payload, so enable only for word-level bboxattribution.+ include_text_metadata: Include rich PDF text metadata on returned+ text items (MCID, glyph width, font metrics/weight/buggy state,+ colors, raw character codes, and ``tsg``). Default False.crop_box: Restrict output to a page sub-region, as a``(top, right, bottom, left)`` tuple where each value is thefraction cropped from that side (top-left origin, each in@@ -249,6 +253,8 @@ class LiteParse:kwargs["ocr_hedge_delays_ms"] = ocr_hedge_delays_msif emit_word_boxes is not None:kwargs["emit_word_boxes"] = emit_word_boxes+ if include_text_metadata is not None:+ kwargs["include_text_metadata"] = include_text_metadataif crop_box is not None:kwargs["crop_box"] = crop_boxif skip_diagonal_text is not None:@@ -395,6 +401,7 @@ class LiteParse:crop_box=cfg.crop_box,skip_diagonal_text=cfg.skip_diagonal_text,include_complexity=cfg.include_complexity,+ include_text_metadata=cfg.include_text_metadata,)def __repr__(self) -> str:diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.pyindex 989d8b6..3951bbc 100644--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@ -179,6 +179,7 @@ class LiteParseConfig:crop_box: Optional[Tuple[float, float, float, float]]skip_diagonal_text: boolinclude_complexity: bool+ include_text_metadata: bool = Falseclass ParseError(Exception):