A fast, helpful, and open-source document parser
Expose PDF text extraction metadata
Pierre-Loic doulcetfa3881535792
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rsindex 69ad85a..c31e71d 100644--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@ -251,6 +251,21 @@ pub struct JsTextItem {pub height: f64,pub font_name: Option<String>,pub font_size: Option<f64>,+ pub font_height: Option<f64>,+ pub font_ascent: Option<f64>,+ pub font_descent: Option<f64>,+ pub font_weight: Option<i32>,+ pub text_width: Option<f64>,+ pub font_is_buggy: Option<bool>,+ pub mcid: Option<i32>,+ /// Fill color as an eight-character ARGB hex string.+ pub fill_color: Option<String>,+ /// Stroke color as an eight-character ARGB hex string.+ pub stroke_color: Option<String>,+ /// Raw PDF content-stream character codes for the source glyphs.+ pub char_codes: Option<Vec<u32>>,+ /// True when the trailing source space was synthesized by PDFium.+ pub tsg: Option<bool>,pub confidence: Option<f64>,/// Rotation in degrees (viewport space). Defaults to 0 when omitted.pub rotation: Option<f64>,@@ -270,6 +285,17 @@ impl JsTextItem {rotation: self.rotation.unwrap_or(0.0) as f32,font_name: self.font_name.clone(),font_size: self.font_size.map(|v| v as f32),+ font_height: self.font_height.map(|v| v as f32),+ font_ascent: self.font_ascent.map(|v| v as f32),+ font_descent: self.font_descent.map(|v| v as f32),+ font_weight: self.font_weight,+ text_width: self.text_width.map(|v| v as f32),+ font_is_buggy: self.font_is_buggy.unwrap_or(false),+ mcid: self.mcid,+ fill_color: self.fill_color.clone(),+ stroke_color: self.stroke_color.clone(),+ char_codes: self.char_codes.clone().unwrap_or_default(),+ tsg: self.tsg.unwrap_or(false),confidence: self.confidence.map(|v| v as f32),..Default::default()}@@ -285,6 +311,17 @@ impl JsTextItem {rotation: Some(item.rotation as f64),font_name: item.font_name.clone(),font_size: item.font_size.map(|v| v as f64),+ font_height: item.font_height.map(|v| v as f64),+ font_ascent: item.font_ascent.map(|v| v as f64),+ font_descent: item.font_descent.map(|v| v as f64),+ font_weight: item.font_weight,+ text_width: item.text_width.map(|v| v as f64),+ font_is_buggy: Some(item.font_is_buggy),+ mcid: item.mcid,+ fill_color: item.fill_color.clone(),+ stroke_color: item.stroke_color.clone(),+ char_codes: Some(item.char_codes.clone()),+ tsg: Some(item.tsg),confidence: item.confidence.map(|v| v as f64).or(Some(1.0)),words: item.words.iter().map(JsWordBox::from_rust).collect(),}@@ -562,3 +599,44 @@ impl JsParseResult {}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn text_metadata_round_trips_through_napi_type() {+ let item = TextItem {+ text: "A".into(),+ font_height: Some(12.0),+ font_ascent: Some(9.0),+ font_descent: Some(-3.0),+ font_weight: Some(700),+ text_width: Some(8.0),+ font_is_buggy: true,+ mcid: Some(2),+ fill_color: Some("ff112233".into()),+ stroke_color: Some("ff445566".into()),+ char_codes: vec![65, 32],+ tsg: true,+ ..Default::default()+ };++ let js = JsTextItem::from_rust(&item);+ assert_eq!(js.char_codes, Some(vec![65, 32]));+ assert_eq!(js.tsg, Some(true));+ assert_eq!(js.fill_color.as_deref(), Some("ff112233"));++ let round_trip = js.to_rust();+ assert_eq!(round_trip.font_height, Some(12.0));+ assert_eq!(round_trip.font_ascent, Some(9.0));+ assert_eq!(round_trip.font_descent, Some(-3.0));+ assert_eq!(round_trip.font_weight, Some(700));+ assert_eq!(round_trip.text_width, Some(8.0));+ assert!(round_trip.font_is_buggy);+ assert_eq!(round_trip.mcid, Some(2));+ assert_eq!(round_trip.stroke_color.as_deref(), Some("ff445566"));+ assert_eq!(round_trip.char_codes, vec![65, 32]);+ assert!(round_trip.tsg);+ }+}diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rsindex 4be62da..56e4cda 100644--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@ -67,6 +67,32 @@ struct PyTextItem {#[pyo3(get)]font_size: Option<f64>,#[pyo3(get)]+ font_height: Option<f64>,+ #[pyo3(get)]+ font_ascent: Option<f64>,+ #[pyo3(get)]+ font_descent: Option<f64>,+ #[pyo3(get)]+ font_weight: Option<i32>,+ #[pyo3(get)]+ text_width: Option<f64>,+ #[pyo3(get)]+ font_is_buggy: bool,+ #[pyo3(get)]+ mcid: Option<i32>,+ /// Fill color as an eight-character ARGB hex string.+ #[pyo3(get)]+ fill_color: Option<String>,+ /// Stroke color as an eight-character ARGB hex string.+ #[pyo3(get)]+ stroke_color: Option<String>,+ /// Raw PDF content-stream character codes for the source glyphs.+ #[pyo3(get)]+ char_codes: Vec<u32>,+ /// True when the trailing source space was synthesized by PDFium.+ #[pyo3(get)]+ tsg: bool,+ #[pyo3(get)]confidence: Option<f64>,/// Rotation in degrees (viewport space). Defaults to 0.#[pyo3(get)]@@ -88,6 +114,32 @@ impl PyTextItem {}impl PyTextItem {+ fn to_rust(&self) -> liteparse::types::TextItem {+ liteparse::types::TextItem {+ text: self.text.clone(),+ x: self.x as f32,+ y: self.y as f32,+ width: self.width as f32,+ height: self.height as f32,+ rotation: self.rotation as f32,+ font_name: self.font_name.clone(),+ font_size: self.font_size.map(|v| v as f32),+ font_height: self.font_height.map(|v| v as f32),+ font_ascent: self.font_ascent.map(|v| v as f32),+ font_descent: self.font_descent.map(|v| v as f32),+ font_weight: self.font_weight,+ text_width: self.text_width.map(|v| v as f32),+ font_is_buggy: self.font_is_buggy,+ mcid: self.mcid,+ fill_color: self.fill_color.clone(),+ stroke_color: self.stroke_color.clone(),+ char_codes: self.char_codes.clone(),+ tsg: self.tsg,+ confidence: self.confidence.map(|v| v as f32),+ ..Default::default()+ }+ }+fn from_rust(item: liteparse::types::TextItem) -> Self {Self {text: item.text,@@ -97,6 +149,17 @@ impl PyTextItem {height: item.height as f64,font_name: item.font_name,font_size: item.font_size.map(|v| v as f64),+ font_height: item.font_height.map(|v| v as f64),+ font_ascent: item.font_ascent.map(|v| v as f64),+ font_descent: item.font_descent.map(|v| v as f64),+ font_weight: item.font_weight,+ text_width: item.text_width.map(|v| v as f64),+ font_is_buggy: item.font_is_buggy,+ mcid: item.mcid,+ fill_color: item.fill_color,+ stroke_color: item.stroke_color,+ char_codes: item.char_codes,+ tsg: item.tsg,confidence: item.confidence.map(|v| v as f64).or(Some(1.0)),rotation: item.rotation as f64,words: item.words.into_iter().map(PyWordBox::from_rust).collect(),@@ -818,3 +881,44 @@ fn _liteparse(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_function(wrap_pyfunction!(search_items, m)?)?;Ok(())}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn text_metadata_round_trips_through_python_type() {+ let item = liteparse::types::TextItem {+ text: "A".into(),+ font_height: Some(12.0),+ font_ascent: Some(9.0),+ font_descent: Some(-3.0),+ font_weight: Some(700),+ text_width: Some(8.0),+ font_is_buggy: true,+ mcid: Some(2),+ fill_color: Some("ff112233".into()),+ stroke_color: Some("ff445566".into()),+ char_codes: vec![65, 32],+ tsg: true,+ ..Default::default()+ };++ let py = PyTextItem::from_rust(item);+ assert_eq!(py.char_codes, vec![65, 32]);+ assert!(py.tsg);+ assert_eq!(py.fill_color.as_deref(), Some("ff112233"));++ let round_trip = py.to_rust();+ assert_eq!(round_trip.font_height, Some(12.0));+ assert_eq!(round_trip.font_ascent, Some(9.0));+ assert_eq!(round_trip.font_descent, Some(-3.0));+ assert_eq!(round_trip.font_weight, Some(700));+ assert_eq!(round_trip.text_width, Some(8.0));+ assert!(round_trip.font_is_buggy);+ assert_eq!(round_trip.mcid, Some(2));+ assert_eq!(round_trip.stroke_color.as_deref(), Some("ff445566"));+ assert_eq!(round_trip.char_codes, vec![65, 32]);+ assert!(round_trip.tsg);+ }+}diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex 9eb1523..6f25c3a 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -646,9 +646,16 @@ fn extract_page_text_items(continue;}- // Spaces: mark that we're in a pending-space state+ // Whitespace: mark that we're in a pending-space state while retaining+ // the source character code and PDFium-generated distinction.if c.is_whitespace() {- seg.mark_pending_space();+ seg.mark_pending_space(is_generated, ch.char_code());+ // Keep PDFium-generated gaps as item boundaries so the emitted+ // item can retain the same trailing-space-generated distinction+ // as the source extractor. The visible text remains trimmed.+ if is_generated {+ seg.flush(&mut items);+ }continue;}@@ -1491,6 +1498,9 @@ struct SegmentBuilder {mcid: Option<i32>,fill_color: Option<String>,stroke_color: Option<String>,+ char_codes: Vec<u32>,+ pending_space_char_codes: Vec<u32>,+ pending_space_generated: bool,has_content: bool,pending_space: bool,// Per-word sub-boxes, finalized at each inter-word space break. The@@ -1535,6 +1545,9 @@ impl SegmentBuilder {mcid: None,fill_color: None,stroke_color: None,+ char_codes: Vec::new(),+ pending_space_char_codes: Vec::new(),+ pending_space_generated: false,has_content: false,pending_space: false,words: Vec::new(),@@ -1632,10 +1645,14 @@ impl SegmentBuilder {};self.has_content = true;self.pending_space = false;+ self.pending_space_char_codes.clear();+ self.pending_space_generated = false;self.words.clear();self.word_has = false;self.add_word_char(c, vp_loose);self.text_width = 0.0;+ self.char_codes.clear();+ self.char_codes.push(ch.char_code());self.font_is_buggy = false;self.font_is_embedded = false;self.font = None;@@ -1735,6 +1752,7 @@ impl SegmentBuilder {self.last_char_loose_right = vp_loose.right;self.last_char_bottom = vp_strict.bottom;self.char_count += 1;+ self.char_codes.push(ch.char_code());if counts_as_unmapped(recovered, ch.has_unicode_map_error()) {self.unmapped_char_count += 1;}@@ -1777,9 +1795,11 @@ impl SegmentBuilder {}/// Record that a space was seen.- fn mark_pending_space(&mut self) {+ fn mark_pending_space(&mut self, is_generated: bool, char_code: u32) {if self.has_content {self.pending_space = true;+ self.pending_space_generated = is_generated;+ self.pending_space_char_codes.push(char_code);}}@@ -1788,7 +1808,9 @@ impl SegmentBuilder {if self.pending_space {self.break_word();self.text.push(' ');+ self.char_codes.append(&mut self.pending_space_char_codes);self.pending_space = false;+ self.pending_space_generated = false;}}@@ -1799,6 +1821,7 @@ impl SegmentBuilder {}self.break_word();+ self.char_codes.append(&mut self.pending_space_char_codes);let trimmed = self.text.trim();if !trimmed.is_empty() {let width = self.vp_right - self.vp_left;@@ -1834,6 +1857,8 @@ impl SegmentBuilder {mcid: self.mcid,fill_color: self.fill_color.clone(),stroke_color: self.stroke_color.clone(),+ char_codes: std::mem::take(&mut self.char_codes),+ tsg: self.pending_space_generated,confidence: None,link: None,strike: false,@@ -1873,6 +1898,43 @@ mod tests {assert!(!counts_as_unmapped(true, false));}+ fn segment_with_one_char() -> SegmentBuilder {+ let mut segment = SegmentBuilder::new(false);+ segment.text = "A".into();+ segment.vp_left = 1.0;+ segment.vp_right = 7.0;+ segment.vp_top = 2.0;+ segment.vp_bottom = 12.0;+ segment.char_count = 1;+ segment.has_content = true;+ segment.char_codes = vec![65];+ segment+ }++ #[test]+ fn generated_trailing_space_is_exposed_without_changing_trimmed_text() {+ let mut segment = segment_with_one_char();+ segment.mark_pending_space(true, 32);+ let mut items = Vec::new();+ segment.flush(&mut items);++ assert_eq!(items.len(), 1);+ assert_eq!(items[0].text, "A");+ assert_eq!(items[0].char_codes, vec![65, 32]);+ assert!(items[0].tsg);+ }++ #[test]+ fn real_trailing_space_is_not_marked_as_generated() {+ let mut segment = segment_with_one_char();+ segment.mark_pending_space(false, 32);+ let mut items = Vec::new();+ segment.flush(&mut items);++ assert_eq!(items[0].char_codes, vec![65, 32]);+ assert!(!items[0].tsg);+ }+fn strike_item() -> TextItem {TextItem {text: "word".to_string(),diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rsindex 980de9d..6ebe7a2 100644--- a/crates/liteparse/src/output/json.rs+++ b/crates/liteparse/src/output/json.rs@@ -9,11 +9,33 @@ pub(crate) struct JsonTextItem {pub y: f32,pub width: f32,pub height: f32,+ pub rotation: f32,#[serde(skip_serializing_if = "Option::is_none")]pub font_name: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]pub font_size: Option<f32>,#[serde(skip_serializing_if = "Option::is_none")]+ pub font_height: Option<f32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub font_ascent: Option<f32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub font_descent: Option<f32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub font_weight: Option<i32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub text_width: Option<f32>,+ pub font_is_buggy: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub mcid: Option<i32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub fill_color: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub stroke_color: Option<String>,+ #[serde(skip_serializing_if = "Vec::is_empty")]+ pub char_codes: Vec<u32>,+ #[serde(skip_serializing_if = "std::ops::Not::not")]+ pub tsg: bool,+ #[serde(skip_serializing_if = "Option::is_none")]pub confidence: Option<f32>,}@@ -52,8 +74,20 @@ pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson {y: item.y,width: item.width,height: item.height,+ rotation: item.rotation,font_name: item.font_name.clone(),font_size: item.font_size,+ font_height: item.font_height,+ font_ascent: item.font_ascent,+ font_descent: item.font_descent,+ font_weight: item.font_weight,+ text_width: item.text_width,+ font_is_buggy: item.font_is_buggy,+ mcid: item.mcid,+ fill_color: item.fill_color.clone(),+ stroke_color: item.stroke_color.clone(),+ char_codes: item.char_codes.clone(),+ tsg: item.tsg,confidence: item.confidence.or(Some(1.0)),}).collect(),@@ -129,6 +163,38 @@ mod tests {assert!(s.contains("\"page\": 1"));}+ #[test]+ fn test_build_json_preserves_text_metadata() {+ let mut text_item = item("hi", None);+ text_item.font_height = Some(11.0);+ text_item.font_ascent = Some(8.0);+ text_item.font_descent = Some(-2.0);+ text_item.font_weight = Some(700);+ text_item.text_width = Some(9.5);+ text_item.font_is_buggy = true;+ text_item.mcid = Some(4);+ text_item.fill_color = Some("ff112233".into());+ text_item.stroke_color = Some("ff445566".into());+ text_item.char_codes = vec![104, 105, 32];+ text_item.tsg = true;++ let value: serde_json::Value =+ serde_json::from_str(&format_json(&[page(vec![text_item])]).unwrap()).unwrap();+ let item = &value["pages"][0]["text_items"][0];+ assert_eq!(item["font_height"], 11.0);+ assert_eq!(item["font_ascent"], 8.0);+ assert_eq!(item["font_descent"], -2.0);+ assert_eq!(item["font_weight"], 700);+ assert_eq!(item["text_width"], 9.5);+ assert_eq!(item["font_is_buggy"], true);+ assert_eq!(item["mcid"], 4);+ assert_eq!(item["fill_color"], "ff112233");+ assert_eq!(item["stroke_color"], "ff445566");+ assert_eq!(item["char_codes"], serde_json::json!([104, 105, 32]));+ assert_eq!(item["tsg"], true);+ assert_eq!(item["rotation"], 0.0);+ }+#[test]fn test_build_json_empty() {let j = build_json(&[]);diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rsindex ea008c9..81bea49 100644--- a/crates/liteparse/src/types.rs+++ b/crates/liteparse/src/types.rs@@ -55,6 +55,15 @@ pub struct TextItem {/// Stroke color as ARGB hex string.#[serde(skip_serializing_if = "Option::is_none")]pub stroke_color: Option<String>,+ /// Raw character codes from the PDF content stream. These correspond to+ /// source glyphs rather than Unicode scalar values, so ligature expansion+ /// can produce more text characters than entries in this array.+ #[serde(skip_serializing_if = "Vec::is_empty")]+ pub char_codes: Vec<u32>,+ /// Whether the trailing source space was synthesized by PDFium rather than+ /// represented by a real space glyph in the PDF content stream.+ #[serde(skip_serializing_if = "std::ops::Not::not")]+ pub tsg: bool,/// OCR confidence score (0.0–1.0). None for native PDF text.#[serde(skip_serializing_if = "Option::is_none")]pub confidence: Option<f32>,diff --git a/packages/node/native.d.ts b/packages/node/native.d.tsindex c9d5f69..5e8dfd7 100644--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@ -106,6 +106,21 @@ export interface JsTextItem {height: numberfontName?: stringfontSize?: number+ fontHeight?: number+ fontAscent?: number+ fontDescent?: number+ fontWeight?: number+ textWidth?: number+ fontIsBuggy?: boolean+ mcid?: number+ /** Fill color as an eight-character ARGB hex string. */+ fillColor?: string+ /** Stroke color as an eight-character ARGB hex string. */+ strokeColor?: string+ /** Raw PDF content-stream character codes for the source glyphs. */+ charCodes?: Array<number>+ /** True when the trailing source space was synthesized by PDFium. */+ tsg?: booleanconfidence?: number/** Rotation in degrees (viewport space). Defaults to 0 when omitted. */rotation?: numberdiff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex a93fbde..926fa39 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -111,6 +111,24 @@ export interface TextItem {height: number;fontName?: string;fontSize?: number;+ /** Font size after applying the text matrix's vertical scale. */+ fontHeight?: number;+ fontAscent?: number;+ fontDescent?: number;+ fontWeight?: number;+ /** Sum of source glyph widths in points. */+ textWidth?: number;+ fontIsBuggy?: boolean;+ /** Marked-content ID from the PDF structure tree. */+ mcid?: number;+ /** Fill color as an eight-character ARGB hex string. */+ fillColor?: string;+ /** Stroke color as an eight-character ARGB hex string. */+ strokeColor?: string;+ /** Raw PDF content-stream character codes for the source glyphs. */+ charCodes?: number[];+ /** True when the trailing source space was synthesized by PDFium. */+ tsg?: boolean;confidence?: number;/** Rotation in degrees (viewport space). Defaults to 0 when omitted. */rotation?: number;@@ -466,6 +484,17 @@ function toTextItem(item: NativeTextItem): TextItem {height: item.height,fontName: item.fontName,fontSize: item.fontSize,+ fontHeight: item.fontHeight,+ fontAscent: item.fontAscent,+ fontDescent: item.fontDescent,+ fontWeight: item.fontWeight,+ textWidth: item.textWidth,+ fontIsBuggy: item.fontIsBuggy,+ mcid: item.mcid,+ fillColor: item.fillColor,+ strokeColor: item.strokeColor,+ charCodes: item.charCodes,+ tsg: item.tsg,confidence: item.confidence,rotation: item.rotation,words: item.words,diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex f9a0967..29d8b2a 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -66,6 +66,17 @@ export interface NativeTextItem {height: number;fontName?: string;fontSize?: number;+ fontHeight?: number;+ fontAscent?: number;+ fontDescent?: number;+ fontWeight?: number;+ textWidth?: number;+ fontIsBuggy?: boolean;+ mcid?: number;+ fillColor?: string;+ strokeColor?: string;+ charCodes?: number[];+ tsg?: boolean;confidence?: number;rotation?: number;words?: NativeWordBox[];diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex f41cd26..e5f76fc 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -66,6 +66,17 @@ def _convert_native_result(native_result: Any) -> ParseResult:height=item.height,font_name=item.font_name,font_size=item.font_size,+ font_height=getattr(item, "font_height", None),+ font_ascent=getattr(item, "font_ascent", None),+ font_descent=getattr(item, "font_descent", None),+ font_weight=getattr(item, "font_weight", None),+ text_width=getattr(item, "text_width", None),+ font_is_buggy=getattr(item, "font_is_buggy", False),+ mcid=getattr(item, "mcid", None),+ fill_color=getattr(item, "fill_color", None),+ stroke_color=getattr(item, "stroke_color", None),+ char_codes=list(getattr(item, "char_codes", [])),+ tsg=getattr(item, "tsg", False),confidence=item.confidence,rotation=getattr(item, "rotation", 0.0),words=[diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.pyindex b8d30d1..989d8b6 100644--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@ -27,6 +27,21 @@ class TextItem:height: floatfont_name: Optional[str] = Nonefont_size: Optional[float] = None+ font_height: Optional[float] = None+ font_ascent: Optional[float] = None+ font_descent: Optional[float] = None+ font_weight: Optional[int] = None+ text_width: Optional[float] = None+ font_is_buggy: bool = False+ mcid: Optional[int] = None+ #: Fill color as an eight-character ARGB hex string.+ fill_color: Optional[str] = None+ #: Stroke color as an eight-character ARGB hex string.+ stroke_color: Optional[str] = None+ #: Raw PDF content-stream character codes for the source glyphs.+ char_codes: List[int] = field(default_factory=list)+ #: True when the trailing source space was synthesized by PDFium.+ tsg: bool = Falseconfidence: Optional[float] = Nonerotation: float = 0.0#: Per-word sub-boxes. Empty unless the parser was configured with