A fast, helpful, and open-source document parser
improve text extraction
Logan Markewich42c8d72f98fa
diff --git a/packages/rust/crates/liteparse/src/extract.rs b/packages/rust/crates/liteparse/src/extract.rsindex 29ed8b1..6efe1bb 100644--- a/packages/rust/crates/liteparse/src/extract.rs+++ b/packages/rust/crates/liteparse/src/extract.rs@@ -45,93 +45,247 @@ pub fn extract(pdf_path: &str, page_num: Option<u32>) -> Result<(), Box<dyn std:Ok(())}-/// Use pdfium's text rect API (FPDFText_CountRects/GetRect/GetBoundedText) to extract-/// text segments. Pdfium auto-merges characters sharing the same baseline and font into-/// rectangular regions — this is the same approach pdfium-render's segments() used.+/// Character-level text extraction.+///+/// Instead of using PDFium's rect API (which splits text at every font attribute+/// change), we iterate through individual characters and group them by spatial+/// proximity. This keeps words like "A-MEM" together even when internal characters+/// have different font sizes (e.g. small-caps), and keeps punctuation attached to+/// adjacent text (e.g. citation commas/semicolons).+///+/// Segments break at:+/// - Line changes (large vertical shift)+/// - Column breaks (large horizontal gap)+/// - Explicit newline charactersfn extract_page_text_items(text_page: &TextPage) -> Result<Vec<TextItem>, Box<dyn std::error::Error>> {let char_count = text_page.char_count();if char_count <= 0 {return Ok(Vec::new());}- // Count rects for all characters — this caches the results for subsequent rect() calls- let rect_count = text_page.count_rects(0, char_count);- let mut text_items = Vec::new();+ // Hard limit: gaps larger than this always cause a split (column breaks).+ const MAX_INLINE_GAP: f64 = 15.0;- for rect_idx in 0..rect_count {- let rect = match text_page.rect(rect_idx) {- Some(r) => r,- None => continue,+ let mut items: Vec<TextItem> = Vec::new();+ let mut seg = SegmentBuilder::new();++ for i in 0..char_count {+ let Some(ch) = text_page.char_at(i) else { continue };+ let unicode = ch.unicode();+ let is_generated = ch.is_generated();++ // Skip null / invalid sentinels+ if unicode == 0 || unicode == 0xFFFE || unicode == 0xFFFF {+ continue;+ }++ // Map to a Rust char, with special-case replacements+ let c = match unicode {+ 0x02 => '-', // STX → hyphen (common in some PDF encodings)+ _ => match char::from_u32(unicode) {+ Some(c) => c,+ None => continue,+ },};- let seg_text = text_page- .bounded_text(rect.left, rect.top, rect.right, rect.bottom)- .replace('\x02', "-");+ // Newlines: flush the current segment+ if c == '\n' || c == '\r' {+ seg.flush(&mut items);+ continue;+ }- if seg_text.trim().is_empty() {+ // Spaces: mark that we're in a pending-space state.+ // The space will be committed (or cause a split) when the next+ // visible character arrives and we can measure the actual gap.+ if c == ' ' {+ seg.mark_pending_space();continue;}- let x = rect.left as f32;- let y = rect.bottom as f32;- let width = (rect.right - rect.left) as f32;- let height = (rect.top - rect.bottom) as f32;-- // Find the first character in this rect to get font info and rotation- let (font_name, font_size, rotation) = first_char_info_in_rect(text_page, &rect, height);-- text_items.push(TextItem {- text: seg_text,- x,- y,- width,- height,- rotation,- font_name,- font_size,- });+ // Skip non-space generated characters (synthetic glyphs)+ if is_generated {+ continue;+ }++ // Get the character's bounding box (PDF coordinates: bottom-left origin)+ let Some(cb) = ch.char_box() else { continue };++ if seg.has_content {+ // Check whether this character belongs to the current segment.+ // Use actual bounding box overlap with a small tolerance (2pt)+ // to handle floating point imprecision while still keeping+ // separate lines apart (~3pt gap between 10pt lines).+ // Superscripts/subscripts naturally overlap the baseline bbox.+ let y_tolerance = 2.0;+ let y_overlap = cb.bottom < seg.top + y_tolerance+ && cb.top > seg.bottom - y_tolerance;++ let gap = cb.left - seg.last_char_right;++ if !y_overlap || gap >= MAX_INLINE_GAP {+ // Different line or huge gap — always split.+ seg.flush(&mut items);+ seg.start(c, &cb, &ch);+ } else if seg.pending_space {+ // There was a space before this character. Decide whether+ // to keep it in the same segment or split.+ // Split when the gap is large relative to the avg char width —+ // this separates table columns while keeping body text words+ // together.+ let avg_cw = seg.avg_char_width();+ if gap > avg_cw * 1.6 {+ seg.flush(&mut items);+ seg.start(c, &cb, &ch);+ } else {+ seg.commit_pending_space();+ seg.push_char(c, &cb);+ }+ } else {+ seg.push_char(c, &cb);+ }+ } else {+ seg.start(c, &cb, &ch);+ }}- Ok(text_items)+ seg.flush(&mut items);+ Ok(items)}-/// Get font name, font size, and rotation angle (degrees) from the first character-/// whose center falls within the given rect.-fn first_char_info_in_rect(- text_page: &TextPage,- rect: &pdfium::TextRect,- fallback_height: f32,-) -> (Option<String>, Option<f32>, f32) {- let char_count = text_page.char_count();- for i in 0..char_count {- let ch = match text_page.char_at(i) {- Some(c) => c,- None => continue,+/// Accumulates characters into a single TextItem segment.+struct SegmentBuilder {+ text: String,+ // Bounding box in PDF coordinates (bottom-left origin)+ left: f64,+ right: f64,+ bottom: f64,+ top: f64,+ // Right edge of the last non-space character (for gap calculation)+ last_char_right: f64,+ // Count of non-space characters (for avg width calculation)+ char_count: usize,+ // Font metadata (captured from the first character)+ font_name: Option<String>,+ font_size: f32,+ rotation_deg: f32,+ has_content: bool,+ // True when we've seen a space but haven't yet decided whether to+ // commit it (keep in segment) or split on it.+ pending_space: bool,+}++impl SegmentBuilder {+ fn new() -> Self {+ Self {+ text: String::new(),+ left: f64::MAX,+ right: f64::MIN,+ bottom: f64::MAX,+ top: f64::MIN,+ last_char_right: f64::MIN,+ char_count: 0,+ font_name: None,+ font_size: 0.0,+ rotation_deg: 0.0,+ has_content: false,+ pending_space: false,+ }+ }++ /// Average width of non-space characters in the current segment.+ fn avg_char_width(&self) -> f64 {+ if self.char_count == 0 {+ return 5.0; // sensible default+ }+ (self.right - self.left) / self.char_count as f64+ }++ /// Start a new segment with the given character.+ fn start(&mut self, c: char, cb: &pdfium::CharBox, ch: &pdfium::TextChar) {+ self.text.clear();+ self.text.push(c);+ self.left = cb.left;+ self.right = cb.right;+ self.bottom = cb.bottom;+ self.top = cb.top;+ self.last_char_right = cb.right;+ self.char_count = 1;+ self.has_content = true;+ self.pending_space = false;++ self.font_name = ch.font_name();+ let fs = ch.font_size() as f32;+ self.font_size = if fs > 0.0 { fs } else { (cb.top - cb.bottom) as f32 };++ let angle_rad = ch.angle();+ self.rotation_deg = if angle_rad >= 0.0 {+ let mut deg = angle_rad.to_degrees();+ if deg < 0.0 { deg += 360.0; }+ deg+ } else {+ 0.0};+ }- if let Some(cb) = ch.char_box() {- let cx = (cb.left + cb.right) / 2.0;- let cy = (cb.bottom + cb.top) / 2.0;- if cx >= rect.left && cx <= rect.right && cy >= rect.bottom && cy <= rect.top {- let fname = ch.font_name();- let fsize = ch.font_size() as f32;- let fsize = if fsize > 0.0 { fsize } else { fallback_height };-- // angle() returns radians; convert to degrees- let angle_rad = ch.angle();- let mut angle_deg = if angle_rad >= 0.0 {- angle_rad.to_degrees()- } else {- 0.0- };- if angle_deg < 0.0 {- angle_deg += 360.0;- }+ /// Add a visible character to the current segment.+ fn push_char(&mut self, c: char, cb: &pdfium::CharBox) {+ self.text.push(c);+ self.left = self.left.min(cb.left);+ self.right = self.right.max(cb.right);+ self.bottom = self.bottom.min(cb.bottom);+ self.top = self.top.max(cb.top);+ self.last_char_right = cb.right;+ self.char_count += 1;+ }- return (fname, Some(fsize), angle_deg);- }+ /// Record that a space was seen; the actual decision to include it+ /// or to split is deferred until the next visible character.+ fn mark_pending_space(&mut self) {+ if self.has_content {+ self.pending_space = true;+ }+ }++ /// Commit a pending space into the segment text.+ fn commit_pending_space(&mut self) {+ if self.pending_space {+ self.text.push(' ');+ self.pending_space = false;}}- (None, Some(fallback_height), 0.0)+ /// Flush the current segment into the items list and reset.+ fn flush(&mut self, items: &mut Vec<TextItem>) {+ if !self.has_content {+ return;+ }++ let trimmed = self.text.trim();+ if !trimmed.is_empty() {+ let height = (self.top - self.bottom) as f32;+ items.push(TextItem {+ text: trimmed.to_string(),+ x: self.left as f32,+ y: self.bottom as f32,+ width: (self.right - self.left) as f32,+ height,+ rotation: self.rotation_deg,+ font_name: self.font_name.clone(),+ font_size: Some(if self.font_size > 0.0 { self.font_size } else { height }),+ });+ }++ self.text.clear();+ self.left = f64::MAX;+ self.right = f64::MIN;+ self.bottom = f64::MAX;+ self.top = f64::MIN;+ self.last_char_right = f64::MIN;+ self.char_count = 0;+ self.font_name = None;+ self.font_size = 0.0;+ self.rotation_deg = 0.0;+ self.has_content = false;+ self.pending_space = false;+ }}