A fast, helpful, and open-source document parser
improve font-recovery semantics vs ocr
Logan Markewichff85e38e45b4
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex c4ae420..fada7b9 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -594,6 +594,11 @@ fn extract_page_text_items(} else {glyph_decoder.decode(&ch, unicode)};+ // A glyph the decoder recovered (glyph-name / reverse-cmap / outline-hash+ // resolver) carries correct text even though PDFium still reports a+ // /ToUnicode map error for its raw char code. Don't count these toward the+ // item's unmapped-char tally.+ let recovered = decoded.is_some();// Skip null / invalid sentinels (unless the glyph name recovered them)if decoded.is_none() && (unicode == 0 || unicode == 0xFFFE || unicode == 0xFFFF) {@@ -770,13 +775,13 @@ fn extract_page_text_items(}if !y_overlap || line_changed || gap >= MAX_INLINE_GAP || dot_leader_break {seg.flush(&mut items);- seg.start(c, &vp_loose, &vp_strict, &ch, page_rotation);+ seg.start(c, &vp_loose, &vp_strict, &ch, recovered, page_rotation);seg.append_ligature_tail(ligature_tail);} else if seg.pending_space {let avg_cw = seg.avg_char_width();if gap > avg_cw * 2.2 {seg.flush(&mut items);- seg.start(c, &vp_loose, &vp_strict, &ch, page_rotation);+ seg.start(c, &vp_loose, &vp_strict, &ch, recovered, page_rotation);seg.append_ligature_tail(ligature_tail);} else {// Genuine inline space PDFium emitted: sample its size@@ -800,7 +805,7 @@ fn extract_page_text_items(}}seg.commit_pending_space();- seg.push_char(c, &vp_loose, &vp_strict, &ch);+ seg.push_char(c, &vp_loose, &vp_strict, &ch, recovered);seg.append_ligature_tail(ligature_tail);}} else {@@ -843,11 +848,11 @@ fn extract_page_text_items(seg.text.push(' ');seg.break_word();}- seg.push_char(c, &vp_loose, &vp_strict, &ch);+ seg.push_char(c, &vp_loose, &vp_strict, &ch, recovered);seg.append_ligature_tail(ligature_tail);}} else {- seg.start(c, &vp_loose, &vp_strict, &ch, page_rotation);+ seg.start(c, &vp_loose, &vp_strict, &ch, recovered, page_rotation);seg.append_ligature_tail(ligature_tail);}}@@ -1444,6 +1449,13 @@ fn detect_garbage_unicode_fonts(.collect()}+/// Whether a just-decoded glyph should count toward its item's unmapped-char+/// tally (which sets `TextItem::has_unicode_map_error` by majority vote in+/// `flush`).+fn counts_as_unmapped(recovered: bool, raw_map_error: bool) -> bool {+ !recovered && raw_map_error+}+/// Accumulates characters into a single TextItem segment.struct SegmentBuilder {text: String,@@ -1600,6 +1612,7 @@ impl SegmentBuilder {vp_loose: &RectF,vp_strict: &RectF,ch: &pdfium::TextChar,+ recovered: bool,page_rotation: i32,) {self.text.clear();@@ -1612,7 +1625,11 @@ impl SegmentBuilder {self.last_char_loose_right = vp_loose.right;self.last_char_bottom = vp_strict.bottom;self.char_count = 1;- self.unmapped_char_count = if ch.has_unicode_map_error() { 1 } else { 0 };+ self.unmapped_char_count = if counts_as_unmapped(recovered, ch.has_unicode_map_error()) {+ 1+ } else {+ 0+ };self.has_content = true;self.pending_space = false;self.words.clear();@@ -1700,7 +1717,14 @@ impl SegmentBuilder {}/// Add a visible character to the current segment.- fn push_char(&mut self, c: char, vp_loose: &RectF, vp_strict: &RectF, ch: &pdfium::TextChar) {+ fn push_char(+ &mut self,+ c: char,+ vp_loose: &RectF,+ vp_strict: &RectF,+ ch: &pdfium::TextChar,+ recovered: bool,+ ) {self.text.push(c);self.add_word_char(c, vp_loose);self.vp_left = self.vp_left.min(vp_loose.left);@@ -1711,7 +1735,7 @@ impl SegmentBuilder {self.last_char_loose_right = vp_loose.right;self.last_char_bottom = vp_strict.bottom;self.char_count += 1;- if ch.has_unicode_map_error() {+ if counts_as_unmapped(recovered, ch.has_unicode_map_error()) {self.unmapped_char_count += 1;}@@ -1826,6 +1850,29 @@ mod tests {use super::*;use std::f32::consts::PI;+ // A glyph PDFium flags with a raw /ToUnicode map error normally counts+ // toward the item's unmapped tally...+ #[test]+ fn unrecovered_map_error_counts_as_unmapped() {+ assert!(counts_as_unmapped(false, true));+ }++ // ...but once the decoder recovers it (glyph-name / reverse-cmap /+ // outline-hash resolver) the text is correct, so it must NOT count — this+ // is the fix that stops the OCR merge from discarding fully-recovered+ // buggy-font items as "unusable native".+ #[test]+ fn recovered_map_error_does_not_count_as_unmapped() {+ assert!(!counts_as_unmapped(true, true));+ }++ // A cleanly-mapped glyph never counts, recovered or not.+ #[test]+ fn cleanly_mapped_glyph_never_counts_as_unmapped() {+ assert!(!counts_as_unmapped(false, false));+ assert!(!counts_as_unmapped(true, false));+ }+fn strike_item() -> TextItem {TextItem {text: "word".to_string(),