A fast, helpful, and open-source document parser
fix: is_garbled checks for broken fonts
Logan Markewich0f500f878bbd
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex a8d0457..ad7124a 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -1454,8 +1454,20 @@ fn garbled_scope(page: &Page) -> GarbledScope {}}+/// Page-level garbled signal: substitution-cipher text (see [`garbled_scope`])+/// or a substantial share of text whose Unicode mapping failed outright.fn page_is_garbled(page: &Page) -> bool {- !matches!(garbled_scope(page), GarbledScope::None)+ !matches!(garbled_scope(page), GarbledScope::None) || page_has_unmapped_text(page)+}++/// True when text with a failed Unicode mapping is both a real amount (not a+/// stray Type3 symbol glyph) and a real share of the page's native text.+fn page_has_unmapped_text(page: &Page) -> bool {+ let (unmapped, total) = page.text_items.iter().fold((0usize, 0usize), |(u, t), it| {+ let n = it.text.chars().filter(|c| !c.is_whitespace()).count();+ (u + if it.has_unicode_map_error { n } else { 0 }, t + n)+ });+ unmapped >= GARBLE_MIN_LETTERS && unmapped as f32 >= total as f32 * GARBLE_MIN_FONT_SHARE}/// Recover a discrete CCW rotation in degrees from a 4-point OCR polygon.@@ -2200,6 +2212,51 @@ mod tests {assert!(page_is_garbled(&page));}+ /// A page of unmappable text (PUA char-code fallback) has no ASCII letters+ /// for the vowel ratio to judge, and every item is excluded from+ /// `text_length` — it must still report as garbled.+ #[test]+ fn test_page_is_garbled_unmapped_text() {+ let mut page = make_font_page(&[+ (+ "T3Font",+ "\u{E001}\u{E002}\u{E003}\u{E004}\u{E005}\u{E006}\u{E007}\u{E008}",+ ),+ (+ "T3Font",+ "\u{E011}\u{E012}\u{E013}\u{E014}\u{E015}\u{E016}\u{E017}\u{E018}",+ ),+ (+ "T3Font",+ "\u{E021}\u{E022}\u{E023}\u{E024}\u{E025}\u{E026}\u{E027}\u{E028}",+ ),+ (+ "T3Font",+ "\u{E031}\u{E032}\u{E033}\u{E034}\u{E035}\u{E036}\u{E037}\u{E038}",+ ),+ ]);+ for it in &mut page.text_items {+ it.has_unicode_map_error = true;+ }+ assert!(matches!(garbled_scope(&page), GarbledScope::None));+ assert_eq!(native_text_length(&page), 0);+ assert!(page_is_garbled(&page));+ }++ /// A stray unmappable symbol glyph (e.g. a Type3 checkmark) on a healthy+ /// page must not flag it.+ #[test]+ fn test_page_is_garbled_ignores_stray_unmapped_glyph() {+ let mut items: Vec<(&str, &str)> = HEALTHY_PARAGRAPH+ .iter()+ .map(|t| ("Helvetica", *t))+ .collect();+ items.push(("T3Font", "\u{E001}"));+ let mut page = make_font_page(&items);+ page.text_items.last_mut().unwrap().has_unicode_map_error = true;+ assert!(!page_is_garbled(&page));+ }+/// A small run of low-vowel text (acronyms, tickers, part numbers) in its/// own font must not flag an otherwise healthy page.#[test]