Skip to content

A fast, helpful, and open-source document parser

feat: improve garbled text detection for mixed fonts

Logan Markewich539f6560e06e
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index a484d5a..2949175 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -1,3 +1,4 @@
+use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use crate::error::LiteParseError;
@@ -628,13 +629,19 @@ pub(crate) async fn ocr_and_merge_rendered(
// unmappable Type3 text) so OCR can replace them. Without this,
// garbled-but-spatially-present native text suppresses every OCR
// result that overlaps it via the overlap check below, leaving the
- // output stuck with unreadable text. We apply both per-item and
+ // output stuck with unreadable text. We apply per-item, per-font, and
// per-page checks: short garbled labels ("GDWH", "XVG") can't be
- // flagged alone, but their host page can.
- if page_is_garbled(page) {
- page.text_items.clear();
- } else {
- page.text_items.retain(|item| !is_unusable_native(item));
+ // flagged alone, but their font group or host page can.
+ match garbled_scope(page) {
+ GarbledScope::None => page.text_items.retain(|item| !is_unusable_native(item)),
+ GarbledScope::WholePage => page.text_items.clear(),
+ // Only the corrupt font's items go; text in the page's healthy
+ // fonts is still the best source for its part of the page, and
+ // keeping it lets the overlap check below suppress redundant OCR
+ // there while OCR fills in what was dropped.
+ GarbledScope::Fonts(fonts) => page
+ .text_items
+ .retain(|item| !fonts.contains(&item.font_name) && !is_unusable_native(item)),
}
// Only check overlap against native (already-extracted) PDF text. Comparing
@@ -849,28 +856,88 @@ fn count_letters_and_vowels(text: &str) -> (usize, usize) {
(letters, vowels)
}
+/// Minimum ASCII letters before a group of text is worth judging for garbling.
+/// Below this the vowel ratio is too noisy — an acronym run or a short label
+/// can sit near zero without being corrupt.
+const GARBLE_MIN_LETTERS: usize = 30;
+
+/// A font group must hold at least this share of the page's letters before its
+/// own vowel ratio can flag the page. Corruption travels with a font, so a
+/// broken map affects a substantial block of the page; requiring a real share
+/// keeps a stray caption or ticker run in an odd font from flagging a healthy
+/// page.
+const GARBLE_MIN_FONT_SHARE: f32 = 0.15;
+
+/// True when a body of text's vowel ratio is below the natural-language floor.
+/// Real Latin-script vowel ratios sit ~30–45% across English, Portuguese,
+/// Spanish, French, etc. A ratio under 20% is well outside any natural-language
+/// range and signals substitution-style corruption. (A simple +3 Caesar shift
+/// still leaves some U/Y letters from the original O/Y mapping, so a 10% bound
+/// is too tight to catch this in practice.)
+fn ratio_is_garbled(letters: usize, vowels: usize) -> bool {
+ letters >= GARBLE_MIN_LETTERS && vowels * 5 < letters
+}
+
+/// Which of a page's native text items are untrustworthy because they decode
+/// to garbage. Corruption is a property of a *font's* ToUnicode map, not of the
+/// page, so this is resolved per font group as well as page-wide.
+pub(crate) enum GarbledScope {
+ /// Nothing looks garbled.
+ None,
+ /// The page's text as a whole decodes to garbage — every native item is
+ /// suspect, including items too short to judge on their own.
+ WholePage,
+ /// Specific font groups are garbled while the rest of the page extracts
+ /// fine. Holds the `font_name` of each flagged group.
+ Fonts(HashSet<Option<String>>),
+}
+
/// Page-level garbled check: even when individual items are too short to judge
-/// in isolation (e.g. "GDWH", "FXUUHQFB XVG"), a page whose aggregate vowel
-/// ratio collapses to single digits is almost certainly substitution-encoded.
-/// Used to drop all native items on the page before OCR merge, so short
-/// garbled labels don't suppress overlapping OCR results.
-fn page_is_garbled(page: &Page) -> bool {
+/// in isolation (e.g. "GDWH", "FXUUHQFB XVG"), text whose aggregate vowel ratio
+/// collapses is almost certainly substitution-encoded. Used to drop untrusted
+/// native items before OCR merge, so short garbled labels don't suppress
+/// overlapping OCR results.
+///
+/// Judged per font group, not only page-wide: a page mixing a healthy standard
+/// font with an embedded font whose ToUnicode map is wrong averages out to a
+/// natural-looking ratio, so a page-wide sum alone reports the page as clean
+/// while half its text is corrupt. The page-wide check is still applied first,
+/// since it catches pages whose fonts are individually too small to judge (and
+/// pages whose items carry no font name at all).
+fn garbled_scope(page: &Page) -> GarbledScope {
let mut total_letters = 0usize;
let mut total_vowels = 0usize;
+ // Keyed by font name, so text sharing a ToUnicode map is judged together.
+ let mut per_font: HashMap<Option<&str>, (usize, usize)> = HashMap::new();
for it in &page.text_items {
let (l, v) = count_letters_and_vowels(&it.text);
total_letters += l;
total_vowels += v;
+ let entry = per_font.entry(it.font_name.as_deref()).or_insert((0, 0));
+ entry.0 += l;
+ entry.1 += v;
}
- if total_letters < 30 {
- return false;
+
+ if ratio_is_garbled(total_letters, total_vowels) {
+ return GarbledScope::WholePage;
+ }
+
+ let min_share = (total_letters as f32 * GARBLE_MIN_FONT_SHARE).ceil() as usize;
+ let fonts: HashSet<Option<String>> = per_font
+ .into_iter()
+ .filter(|&(_, (letters, vowels))| letters >= min_share && ratio_is_garbled(letters, vowels))
+ .map(|(font, _)| font.map(str::to_string))
+ .collect();
+
+ if fonts.is_empty() {
+ GarbledScope::None
+ } else {
+ GarbledScope::Fonts(fonts)
}
- // Real Latin-script vowel ratios sit ~30–45% across English, Portuguese,
- // Spanish, French, etc. A page-wide ratio under 20% is well outside any
- // natural-language range and signals substitution-style corruption. (A
- // simple +3 Caesar shift still leaves some U/Y letters from the original
- // O/Y mapping, so a 10% bound is too tight to catch this in practice.)
- total_vowels * 5 < total_letters
+}
+
+fn page_is_garbled(page: &Page) -> bool {
+ !matches!(garbled_scope(page), GarbledScope::None)
}
/// Recover a discrete CCW rotation in degrees from a 4-point OCR polygon.
@@ -1233,6 +1300,114 @@ mod tests {
}
}
+ /// A page whose items carry per-item `(font_name, text)` pairs. Geometry is
+ /// irrelevant to the garble checks, so every item shares one bbox.
+ fn make_font_page(items: &[(&str, &str)]) -> Page {
+ let mut page = make_blank_page(1);
+ page.text_items = items
+ .iter()
+ .map(|(font, text)| TextItem {
+ text: (*text).into(),
+ font_name: Some((*font).to_string()),
+ width: 10.0,
+ height: 10.0,
+ ..Default::default()
+ })
+ .collect();
+ page
+ }
+
+ /// Caesar-shifted table text: the +11 letter / +3 digit map from the
+ /// mixed-font fixture in issue #413, applied to the visible row text.
+ const GARBLED_ROWS: &[&str] = &[
+ "Tepx Nzop Opdnctaetzy Bflyetej Cpgtph Olep",
+ "KI-334 Cpawlnpxpye Qtwepc Lddpxmwj 44 5359-30-34",
+ "KI-335 Azcelmwp Dpydzc Wzfdtyr 45 5359-30-35",
+ "KI-336 Nlwtmcletzy Mclnvpe 46 5359-30-36",
+ ];
+
+ const HEALTHY_PARAGRAPH: &[&str] = &[
+ "This fictional report is a public parser-routing fixture.",
+ "The upper section uses a standard font and extracts normally.",
+ "All names, identifiers, quantities, and dates are fabricated.",
+ "A PDF viewer still renders both sections as readable text.",
+ ];
+
+ #[test]
+ fn test_garbled_scope_clean_page() {
+ let items: Vec<(&str, &str)> = HEALTHY_PARAGRAPH
+ .iter()
+ .map(|t| ("Helvetica", *t))
+ .collect();
+ let page = make_font_page(&items);
+ assert!(matches!(garbled_scope(&page), GarbledScope::None));
+ assert!(!page_is_garbled(&page));
+ }
+
+ #[test]
+ fn test_garbled_scope_whole_page() {
+ let items: Vec<(&str, &str)> = GARBLED_ROWS.iter().map(|t| ("BadFont", *t)).collect();
+ let page = make_font_page(&items);
+ assert!(matches!(garbled_scope(&page), GarbledScope::WholePage));
+ }
+
+ /// Issue #413: healthy text in one font must not mask a corrupt ToUnicode
+ /// map in another. Page-wide the vowel ratio lands ~0.28 (natural-looking),
+ /// while the corrupt font group alone sits ~0.19.
+ #[test]
+ fn test_garbled_scope_mixed_fonts_flags_only_bad_font() {
+ let mut items: Vec<(&str, &str)> = HEALTHY_PARAGRAPH
+ .iter()
+ .map(|t| ("Helvetica", *t))
+ .collect();
+ items.extend(GARBLED_ROWS.iter().map(|t| ("BitstreamVeraSans-Roman", *t)));
+ let page = make_font_page(&items);
+
+ // The page-wide sum alone does not trip: this is the regression.
+ let (letters, vowels) = page.text_items.iter().fold((0, 0), |(l, v), it| {
+ let (il, iv) = count_letters_and_vowels(&it.text);
+ (l + il, v + iv)
+ });
+ assert!(!ratio_is_garbled(letters, vowels));
+
+ match garbled_scope(&page) {
+ GarbledScope::Fonts(fonts) => {
+ assert_eq!(fonts.len(), 1);
+ assert!(fonts.contains(&Some("BitstreamVeraSans-Roman".to_string())));
+ }
+ _ => panic!("expected only the corrupt font group to be flagged"),
+ }
+ assert!(page_is_garbled(&page));
+ }
+
+ /// A small run of low-vowel text (acronyms, tickers, part numbers) in its
+ /// own font must not flag an otherwise healthy page.
+ #[test]
+ fn test_garbled_scope_ignores_small_low_vowel_font() {
+ let mut items: Vec<(&str, &str)> = HEALTHY_PARAGRAPH
+ .iter()
+ .map(|t| ("Helvetica", *t))
+ .collect();
+ items.push(("TickerFont", "NYSE NASDAQ FTSE DAX SPX"));
+ let page = make_font_page(&items);
+ assert!(matches!(garbled_scope(&page), GarbledScope::None));
+ }
+
+ /// Items with no font name still fall back to the page-wide verdict.
+ #[test]
+ fn test_garbled_scope_without_font_names() {
+ let mut page = make_blank_page(1);
+ page.text_items = GARBLED_ROWS
+ .iter()
+ .map(|text| TextItem {
+ text: (*text).into(),
+ font_name: None,
+ ..Default::default()
+ })
+ .collect();
+ assert!(matches!(garbled_scope(&page), GarbledScope::WholePage));
+ }
+
fn make_rendered(idx: usize) -> RenderedPage {
RenderedPage {
idx,