A fast, helpful, and open-source document parser
Text extraction: re-OCR fonts with a few misread glyphs; Arabic lam-alef and mark order; trust sparse raster-free text layers
Logan Markewich70f17f0d1666
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex 49a5441..2b06483 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -1558,7 +1558,7 @@ fn extract_page_text_items(seg.flush(&mut items);continue;}- let unicode = cv.unicode();+ let unicode = lam_alef_order(text_page, i, char_count, cv.unicode(), cv.char_code());let is_generated = cv.is_generated();// Skip invisible text (render mode 3) only when the page also has visible text.@@ -2707,6 +2707,49 @@ impl<'a> GlyphDecoder<'a> {}}+const ARABIC_LAM: u32 = 0x0644;++/// How far (points) past a word's rightmost letter a combining mark may sit and still belong+/// to the letter just before the word (see `SegmentBuilder::place_rtl_mark`).+const RTL_MARK_REACH: f32 = 3.0;++/// Combining marks of the Hebrew and Arabic blocks (points, cantillation, harakat).+fn is_rtl_combining_mark(c: char) -> bool {+ matches!(c as u32, 0x0591..=0x05BD | 0x05BF | 0x05C1..=0x05C2 | 0x05C4..=0x05C5 | 0x05C7+ | 0x0610..=0x061A | 0x064B..=0x065F | 0x0670 | 0x06D6..=0x06DC | 0x06DF..=0x06E4+ | 0x06E7..=0x06E8 | 0x06EA..=0x06ED)+}++/// Alef and its hamza / madda forms, the second letter of a lam-alef ligature.+fn is_alef(u: u32) -> bool {+ matches!(u, 0x0622 | 0x0623 | 0x0625 | 0x0627)+}++/// Logical order for a lam-alef ligature glyph. One glyph draws lam + alef ("لا", "لأ"), and+/// many producers write its ToUnicode entry in visual order, alef first; PDFium then hands back+/// two chars sharing the glyph's char code, alef then lam ("األثاث" for "الأثاث"). Separate+/// alef and lam glyphs (the article "ال") carry different codes and are left alone.+fn lam_alef_order(text_page: &TextPage, i: i32, char_count: i32, unicode: u32, code: u32) -> u32 {+ let shares_glyph = |j: i32| {+ (0..char_count)+ .contains(&j)+ .then(|| {+ let c = text_page.char_at_unchecked(j);+ (c.char_code() == code).then(|| c.unicode())+ })+ .flatten()+ };+ if is_alef(unicode) && shares_glyph(i + 1) == Some(ARABIC_LAM) {+ return ARABIC_LAM;+ }+ if unicode == ARABIC_LAM+ && let Some(prev) = shares_glyph(i - 1).filter(|&u| is_alef(u))+ {+ return prev;+ }+ unicode+}+/// Control/PUA/sentinel codepoints that signal a garbage /ToUnicode mapping.fn is_suspicious_unicode(unicode: u32) -> bool {matches!(unicode, 0 | 0xFFFE | 0xFFFF) || unicode < 0x20 || (0xE000..=0xF8FF).contains(&unicode)@@ -3240,6 +3283,9 @@ struct SegmentBuilder {emit_words: bool,// When false, source-code/trailing-space metadata is not accumulated.extract_text_metadata: bool,+ // Right-to-left letters of the segment as (byte end in `text`, loose left, loose right), so+ // a combining mark drawn over an earlier letter can be placed after it (see `push_char`).+ rtl_spans: Vec<(usize, f32, f32)>,}impl SegmentBuilder {@@ -3287,6 +3333,7 @@ impl SegmentBuilder {word_has: false,emit_words,extract_text_metadata,+ rtl_spans: Vec::new(),}}@@ -3404,6 +3451,11 @@ impl SegmentBuilder {) {self.text.clear();self.text.push(c);+ self.rtl_spans.clear();+ if crate::bidi::is_rtl_char(c) {+ self.rtl_spans+ .push((self.text.len(), vp_loose.left, vp_loose.right));+ }self.vp_left = vp_loose.left;self.vp_right = vp_loose.right;self.vp_top = vp_loose.top;@@ -3527,7 +3579,21 @@ impl SegmentBuilder {cv: &CharView<'_, '_>,recovered: bool,) {+ if self.place_rtl_mark(c, vp_loose) {+ // the segment's box still covers the mark; the trailing-edge fields stay on the+ // last letter, so the next char's gap is measured from it+ self.vp_left = self.vp_left.min(vp_loose.left);+ self.vp_right = self.vp_right.max(vp_loose.right);+ self.vp_top = self.vp_top.min(vp_loose.top);+ self.vp_bottom = self.vp_bottom.max(vp_loose.bottom);+ self.char_count += 1;+ return;+ }self.text.push(c);+ if crate::bidi::is_rtl_char(c) {+ self.rtl_spans+ .push((self.text.len(), vp_loose.left, vp_loose.right));+ }self.add_word_char(c, vp_loose);self.vp_left = self.vp_left.min(vp_loose.left);self.vp_right = self.vp_right.max(vp_loose.right);@@ -3568,6 +3634,62 @@ impl SegmentBuilder {}}+ /// A right-to-left combining mark (Arabic harakat, Hebrew points) that PDFium hands over+ /// after the rest of its word ("يقصدُ" for "يُقصد"): insert it after the letter of the+ /// current word that its box sits over. Returns false when the mark already follows its+ /// letter, or no letter of the word lies under it; the caller then appends it as usual.+ fn place_rtl_mark(&mut self, c: char, vp_loose: &RectF) -> bool {+ if self.dir_rtl != Some(true) || !is_rtl_combining_mark(c) || self.rtl_spans.len() < 2 {+ return false;+ }+ let word_start = self.text.rfind(char::is_whitespace).map_or(0, |i| i + 1);+ let cx = (vp_loose.left + vp_loose.right) / 2.0;+ let Some(k) = self+ .rtl_spans+ .iter()+ .position(|&(end, l, r)| end > word_start && cx >= l - 0.5 && cx <= r + 0.5)+ else {+ // Just past the word's first (rightmost) letter: the mark sits on the letter before+ // the word, which PDFium hands over separately and line assembly puts back in front.+ let word_right = self+ .rtl_spans+ .iter()+ .filter(|s| s.0 > word_start)+ .map(|s| s.2)+ .fold(f32::MIN, f32::max);+ if word_right > f32::MIN && cx > word_right && cx - word_right < RTL_MARK_REACH {+ self.text.insert(word_start, c);+ if self.emit_words && self.word_has {+ self.cur_word.insert(0, c);+ }+ let n = c.len_utf8();+ for s in self.rtl_spans.iter_mut().filter(|s| s.0 > word_start) {+ s.0 += n;+ }+ return true;+ }+ return false;+ };+ if k + 1 == self.rtl_spans.len() {+ return false;+ }+ let at = self.rtl_spans[k].0;+ self.text.insert(at, c);+ if self.emit_words && self.word_has {+ let in_word = self.cur_word.len() + at;+ if let Some(w) = in_word.checked_sub(self.text.len() - c.len_utf8())+ && self.cur_word.is_char_boundary(w)+ {+ self.cur_word.insert(w, c);+ }+ }+ let n = c.len_utf8();+ for s in &mut self.rtl_spans[k..] {+ s.0 += n;+ }+ true+ }+/// Append extra characters to the segment text (for ligature expansion)./// Does not update bounding boxes or char count.fn append_ligature_tail(&mut self, tail: &str) {diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex 1766688..c143a8d 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -20,6 +20,9 @@ use serde::{Deserialize, Serialize};/// text); measured on a 121-page financial report, the trigger fires on ~3/// pages at this threshold.const UNCOVERED_VECTOR_AREA_THRESHOLD: f32 = 400.0;+/// Native characters at which a sparse, raster-free page's text layer is taken as its content+/// (OCR then runs only for vector-drawn text, see `calculate_page_complexity`).+const SPARSE_TEXT_TRUSTED_LEN: usize = 500;/// Minimum side length (pt) for a raster image object to count toward image/// coverage; smaller objects (rule lines, bullets, icons) are ignored.@@ -578,8 +581,11 @@ pub(crate) fn calculate_page_complexity(if page_obj.has_annotation_text() {reasons.push(ComplexityReason::AnnotationText);}- } else if sparse_text {- // There is real text, but it's too thin to be the whole page.+ } else if sparse_text && (full_page_image || text_length < SPARSE_TEXT_TRUSTED_LEN) {+ // There is real text, but it's too thin to be the whole page. A page that already+ // carries plenty of native text and no raster at all has nothing else for OCR to read+ // but text drawn as vector outlines, which the vector check below measures; OCR'ing it+ // anyway only re-reads the native text (every result then dropped as overlapping).reasons.push(ComplexityReason::SparseText);}if has_images {@@ -1269,6 +1275,9 @@ pub fn merge_ocr_results(} else {garbled_scope(page)};+ // Some fonts dropped, the rest kept: OCR fills lines the dropped fonts held, and a kept+ // glyph inside such a line (a bullet, a drop cap) must not veto it (see below).+ let partial = matches!(scope, GarbledScope::Fonts(_));match scope {GarbledScope::None => page.text_items.retain(|item| !is_unusable_native(item)),GarbledScope::WholePage => page.text_items.clear(),@@ -1286,6 +1295,8 @@ pub fn merge_ocr_results(// OCR lines whose bounding boxes touched within tolerance to suppress each// other, dropping every second line on scanned pages.let native_count = page.text_items.len();+ // Kept native items an OCR line superseded on a partial page (see `partial`).+ let mut superseded = vec![false; native_count];let debug_ocr = std::env::var_os("LITEPARSE_DEBUG_OCR").is_some();// A skewed scan: when all of the page's text is OCR, rotate the items upright about the// page centre. Axis-aligned boxes of tilted lines overlap the lines above and below,@@ -1387,14 +1398,33 @@ pub fn merge_ocr_results(}continue;}- let overlaps = overlaps_existing_text(- &page.text_items[..native_count],- ocr_x,- ocr_y,- ocr_w,- ocr_h,- 2.0,- );+ // Native items inside a line that, on a partial page, supersedes them once added.+ let mut inside = Vec::new();+ let overlaps = if partial {+ // On a partial page the line is the kept text's only when kept native text+ // covers most of it; otherwise the native items inside it give way to it.+ let natives = &page.text_items[..native_count];+ let covered = native_width_coverage(natives, ocr_x, ocr_y, ocr_w, ocr_h);+ if covered < PARTIAL_NATIVE_COVERAGE {+ inside = (0..native_count)+ .filter(|&i| {+ let it = &natives[i];+ let (cx, cy) = (it.x + it.width / 2.0, it.y + it.height / 2.0);+ cx >= ocr_x && cx <= ocr_x + ocr_w && cy >= ocr_y && cy <= ocr_y + ocr_h+ })+ .collect();+ }+ covered >= PARTIAL_NATIVE_COVERAGE+ } else {+ overlaps_existing_text(+ &page.text_items[..native_count],+ ocr_x,+ ocr_y,+ ocr_w,+ ocr_h,+ 2.0,+ )+ };if debug_ocr {eprintln!("[ocr-merge] {} conf={:.2} x={:.1} y={:.1} w={:.1} h={:.1} {:?}",@@ -1460,6 +1490,17 @@ pub fn merge_ocr_results(words,..Default::default()});+ for i in inside {+ superseded[i] = true;+ }+ }+ if superseded.iter().any(|&s| s) {+ let mut i = 0;+ page.text_items.retain(|_| {+ let keep = i >= native_count || !superseded[i];+ i += 1;+ keep+ });}if !has_native_text {upright_rotated_scan(page);@@ -1780,7 +1821,20 @@ fn garbled_scope(page: &Page) -> GarbledScope {let (mut total_ctrl, mut total_chars) = (0usize, 0usize);let mut per_font_tok: HashMap<Option<&str>, (usize, usize)> = HashMap::new();let (mut total_judged, mut total_malformed) = (0usize, 0usize);+ // Misread glyphs (see `misread_glyphs`), per base font: intruding char -> its distinct words+ // and the distinct tails it opens.+ let mut per_base_misread: HashMap<Option<&str>, MisreadByChar> = HashMap::new();for it in &page.text_items {+ if !it.math && !is_symbol_font(it.font_name.as_deref()) {+ let words = per_base_misread+ .entry(it.font_name.as_deref().map(base_font_name))+ .or_default();+ for m in misread_glyphs(&it.text) {+ let e = words.entry(m.ch).or_default();+ e.0.insert(m.word);+ e.1.insert(m.tail);+ }+ }// math items carry the TeX pass's own LaTeX (and its fonts' control-position glyphs)if !it.math && !is_symbol_font(it.font_name.as_deref()) {let (c, n) = control_chars(&it.text);@@ -1864,6 +1918,26 @@ fn garbled_scope(page: &Page) -> GarbledScope {.filter(|&(_, (j, m))| tokens_are_garbled(j, m)).map(|(font, _)| font.map(str::to_string)),);+ // A misread glyph condemns every subset of its base font on the page: a producer that+ // wrote one subset's map wrong wrote the others the same way, and the body subset's few+ // misreads ("maGor") are too sparse to judge alone.+ let misread_bases: HashSet<Option<&str>> = per_base_misread+ .into_iter()+ .filter(|(_, words)| {+ words+ .values()+ .any(|(w, tails)| w.len() >= MISREAD_MIN_WORDS && tails.len() >= MISREAD_MIN_TAILS)+ })+ .map(|(base, _)| base)+ .collect();+ if !misread_bases.is_empty() {+ fonts.extend(+ page.text_items+ .iter()+ .filter(|it| misread_bases.contains(&it.font_name.as_deref().map(base_font_name)))+ .map(|it| it.font_name.clone()),+ );+ }if fonts.is_empty() {GarbledScope::None@@ -2087,6 +2161,94 @@ fn symbols_are_garbled(letters: usize, intrusions: usize) -> bool {intrusions >= SYMBOL_MIN && intrusions as f32 >= letters as f32 * SYMBOL_MIN_RATE}+/// Fewest distinct words one misread character must turn up in for its font to count as+/// garbled (see [`misread_glyphs`]). Brand and camel-case names ("arXiv", "iPad", "McRae")+/// repeat one word; a wrong map entry hits every word with that glyph. On ParseBench v2 text,+/// tables and opendataloader (1,210 pages) the broken fonts reach 3-50 words, clean text 2.+const MISREAD_MIN_WORDS: usize = 3;++/// ...and, for a misread capital, at least this many distinct tails it opens: compound names+/// end in one repeated part ("HistoAge", "PhenoAge", "GrimAge"), a wrong map entry lands+/// mid-word in different places ("executiMe", "achieMed").+const MISREAD_MIN_TAILS: usize = 2;++/// Per misread char: the distinct words it sits in and the distinct tails it opens.+type MisreadByChar = HashMap<char, (HashSet<String>, HashSet<String>)>;++/// One misread character (see [`misread_glyphs`]): the char, the word it sits in (letters,+/// lowercased), and the text from it to the word's end (for a symbol, the whole word).+#[derive(Debug)]+struct Misread {+ ch: char,+ word: String,+ tail: String,+}++/// A font's name without its subset tag ("AAAAAB+Futura" -> "Futura").+fn base_font_name(name: &str) -> &str {+ name.split_once('+').map_or(name, |(_, b)| b)+}++/// Characters a font map put where a letter belongs, each with the word (its letters,+/// lowercased) it sits in. A map that is wrong for only a few glyphs leaves text that reads+/// almost right, so none of the font-wide rates above catch it:+/// - `! % *` between two Latin, Greek or Cyrillic letters ("va!í", "kter*ch": č, š, ý lost);+/// - a capital inside a lowercase word whose lowercase tail (under 3 letters) ends the word+/// ("maGor", "OecutiMe", "horiQon"). A capital in second place ("iPad"), words with digits+/// ("dBm", "o9A") and identifiers are skipped.+///+/// `"` is no intruder: a quotation printed without its leading space ("expression"put") is real.+fn misread_glyphs(text: &str) -> Vec<Misread> {+ let mut out = Vec::new();+ for tok in text.split_whitespace() {+ let t = tok.trim_matches(|c: char| {+ matches!(+ c,+ '.' | ',' | ';' | ':' | '(' | ')' | '[' | ']' | '{' | '}' | '\'' | '"' | '!' | '?'+ ) || "«»“”‘’".contains(c)+ });+ if t.starts_with("http") || t.chars().any(|c| matches!(c, '_' | '@' | '\\' | '$' | '/')) {+ continue;+ }+ let cs: Vec<char> = t.chars().collect();+ if cs.len() < 3 {+ continue;+ }+ let word = || -> String {+ cs.iter()+ .filter(|c| c.is_alphabetic())+ .flat_map(|c| c.to_lowercase())+ .collect()+ };+ let alphabetic = |c: char| c.is_alphabetic() && matches!(script_of(c), 0 | 0x370 | 0x400);+ for i in 1..cs.len() - 1 {+ if matches!(cs[i], '!' | '%' | '*') && alphabetic(cs[i - 1]) && alphabetic(cs[i + 1]) {+ out.push(Misread {+ ch: cs[i],+ word: word(),+ tail: word(),+ });+ }+ }+ if cs.iter().any(|c| c.is_ascii_digit()) {+ continue;+ }+ for i in 2..cs.len() - 1 {+ if cs[i - 1].is_lowercase() && cs[i].is_uppercase() && cs[i + 1].is_lowercase() {+ let tail = cs[i + 1..].iter().take_while(|c| c.is_lowercase()).count();+ if i + 1 + tail == cs.len() && tail < 3 {+ out.push(Misread {+ ch: cs[i],+ word: word(),+ tail: cs[i..].iter().collect(),+ });+ }+ }+ }+ }+ out+}+fn intrusions_are_garbled(script: usize, intrusions: usize) -> bool {intrusions >= INTRUSION_MIN && intrusions as f32 >= script as f32 * INTRUSION_MIN_RATE}@@ -2188,6 +2350,36 @@ fn is_rule_artifact(text: &str, w: f32, h: f32) -> bool {false}+/// Least share of an OCR line's width that kept native text must cover, on a page whose+/// garbled fonts were dropped, for the line to count as already present (see `partial`).+const PARTIAL_NATIVE_COVERAGE: f32 = 0.5;++/// Share of the box's width covered by native items on its line (vertical overlap of at+/// least half the shorter height), their horizontal extents merged.+fn native_width_coverage(items: &[TextItem], x: f32, y: f32, w: f32, h: f32) -> f32 {+ if w <= 0.0 {+ return 1.0;+ }+ let mut spans: Vec<(f32, f32)> = items+ .iter()+ .filter(|it| {+ let v = (y + h).min(it.y + it.height) - y.max(it.y);+ v > 0.5 * h.min(it.height)+ })+ .map(|it| (it.x.max(x), (it.x + it.width).min(x + w)))+ .filter(|(a, b)| b > a)+ .collect();+ spans.sort_by(|a, b| a.0.total_cmp(&b.0));+ let (mut covered, mut end) = (0.0f32, f32::NEG_INFINITY);+ for (a, b) in spans {+ if b > end {+ covered += b - a.max(end);+ end = b;+ }+ }+ covered / w+}+fn overlaps_existing_text(items: &[TextItem],ocr_x: f32,@@ -2894,6 +3086,74 @@ mod tests {assert!(!page_is_garbled(&page));}+ /// A font map wrong for a few glyphs (č -> `"`, ý -> `*`, j -> `G`) leaves text that reads+ /// almost right. One misread char across three or more words condemns every subset of+ /// that base font on the page; the healthy font's text stays.+ #[test]+ fn misread_glyphs_flag_every_subset_of_the_base_font() {+ let mut items: Vec<(&str, &str)> = HEALTHY_PARAGRAPH+ .iter()+ .map(|t| ("Helvetica", *t))+ .collect();+ items.push(("AAAAAB+Futura", "Tato kter*ch nabíječka je jednou z jin*m"));+ items.push(("AAAAAC+Futura", "nejnověj*ch a b*t"));+ items.push(("AAAAAD+Futura", "Kontakt"));+ let page = make_font_page(&items);+ match garbled_scope(&page) {+ GarbledScope::Fonts(fonts) => {+ assert!(fonts.contains(&Some("AAAAAB+Futura".to_string())));+ assert!(fonts.contains(&Some("AAAAAD+Futura".to_string())));+ assert!(!fonts.contains(&Some("Helvetica".to_string())));+ }+ _ => panic!("expected the Futura subsets to be flagged"),+ }+ }++ /// Compound names ending in one repeated part are no broken map either.+ #[test]+ fn misread_glyphs_ignore_compound_names_with_one_tail() {+ let mut items: Vec<(&str, &str)> = HEALTHY_PARAGRAPH+ .iter()+ .map(|t| ("Helvetica", *t))+ .collect();+ items.push((+ "STIX-Regular",+ "HistoAge PhenoAge GrimAge pTau acceleration",+ ));+ let page = make_font_page(&items);+ assert!(matches!(garbled_scope(&page), GarbledScope::None));+ }++ /// Brand and camel-case names repeat one word, quotations lack a leading space, units carry+ /// digits: none of them is a misread glyph.+ #[test]+ fn misread_glyphs_ignore_names_quotes_and_units() {+ for text in [+ "arXiv arXiv arXiv iPad iPad McRae McRae",+ "the expression\"put out fire\" and emerging\"care in parentheses\"exemplified",+ "70dBc -94dBc 21dBm 25dBm PubTabNet",+ ] {+ let found = misread_glyphs(text);+ let words: HashSet<&str> = found.iter().map(|m| m.word.as_str()).collect();+ assert!(words.len() < MISREAD_MIN_WORDS, "{text:?} -> {found:?}");+ }+ let found = misread_glyphs("maGor OecutiMe horiQon va!í");+ assert_eq!(found.iter().map(|m| m.ch).collect::<String>(), "GMQ!");+ assert_eq!(found[0].word, "magor");+ }++ /// Kept native glyphs inside an OCR line cover only a sliver of its width.+ #[test]+ fn native_width_coverage_merges_overlapping_spans() {+ let items = vec![+ make_item(0.0, 0.0, 10.0, 10.0),+ make_item(5.0, 0.0, 10.0, 10.0),+ ];+ assert!((native_width_coverage(&items, 0.0, 0.0, 100.0, 10.0) - 0.15).abs() < 1e-4);+ // a line above or below does not count+ assert_eq!(native_width_coverage(&items, 0.0, 20.0, 100.0, 10.0), 0.0);+ }+/// A small run of low-vowel text (acronyms, tickers, part numbers) in its/// own font must not flag an otherwise healthy page.#[test]diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex b995993..f504b45 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -946,9 +946,12 @@ async fn test_ocr_rounds_fill_across_sparse_pages() {let calls = Arc::new(AtomicUsize::new(0));let peak = Arc::new(AtomicUsize::new(0));+ // Fewer workers than the fixture's OCR pages (4: 2 sparse-text, 3 with images, overlapping),+ // so the window has more than one round of pages to fill across native-text pages.+ let workers = 3;let parser = LiteParse::new(LiteParseConfig {ocr_enabled: true,- num_workers: 8,+ num_workers: workers,dpi: 72.0, // Small rasters: this test is about scheduling, not pixels.quiet: true,..Default::default()@@ -966,7 +969,7 @@ async fn test_ocr_rounds_fill_across_sparse_pages() {let recognized = calls.load(Ordering::SeqCst);assert!(- recognized > 8,+ recognized > workers,"expected the fixture to need OCR on more than one round's worth of pages, got {recognized}");assert!(@@ -979,7 +982,7 @@ async fn test_ocr_rounds_fill_across_sparse_pages() {// pages are interleaved with native-text pages it skips.assert_eq!(peak.load(Ordering::SeqCst),- 8,+ workers,"the OCR window must fill to num_workers across skipped pages; a lower peak means \the window is being cut short by page span");