Skip to content

A fast, helpful, and open-source document parser

fix: don't treat short CJK OCR words as chart labels

Diego-Romero-NTT409afad63435
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index ad7124a..c94a190 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -1564,9 +1564,14 @@ fn box_center_in(rect: &Rect, x: f32, y: f32, w: f32, h: f32) -> bool {
}
/// A chart label: a number (with separators/units) or a token of ≤ 3 chars.
+///
+/// The length rule only applies to text without CJK characters. Tesseract
+/// splits Chinese, Japanese and Korean text into words of one to three
+/// characters ("を", "変換", "します"), so in those scripts a short token is
+/// ordinary prose, not an axis tick or legend key.
fn is_chart_label(text: &str) -> bool {
let t = text.trim();
- if t.chars().count() <= 3 {
+ if t.chars().count() <= 3 && !t.chars().any(is_cjk) {
return true;
}
t.chars().any(|c| c.is_ascii_digit())
@@ -1579,6 +1584,21 @@ fn is_chart_label(text: &str) -> bool {
})
}
+/// Han ideographs, kana and Hangul: scripts written without spaces between
+/// words, or with short syllable blocks, where OCR words are naturally short.
+fn is_cjk(c: char) -> bool {
+ matches!(c,
+ '\u{1100}'..='\u{11FF}' // Hangul Jamo
+ | '\u{3040}'..='\u{30FF}' // Hiragana, Katakana
+ | '\u{3130}'..='\u{318F}' // Hangul Compatibility Jamo
+ | '\u{3400}'..='\u{4DBF}' // CJK Unified Ideographs Extension A
+ | '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs
+ | '\u{AC00}'..='\u{D7AF}' // Hangul Syllables
+ | '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs
+ | '\u{FF66}'..='\u{FF9F}' // Halfwidth Katakana
+ )
+}
+
/// For each figure rect, whether its OCR results look like a chart's
/// axes/ticks/legend rather than text worth keeping.
fn chart_like_images(rects: &[Rect], results: &[OcrResult], scale_factor: f32) -> Vec<bool> {
@@ -1699,6 +1719,12 @@ mod tests {
assert!(is_chart_label("Yr"));
assert!(!is_chart_label("Year"));
assert!(!is_chart_label("Fruit Production in British Columbia"));
+ // Short CJK words are prose, not chart labels.
+ assert!(!is_chart_label("を"));
+ assert!(!is_chart_label("変換"));
+ assert!(!is_chart_label("します"));
+ assert!(!is_chart_label("中文"));
+ assert!(!is_chart_label("한국"));
let rect = Rect {
x: 50.0,
@@ -1729,6 +1755,19 @@ mod tests {
mk("2020", 100.0, 240.0),
];
assert_eq!(chart_like_images(&[rect.clone()], &prose, 1.0), vec![false]);
+ // Japanese prose, which OCR returns as one- to three-character words:
+ // not a chart.
+ let japanese = vec![
+ mk("文書", 60.0, 110.0),
+ mk("を", 80.0, 110.0),
+ mk("変換", 100.0, 110.0),
+ mk("します", 120.0, 110.0),
+ mk("。", 140.0, 110.0),
+ ];
+ assert_eq!(
+ chart_like_images(&[rect.clone()], &japanese, 1.0),
+ vec![false]
+ );
// Too few results to judge.
assert_eq!(chart_like_images(&[rect], &chart[..3], 1.0), vec![false]);
}