A fast, helpful, and open-source document parser
support multi-lang with tesseract
Logan Markewiche40e584f0266
diff --git a/crates/liteparse/src/ocr/tesseract.rs b/crates/liteparse/src/ocr/tesseract.rsindex c41570f..fac913e 100644--- a/crates/liteparse/src/ocr/tesseract.rs+++ b/crates/liteparse/src/ocr/tesseract.rs@@ -15,7 +15,7 @@ impl TesseractOcrEngine {Self { tessdata_path }}- fn normalize_language(lang: &str) -> &str {+ fn normalize_language_code(lang: &str) -> &str {match lang.to_lowercase().trim() {"en" => "eng","fr" => "fra",@@ -32,9 +32,20 @@ impl TesseractOcrEngine {"hi" => "hin","th" => "tha","vi" => "vie",- _ => lang,+ _ => lang.trim(),}}++ /// Normalize a tesseract language spec, which may be a single code or a+ /// `+`-separated list (e.g. `ita+eng`). Each component is normalized+ /// independently and rejoined with `+`.+ fn normalize_language(lang: &str) -> String {+ lang.split('+')+ .filter(|part| !part.trim().is_empty())+ .map(Self::normalize_language_code)+ .collect::<Vec<_>>()+ .join("+")+ }}impl OcrEngine for TesseractOcrEngine {@@ -73,8 +84,10 @@ impl OcrEngine for TesseractOcrEngine {.or_else(|| std::env::var("TESSDATA_PREFIX").ok());let resolved_path = tessdata_path.unwrap_or_else(default_tessdata_dir);- ensure_traineddata(Path::new(&resolved_path), language).await?;- api.init(&resolved_path, language)?;+ for code in language.split('+') {+ ensure_traineddata(Path::new(&resolved_path), code).await?;+ }+ api.init(&resolved_path, &language)?;// Match the tesseract CLI's default page segmentation mode (PSM_AUTO,// i.e. 3). The C++ library's own default when SetPageSegMode is never@@ -233,6 +246,25 @@ mod tests {assert_eq!(TesseractOcrEngine::normalize_language("xyz"), "xyz");}+ #[test]+ fn test_normalize_language_multi() {+ // `+`-separated specs normalize each component independently.+ assert_eq!(TesseractOcrEngine::normalize_language("ita+eng"), "ita+eng");+ assert_eq!(TesseractOcrEngine::normalize_language("it+en"), "ita+eng");+ assert_eq!(+ TesseractOcrEngine::normalize_language(" it + en "),+ "ita+eng"+ );+ assert_eq!(TesseractOcrEngine::normalize_language("eng+xyz"), "eng+xyz");+ // Empty components (leading/trailing/double `+`) are dropped.+ assert_eq!(TesseractOcrEngine::normalize_language("eng+"), "eng");+ assert_eq!(TesseractOcrEngine::normalize_language("+eng"), "eng");+ assert_eq!(+ TesseractOcrEngine::normalize_language("ita++eng"),+ "ita+eng"+ );+ }+#[test]fn test_engine_name() {let e = TesseractOcrEngine::new(None);