A fast, helpful, and open-source document parser
perf: load PDF once when OCR is enabled
AdemBoukhris4570a60b656e5a7
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex 25d4cb8..18fa74a 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -1,6 +1,18 @@use crate::error::LiteParseError;use crate::types::{Page as LitePage, PdfInput, TextItem};-use pdfium::{Font, FontType, Library, Page, RectF, TextPage};+use pdfium::{Document, Font, FontType, Library, Page, RectF, TextPage};++/// Open a PDF from path or bytes with an optional password.+pub(crate) fn load_document_from_input(+ input: &PdfInput,+ password: Option<&str>,+) -> Result<Document, LiteParseError> {+ let lib = Library::init();+ match input {+ PdfInput::Path(path) => Ok(lib.load_document(path, password)?),+ PdfInput::Bytes(data) => Ok(lib.load_document_from_bytes(data, password)?),+ }+}/// Extract pages from a PDF file and return them as structured data.pub fn extract_pages(@@ -38,11 +50,16 @@ pub fn extract_pages_from_input(max_pages: usize,password: Option<&str>,) -> Result<Vec<LitePage>, LiteParseError> {- let lib = Library::init();- let document = match input {- PdfInput::Path(path) => lib.load_document(path, password)?,- PdfInput::Bytes(data) => lib.load_document_from_bytes(data, password)?,- };+ let document = load_document_from_input(input, password)?;+ extract_pages_from_document(&document, target_pages, max_pages)+}++/// Extract pages from an already-open PDFium document.+pub(crate) fn extract_pages_from_document(+ document: &Document,+ target_pages: Option<&[u32]>,+ max_pages: usize,+) -> Result<Vec<LitePage>, LiteParseError> {let page_count = document.page_count();let mut pages = Vec::new();diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex b2455af..2b3c7ed 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -1,10 +1,19 @@use std::sync::Arc;use crate::error::LiteParseError;+use crate::extract::load_document_from_input;use crate::ocr::{OcrEngine, OcrOptions, OcrResult};use crate::types::{Page, PdfInput, TextItem};use image::{ImageBuffer, Rgba};-use pdfium::Library;+use pdfium::Document;++/// Owned page bitmap prepared for OCR. Indices refer to positions in the `pages` slice.+pub(crate) struct RenderedPage {+ pub idx: usize,+ pub rgb_bytes: Vec<u8>,+ pub width: u32,+ pub height: u32,+}/// Run OCR on pages that need it and merge results into text_items.///@@ -41,58 +50,62 @@ pub async fn ocr_and_merge_pages_from_input(ocr_language: &str,num_workers: usize,) -> Result<(), LiteParseError> {- // Phase 1: render all pages that need OCR. The pdfium `Document` holds raw- // pointers that are not `Send`, so we must not hold it across an `.await`.- // Render synchronously into owned buffers and drop the document before- // awaiting the OCR engine.- struct RenderedPage {- idx: usize,- rgb_bytes: Vec<u8>,- width: u32,- height: u32,- }-- let rendered: Vec<RenderedPage> = {- let lib = Library::init();- let document = match input {- PdfInput::Path(path) => lib.load_document(path, None)?,- PdfInput::Bytes(data) => lib.load_document_from_bytes(data, None)?,- };-- let mut rendered = Vec::new();- for (idx, page) in pages.iter().enumerate() {- let text_length: usize = page.text_items.iter().map(|item| item.text.len()).sum();- let page_obj = document.page((page.page_number - 1) as i32)?;- let has_images = !page_obj.image_bounds(25.0, 0.9).is_empty();-- if text_length >= 100 && !has_images {- continue;- }+ let document = load_document_from_input(input, None)?;+ let rendered = render_pages_for_ocr(&document, pages, dpi)?;+ ocr_and_merge_rendered(pages, rendered, dpi, ocr_engine, ocr_language, num_workers).await+}- let bitmap = page_obj.render(dpi)?;- let width = bitmap.width() as u32;- let height = bitmap.height() as u32;- let rgba = bitmap.to_rgba();-- let img: ImageBuffer<Rgba<u8>, Vec<u8>> = ImageBuffer::from_raw(width, height, rgba)- .ok_or(LiteParseError::Other(- "failed to create image buffer".into(),- ))?;- let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8();- let rgb_bytes = rgb_img.into_raw();-- rendered.push(RenderedPage {- idx,- rgb_bytes,- width,- height,- });+/// Render pages that need OCR from an already-open document.+///+/// The pdfium `Document` holds raw pointers that are not `Send`, so callers must+/// drop it before awaiting the OCR engine.+pub(crate) fn render_pages_for_ocr(+ document: &Document,+ pages: &[Page],+ dpi: f32,+) -> Result<Vec<RenderedPage>, LiteParseError> {+ let mut rendered = Vec::new();+ for (idx, page) in pages.iter().enumerate() {+ let text_length: usize = page.text_items.iter().map(|item| item.text.len()).sum();+ let page_obj = document.page((page.page_number - 1) as i32)?;+ let has_images = !page_obj.image_bounds(25.0, 0.9).is_empty();++ if text_length >= 100 && !has_images {+ continue;}- rendered- // `document` and `lib` are dropped here before any `.await`- };- // Phase 2: spawn OCR tasks onto the tokio runtime so they run on+ let bitmap = page_obj.render(dpi)?;+ let width = bitmap.width() as u32;+ let height = bitmap.height() as u32;+ let rgba = bitmap.to_rgba();++ let img: ImageBuffer<Rgba<u8>, Vec<u8>> = ImageBuffer::from_raw(width, height, rgba)+ .ok_or(LiteParseError::Other(+ "failed to create image buffer".into(),+ ))?;+ let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8();+ let rgb_bytes = rgb_img.into_raw();++ rendered.push(RenderedPage {+ idx,+ rgb_bytes,+ width,+ height,+ });+ }+ Ok(rendered)+}++/// Run OCR on pre-rendered page bitmaps and merge results into `pages`.+pub(crate) async fn ocr_and_merge_rendered(+ pages: &mut [Page],+ rendered: Vec<RenderedPage>,+ dpi: f32,+ ocr_engine: Arc<dyn OcrEngine>,+ ocr_language: &str,+ num_workers: usize,+) -> Result<(), LiteParseError> {+ // Phase 1: spawn OCR tasks onto the tokio runtime so they run on// separate threads. A semaphore limits concurrency to `num_workers`.let num_workers = num_workers.max(1);let semaphore = Arc::new(tokio::sync::Semaphore::new(num_workers));diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex be046e8..2380d05 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -120,13 +120,28 @@ impl LiteParse {.transpose().map_err(|e| format!("invalid --target-pages: {}", e))?;- // Extract text items from PDF pages- let mut pages = extract::extract_pages_from_input(- &validated_input,- target_pages.as_deref(),- self.config.max_pages,- self.config.password.as_deref(),- )?;+ // Extract text (and pre-render OCR pages in one PDF load when OCR is on).+ let password = self.config.password.as_deref();+ let (mut pages, ocr_rendered) = if self.config.ocr_enabled {+ let document =+ extract::load_document_from_input(&validated_input, password)?;+ let pages = extract::extract_pages_from_document(+ &document,+ target_pages.as_deref(),+ self.config.max_pages,+ )?;+ let rendered =+ ocr_merge::render_pages_for_ocr(&document, &pages, self.config.dpi)?;+ (pages, rendered)+ } else {+ let pages = extract::extract_pages_from_input(+ &validated_input,+ target_pages.as_deref(),+ self.config.max_pages,+ password,+ )?;+ (pages, Vec::new())+ };let t1 = web_time::Instant::now();log(&format!("[liteparse] extract: {:.1}ms ({} pages)",@@ -165,9 +180,9 @@ impl LiteParse {);}};- ocr_merge::ocr_and_merge_pages_from_input(+ ocr_merge::ocr_and_merge_rendered(&mut pages,- &validated_input,+ ocr_rendered,self.config.dpi,engine,&self.config.ocr_language,