Skip to content

A fast, helpful, and open-source document parser

perf: load PDF once when OCR is enabled

AdemBoukhris4570a60b656e5a7
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 25d4cb8..18fa74a 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -1,6 +1,18 @@
use crate::error::LiteParseError;
use crate::types::{Page as LitePage, PdfInput, TextItem};
-use pdfium::{Font, FontType, Library, Page, RectF, TextPage};
+use pdfium::{Document, Font, FontType, Library, Page, RectF, TextPage};
+
+/// Open a PDF from path or bytes with an optional password.
+pub(crate) fn load_document_from_input(
+ input: &PdfInput,
+ password: Option<&str>,
+) -> Result<Document, LiteParseError> {
+ let lib = Library::init();
+ match input {
+ PdfInput::Path(path) => Ok(lib.load_document(path, password)?),
+ PdfInput::Bytes(data) => Ok(lib.load_document_from_bytes(data, password)?),
+ }
+}
/// Extract pages from a PDF file and return them as structured data.
pub fn extract_pages(
@@ -38,11 +50,16 @@ pub fn extract_pages_from_input(
max_pages: usize,
password: Option<&str>,
) -> Result<Vec<LitePage>, LiteParseError> {
- let lib = Library::init();
- let document = match input {
- PdfInput::Path(path) => lib.load_document(path, password)?,
- PdfInput::Bytes(data) => lib.load_document_from_bytes(data, password)?,
- };
+ let document = load_document_from_input(input, password)?;
+ extract_pages_from_document(&document, target_pages, max_pages)
+}
+
+/// Extract pages from an already-open PDFium document.
+pub(crate) fn extract_pages_from_document(
+ document: &Document,
+ target_pages: Option<&[u32]>,
+ max_pages: usize,
+) -> Result<Vec<LitePage>, LiteParseError> {
let page_count = document.page_count();
let mut pages = Vec::new();
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index b2455af..2b3c7ed 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -1,10 +1,19 @@
use std::sync::Arc;
use crate::error::LiteParseError;
+use crate::extract::load_document_from_input;
use crate::ocr::{OcrEngine, OcrOptions, OcrResult};
use crate::types::{Page, PdfInput, TextItem};
use image::{ImageBuffer, Rgba};
-use pdfium::Library;
+use pdfium::Document;
+
+/// Owned page bitmap prepared for OCR. Indices refer to positions in the `pages` slice.
+pub(crate) struct RenderedPage {
+ pub idx: usize,
+ pub rgb_bytes: Vec<u8>,
+ pub width: u32,
+ pub height: u32,
+}
/// Run OCR on pages that need it and merge results into text_items.
///
@@ -41,58 +50,62 @@ pub async fn ocr_and_merge_pages_from_input(
ocr_language: &str,
num_workers: usize,
) -> Result<(), LiteParseError> {
- // Phase 1: render all pages that need OCR. The pdfium `Document` holds raw
- // pointers that are not `Send`, so we must not hold it across an `.await`.
- // Render synchronously into owned buffers and drop the document before
- // awaiting the OCR engine.
- struct RenderedPage {
- idx: usize,
- rgb_bytes: Vec<u8>,
- width: u32,
- height: u32,
- }
-
- let rendered: Vec<RenderedPage> = {
- let lib = Library::init();
- let document = match input {
- PdfInput::Path(path) => lib.load_document(path, None)?,
- PdfInput::Bytes(data) => lib.load_document_from_bytes(data, None)?,
- };
-
- let mut rendered = Vec::new();
- for (idx, page) in pages.iter().enumerate() {
- let text_length: usize = page.text_items.iter().map(|item| item.text.len()).sum();
- let page_obj = document.page((page.page_number - 1) as i32)?;
- let has_images = !page_obj.image_bounds(25.0, 0.9).is_empty();
-
- if text_length >= 100 && !has_images {
- continue;
- }
+ let document = load_document_from_input(input, None)?;
+ let rendered = render_pages_for_ocr(&document, pages, dpi)?;
+ ocr_and_merge_rendered(pages, rendered, dpi, ocr_engine, ocr_language, num_workers).await
+}
- let bitmap = page_obj.render(dpi)?;
- let width = bitmap.width() as u32;
- let height = bitmap.height() as u32;
- let rgba = bitmap.to_rgba();
-
- let img: ImageBuffer<Rgba<u8>, Vec<u8>> = ImageBuffer::from_raw(width, height, rgba)
- .ok_or(LiteParseError::Other(
- "failed to create image buffer".into(),
- ))?;
- let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8();
- let rgb_bytes = rgb_img.into_raw();
-
- rendered.push(RenderedPage {
- idx,
- rgb_bytes,
- width,
- height,
- });
+/// Render pages that need OCR from an already-open document.
+///
+/// The pdfium `Document` holds raw pointers that are not `Send`, so callers must
+/// drop it before awaiting the OCR engine.
+pub(crate) fn render_pages_for_ocr(
+ document: &Document,
+ pages: &[Page],
+ dpi: f32,
+) -> Result<Vec<RenderedPage>, LiteParseError> {
+ let mut rendered = Vec::new();
+ for (idx, page) in pages.iter().enumerate() {
+ let text_length: usize = page.text_items.iter().map(|item| item.text.len()).sum();
+ let page_obj = document.page((page.page_number - 1) as i32)?;
+ let has_images = !page_obj.image_bounds(25.0, 0.9).is_empty();
+
+ if text_length >= 100 && !has_images {
+ continue;
}
- rendered
- // `document` and `lib` are dropped here before any `.await`
- };
- // Phase 2: spawn OCR tasks onto the tokio runtime so they run on
+ let bitmap = page_obj.render(dpi)?;
+ let width = bitmap.width() as u32;
+ let height = bitmap.height() as u32;
+ let rgba = bitmap.to_rgba();
+
+ let img: ImageBuffer<Rgba<u8>, Vec<u8>> = ImageBuffer::from_raw(width, height, rgba)
+ .ok_or(LiteParseError::Other(
+ "failed to create image buffer".into(),
+ ))?;
+ let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8();
+ let rgb_bytes = rgb_img.into_raw();
+
+ rendered.push(RenderedPage {
+ idx,
+ rgb_bytes,
+ width,
+ height,
+ });
+ }
+ Ok(rendered)
+}
+
+/// Run OCR on pre-rendered page bitmaps and merge results into `pages`.
+pub(crate) async fn ocr_and_merge_rendered(
+ pages: &mut [Page],
+ rendered: Vec<RenderedPage>,
+ dpi: f32,
+ ocr_engine: Arc<dyn OcrEngine>,
+ ocr_language: &str,
+ num_workers: usize,
+) -> Result<(), LiteParseError> {
+ // Phase 1: spawn OCR tasks onto the tokio runtime so they run on
// separate threads. A semaphore limits concurrency to `num_workers`.
let num_workers = num_workers.max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(num_workers));
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index be046e8..2380d05 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -120,13 +120,28 @@ impl LiteParse {
.transpose()
.map_err(|e| format!("invalid --target-pages: {}", e))?;
- // Extract text items from PDF pages
- let mut pages = extract::extract_pages_from_input(
- &validated_input,
- target_pages.as_deref(),
- self.config.max_pages,
- self.config.password.as_deref(),
- )?;
+ // Extract text (and pre-render OCR pages in one PDF load when OCR is on).
+ let password = self.config.password.as_deref();
+ let (mut pages, ocr_rendered) = if self.config.ocr_enabled {
+ let document =
+ extract::load_document_from_input(&validated_input, password)?;
+ let pages = extract::extract_pages_from_document(
+ &document,
+ target_pages.as_deref(),
+ self.config.max_pages,
+ )?;
+ let rendered =
+ ocr_merge::render_pages_for_ocr(&document, &pages, self.config.dpi)?;
+ (pages, rendered)
+ } else {
+ let pages = extract::extract_pages_from_input(
+ &validated_input,
+ target_pages.as_deref(),
+ self.config.max_pages,
+ password,
+ )?;
+ (pages, Vec::new())
+ };
let t1 = web_time::Instant::now();
log(&format!(
"[liteparse] extract: {:.1}ms ({} pages)",
@@ -165,9 +180,9 @@ impl LiteParse {
);
}
};
- ocr_merge::ocr_and_merge_pages_from_input(
+ ocr_merge::ocr_and_merge_rendered(
&mut pages,
- &validated_input,
+ ocr_rendered,
self.config.dpi,
engine,
&self.config.ocr_language,