A fast, helpful, and open-source document parser
refactor and simplify
Logan Markewichebf87f91a75f
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rsindex 351d076..4d7df3d 100644--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@ -119,14 +119,6 @@ pub struct JsLiteParseConfig {pub include_complexity: Option<bool>,/// Expose page-scoped vector path extraction. Default false.pub extract_vector_graphics: Option<bool>,- /// Approximate memory budget (MiB) for extracted content accumulated- /// during a parse; the parse rejects with a clear error instead of- /// growing without bound. 0 disables. Default 4096.- pub memory_budget_mb: Option<u32>,- /// Approximate budget (MiB) for OCR page rasters held in memory at once;- /// pages render and recognize in rounds sized to this budget. 0 renders- /// all pages up front. Default 1024.- pub ocr_raster_budget_mb: Option<u32>,}/// A page sub-region as the fraction cropped from each side (top-left origin,@@ -267,12 +259,6 @@ impl JsLiteParseConfig {if let Some(v) = self.extract_vector_graphics {cfg.extract_vector_graphics = v;}- if let Some(v) = self.memory_budget_mb {- cfg.memory_budget_mb = v as usize;- }- if let Some(v) = self.ocr_raster_budget_mb {- cfg.ocr_raster_budget_mb = v as usize;- }cfg}@@ -337,8 +323,6 @@ impl JsLiteParseConfig {skip_diagonal_text: Some(cfg.skip_diagonal_text),include_complexity: Some(cfg.include_complexity),extract_vector_graphics: Some(cfg.extract_vector_graphics),- memory_budget_mb: Some(cfg.memory_budget_mb as u32),- ocr_raster_budget_mb: Some(cfg.ocr_raster_budget_mb as u32),}}}diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rsindex db21b9e..5ad832d 100644--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@ -1268,10 +1268,6 @@ struct PyLiteParseConfig {extract_images: bool,#[pyo3(get)]extract_vector_graphics: bool,- #[pyo3(get)]- memory_budget_mb: usize,- #[pyo3(get)]- ocr_raster_budget_mb: usize,}#[pymethods]@@ -1339,8 +1335,6 @@ impl PyLiteParseConfig {image_output_dir: cfg.image_output_dir.clone(),extract_images: cfg.extract_images,extract_vector_graphics: cfg.extract_vector_graphics,- memory_budget_mb: cfg.memory_budget_mb,- ocr_raster_budget_mb: cfg.ocr_raster_budget_mb,}}}@@ -1468,8 +1462,6 @@ impl LiteParse {skip_diagonal_text = None,include_complexity = None,extract_vector_graphics = None,- memory_budget_mb = None,- ocr_raster_budget_mb = None,))]fn new(ocr_language: Option<String>,@@ -1509,8 +1501,6 @@ impl LiteParse {skip_diagonal_text: Option<bool>,include_complexity: Option<bool>,extract_vector_graphics: Option<bool>,- memory_budget_mb: Option<usize>,- ocr_raster_budget_mb: Option<usize>,) -> PyResult<Self> {let mut cfg = LiteParseConfig::default();if let Some(v) = ocr_language {@@ -1637,12 +1627,6 @@ impl LiteParse {if let Some(v) = extract_vector_graphics {cfg.extract_vector_graphics = v;}- if let Some(v) = memory_budget_mb {- cfg.memory_budget_mb = v;- }- if let Some(v) = ocr_raster_budget_mb {- cfg.ocr_raster_budget_mb = v;- }let inner = liteparse::parser::LiteParse::new(cfg.clone());let runtime = std::sync::Arc::new(diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rsindex 527bbea..1341a02 100644--- a/crates/liteparse-wasm/src/lib.rs+++ b/crates/liteparse-wasm/src/lib.rs@@ -48,13 +48,6 @@ pub struct LiteParseConfig {ocr_server_headers: Option<HashMap<String, String>>,tessdata_path: Option<String>,max_pages: Option<usize>,- /// Approximate memory budget (MiB) for extracted content accumulated- /// during a parse; the parse rejects with a clear error instead of- /// growing without bound. 0 disables. Default 4096.- memory_budget_mb: Option<usize>,- /// Approximate budget (MiB) for OCR page rasters held in memory at once.- /// 0 renders all pages up front. Default 1024.- ocr_raster_budget_mb: Option<usize>,target_pages: Option<String>,/// Skip page-level PDF extraction failures and report them in/// `ParseResult.pageErrors`. Document-level failures remain fatal.@@ -151,12 +144,6 @@ impl LiteParseConfig {if let Some(v) = self.max_pages {cfg.max_pages = v;}- if let Some(v) = self.memory_budget_mb {- cfg.memory_budget_mb = v;- }- if let Some(v) = self.ocr_raster_budget_mb {- cfg.ocr_raster_budget_mb = v;- }if self.target_pages.is_some() {cfg.target_pages = self.target_pages;}@@ -279,8 +266,6 @@ impl LiteParseConfig {},tessdata_path: cfg.tessdata_path.clone(),max_pages: Some(cfg.max_pages),- memory_budget_mb: Some(cfg.memory_budget_mb),- ocr_raster_budget_mb: Some(cfg.ocr_raster_budget_mb),target_pages: cfg.target_pages.clone(),continue_on_page_error: Some(cfg.continue_on_page_error),extract_screenshots: Some(cfg.extract_screenshots),diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex 79b62db..ff3a8f7 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -163,47 +163,6 @@ pub struct LiteParseConfig {/// vertical `lines`) in parse results. Default `false`; path objects are/// still inspected internally for layout detection when disabled.pub extract_vector_graphics: bool,- /// Approximate budget, in MiB, for extracted content accumulated in- /// memory during a parse (text items, word boxes, embedded image bytes,- /// screenshot PNGs). When the running estimate crosses the budget the- /// parse fails with [`crate::LiteParseError::MemoryBudget`] instead of- /// growing without bound on pathological inputs. `0` disables the check.- ///- /// The estimate is a lower bound on real usage (it counts payload bytes- /// and struct sizes, not allocator or formatting overhead), so treat the- /// budget as "reject clearly-oversized documents", not as a precise cap.- /// Not counted: vector-graphics paths, structure-tree nodes, annotations- /// and form-field records — documents dominated by those can exceed the- /// budget's view of memory. The standalone `screenshot()` API is exempt.- /// In batch parsing the budget applies per batch, not across the whole- /// session — the caller controls cross-batch accumulation.- /// Default 4096 MiB. For large documents prefer narrowing- /// `target_pages`/`max_pages` or batch parsing over raising the budget.- #[serde(default = "default_memory_budget_mb")]- pub memory_budget_mb: usize,- /// Approximate budget, in MiB, for OCR page rasters held in memory at- /// once. Pages are rendered and OCR'd in bounded rounds sized to this- /// budget, instead of pre-rendering every text-poor page up front — on a- /// long scanned document the up-front strategy holds every raster- /// simultaneously. `0` restores the unbounded single-round behavior.- /// Default 1024 MiB; at least one page is always rendered per round.- ///- /// Rounds run serially, so effective OCR concurrency is- /// `min(num_workers, pages per round)` — a budget small enough to admit- /// fewer pages than `num_workers` reduces OCR throughput. The default- /// admits far more pages per round than any realistic worker count.- /// Each round also reopens the document once (one document load per- /// round; small next to rendering plus recognition for a round's pages).- #[serde(default = "default_ocr_raster_budget_mb")]- pub ocr_raster_budget_mb: usize,-}--fn default_memory_budget_mb() -> usize {- 4096-}--fn default_ocr_raster_budget_mb() -> usize {- 1024}/// A page sub-region expressed as the fraction cropped from each side.@@ -245,11 +204,6 @@ pub enum OutputFormat {}impl LiteParseConfig {- /// The extracted-content memory budget in bytes; `0` means disabled.- pub fn memory_budget_bytes(&self) -> u64 {- self.memory_budget_mb as u64 * 1024 * 1024- }-/// Whether embedded-image extraction should run. True when/// `extract_images` is set, or when the legacy `ImageMode::Embed` is/// configured (`Embed` implies byte extraction for backwards@@ -309,8 +263,6 @@ impl Default for LiteParseConfig {skip_diagonal_text: false,include_complexity: false,extract_vector_graphics: false,- memory_budget_mb: default_memory_budget_mb(),- ocr_raster_budget_mb: default_ocr_raster_budget_mb(),}}}diff --git a/crates/liteparse/src/error.rs b/crates/liteparse/src/error.rsindex d918dbb..bb99e54 100644--- a/crates/liteparse/src/error.rs+++ b/crates/liteparse/src/error.rs@@ -1,7 +1,6 @@use thiserror::Error;#[derive(Debug, Error)]-#[non_exhaustive]pub enum LiteParseError {#[error("PDF error: {0}")]Pdf(#[from] pdfium::PdfiumError),@@ -24,15 +23,6 @@ pub enum LiteParseError {#[error("invalid config: {0}")]Config(String),- #[error(- "memory budget exceeded: extracted content reached ~{used_mib} MiB (budget {budget_mib} MiB) at page {page_number}; raise `memory_budget_mb` (0 disables the check), narrow `target_pages`/`max_pages`, or parse in batches"- )]- MemoryBudget {- used_mib: u64,- budget_mib: u64,- page_number: u32,- },-#[error("{0}")]Other(String),}diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex da90966..693756f 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -81,10 +81,6 @@ pub(crate) struct ExtractedPages {/// pages — flattening a page extraction never touched hides that page's/// non-widget annotations from the raster.pub flattened_page_numbers: Vec<u32>,- /// Running estimate of the bytes held by `pages` + `images`, as counted- /// against `ExtractionOutputOptions::memory_budget_bytes`. Lets callers- /// charge later allocations (e.g. screenshot PNGs) to the same budget.- pub approx_extracted_bytes: u64,}/// Same as `extract_pages_from_document` but optionally also renders every@@ -107,7 +103,6 @@ pub(crate) fn extract_pages_and_images(let mut image_error_count = 0u32;let mut flattened_form_widgets = false;let mut flattened_page_numbers: Vec<u32> = Vec::new();- let mut approx_extracted_bytes = 0u64;// One FFI call keeps the per-page annotation walk off the hot path for// every document without an AcroForm catalog, which is nearly all of them.let document_has_form = document.form_type() != 0;@@ -148,10 +143,6 @@ pub(crate) fn extract_pages_and_images(&mut page_errors,)? {Some(extraction) => {- approx_extracted_bytes += approx_page_bytes(&extraction.page);- for image in &extraction.images {- approx_extracted_bytes += approx_image_bytes(image);- }if extraction.flattened_form_widgets {flattened_page_numbers.push(page_number);}@@ -159,15 +150,6 @@ pub(crate) fn extract_pages_and_images(images.extend(extraction.images);image_error_count += extraction.image_error_count;flattened_form_widgets |= extraction.flattened_form_widgets;- if output_options.memory_budget_bytes > 0- && approx_extracted_bytes > output_options.memory_budget_bytes- {- return Err(LiteParseError::MemoryBudget {- used_mib: approx_extracted_bytes / (1024 * 1024),- budget_mib: output_options.memory_budget_bytes / (1024 * 1024),- page_number,- });- }}// A failed page's cached renders must not seed dedup for later// pages: a hit would emit `duplicate_of` pointing at an image id@@ -183,7 +165,6 @@ pub(crate) fn extract_pages_and_images(image_error_count,flattened_form_widgets,flattened_page_numbers,- approx_extracted_bytes,})}@@ -468,52 +449,6 @@ pub(crate) struct ExtractionOutputOptions {pub extract_form_fields: bool,pub extract_structure_tree: bool,pub emit_word_boxes: bool,- /// Approximate byte budget for accumulated pages + images; `0` disables- /// the check. See `LiteParseConfig::memory_budget_mb`.- pub memory_budget_bytes: u64,-}--/// Approximate resident bytes of one extracted page: struct sizes plus the-/// owned payloads that dominate on text-dense documents (item text, per-word-/// boxes, char codes, string fields). Deliberately a lower bound — allocator-/// slack and later formatting copies are not counted — so the budget check-/// errs toward admitting.-pub(crate) fn approx_page_bytes(page: &LitePage) -> u64 {- let mut bytes = std::mem::size_of::<LitePage>() as u64;- for item in &page.text_items {- bytes += std::mem::size_of::<TextItem>() as u64;- bytes += item.text.len() as u64;- bytes += (item.char_codes.len() * std::mem::size_of::<u32>()) as u64;- bytes += item.font_name.as_deref().map_or(0, str::len) as u64;- bytes += item.fill_color.as_deref().map_or(0, str::len) as u64;- bytes += item.stroke_color.as_deref().map_or(0, str::len) as u64;- bytes += item.link.as_deref().map_or(0, str::len) as u64;- for word in &item.words {- bytes += std::mem::size_of::<WordBox>() as u64 + word.text.len() as u64;- }- }- bytes += (page.graphics.len() * std::mem::size_of::<GraphicPrimitive>()) as u64;- if let Some(vector_graphics) = &page.vector_graphics {- bytes += (vector_graphics.shapes.len() * std::mem::size_of::<VectorShape>()) as u64;- bytes += (vector_graphics.lines.len() * std::mem::size_of::<VectorLine>()) as u64;- }- bytes += (page.struct_nodes.len() * std::mem::size_of::<StructNode>()) as u64;- bytes-}--/// Approximate resident bytes of one extracted image. Duplicate entries share-/// the canonical image's `Arc` buffer, so only the first occurrence pays for-/// the pixel bytes.-fn approx_image_bytes(image: &ExtractedImage) -> u64 {- let payload = if image.duplicate_of.is_none() {- image.bytes.len() as u64- } else {- 0- };- std::mem::size_of::<ExtractedImage>() as u64- + payload- + image.id.len() as u64- + image.name.len() as u64}fn document_annotation(annotation: &pdfium::PdfAnnotation) -> DocumentAnnotation {@@ -3740,60 +3675,4 @@ mod tests {Some("p3_1"));}-- #[test]- fn memory_budget_rejects_oversized_extraction() {- let bytes = rotated_text_pdf();-- // The document borrows the input buffer, so the input must outlive it.- let input = PdfInput::Bytes(bytes);-- // A budget of one byte trips on the first extracted page.- let error = {- let lib = Library::init();- let document = load_document_from_input(&lib, &input, None).unwrap();- match extract_pages_and_images(- &document,- None,- usize::MAX,- false,- None,- ExtractionOutputOptions {- memory_budget_bytes: 1,- ..Default::default()- },- ) {- Ok(_) => panic!("expected the one-byte budget to reject the extraction"),- Err(error) => error,- }- };- match error {- LiteParseError::MemoryBudget {- budget_mib,- page_number,- ..- } => {- assert_eq!(budget_mib, 0); // one byte rounds down to 0 MiB- assert_eq!(page_number, 1);- }- other => panic!("expected MemoryBudget, got: {other}"),- }-- // Budget 0 disables the check, and the running estimate is reported.- let lib = Library::init();- let document = load_document_from_input(&lib, &input, None).unwrap();- let extracted = extract_pages_and_images(- &document,- None,- usize::MAX,- false,- None,- ExtractionOutputOptions::default(),- )- .unwrap();- assert_eq!(extracted.pages.len(), 2);- assert!(extracted.approx_extracted_bytes > 0);- let per_page_sum: u64 = extracted.pages.iter().map(approx_page_bytes).sum();- assert_eq!(extracted.approx_extracted_bytes, per_page_sum);- }}diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rsindex 080ad7a..b0dd380 100644--- a/crates/liteparse/src/main.rs+++ b/crates/liteparse/src/main.rs@@ -75,18 +75,6 @@ struct ParseCommand {#[arg(long, default_value = "1000")]max_pages: usize,- /// Approximate memory budget (MiB) for extracted content held in memory;- /// the parse fails with a clear error instead of growing without bound.- /// 0 disables the check.- #[arg(long, default_value = "4096")]- memory_budget_mb: usize,-- /// Approximate budget (MiB) for OCR page rasters held in memory at once;- /// pages are rendered and recognized in rounds sized to this budget.- /// 0 renders all pages up front.- #[arg(long, default_value = "1024")]- ocr_raster_budget_mb: usize,-/// Target pages (e.g., "1-5,10,15-20")#[arg(long)]target_pages: Option<String>,@@ -247,18 +235,6 @@ struct BatchParseCommand {#[arg(long, default_value = "1000")]max_pages: usize,- /// Approximate memory budget (MiB) for extracted content held in memory;- /// the parse fails with a clear error instead of growing without bound.- /// 0 disables the check.- #[arg(long, default_value = "4096")]- memory_budget_mb: usize,-- /// Approximate budget (MiB) for OCR page rasters held in memory at once;- /// pages are rendered and recognized in rounds sized to this budget.- /// 0 renders all pages up front.- #[arg(long, default_value = "1024")]- ocr_raster_budget_mb: usize,-/// Continue after page-level extraction errors and report them in JSON.#[arg(long)]continue_on_page_error: bool,@@ -470,8 +446,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {include_complexity: cmd.complexity,extract_text_metadata: cmd.extract_text_metadata,extract_vector_graphics: cmd.extract_vector_graphics,- memory_budget_mb: cmd.memory_budget_mb,- ocr_raster_budget_mb: cmd.ocr_raster_budget_mb,..Default::default()};if let Some(n) = cmd.num_workers {@@ -574,8 +548,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {extract_blocks: cmd.extract_blocks,extract_xfa_packets: cmd.extract_xfa_packets,extract_content_bounds: cmd.extract_content_bounds,- memory_budget_mb: cmd.memory_budget_mb,- ocr_raster_budget_mb: cmd.ocr_raster_budget_mb,..Default::default()};if let Some(n) = cmd.num_workers {@@ -879,48 +851,6 @@ mod tests {));}- #[test]- fn memory_budget_flags_are_available_for_parse_and_batch() {- let cli = Cli::try_parse_from([- "lit",- "parse",- "document.pdf",- "--memory-budget-mb",- "512",- "--ocr-raster-budget-mb",- "0",- ])- .unwrap();- assert!(matches!(- cli.command,- Commands::Parse(ParseCommand {- memory_budget_mb: 512,- ocr_raster_budget_mb: 0,- ..- })- ));-- let cli = Cli::try_parse_from([- "lit",- "batch-parse",- "input",- "output",- "--memory-budget-mb",- "512",- "--ocr-raster-budget-mb",- "0",- ])- .unwrap();- assert!(matches!(- cli.command,- Commands::BatchParse(BatchParseCommand {- memory_budget_mb: 512,- ocr_raster_budget_mb: 0,- ..- })- ));- }-#[test]fn extract_structure_tree_flag_is_available_for_parse_and_batch() {let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--extract-structure-tree"])diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex 1873ccf..1df6962 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -434,39 +434,33 @@ fn count_columns(region: &Region, total_items: usize) -> usize {/// to avoid excessive memory usage.pub(crate) const MAX_OCR_RENDER_LONG_EDGE_PX: f32 = 4096.0;-/// Upper-bound estimate of one page's OCR raster output bytes at `dpi`,-/// mirroring the long-edge DPI clamp applied in [`render_pages_for_ocr`].-/// Used to size bounded render→recognize rounds before anything is rendered.-/// Models the retained output buffer only; the render itself transiently-/// holds one additional 4-bytes-per-pixel bitmap for the page in flight.-pub(crate) fn estimate_ocr_raster_bytes(- page_width_pt: f32,- page_height_pt: f32,- dpi: f32,- grayscale: bool,-) -> u64 {- let long_edge_pt = page_width_pt.max(page_height_pt);- let mut eff_dpi = dpi;- if long_edge_pt > 0.0 {- let max_dpi = MAX_OCR_RENDER_LONG_EDGE_PX * 72.0 / long_edge_pt;- if eff_dpi > max_dpi {- eff_dpi = max_dpi;- }- }- let width_px = (page_width_pt / 72.0 * eff_dpi).max(1.0) as u64;- let height_px = (page_height_pt / 72.0 * eff_dpi).max(1.0) as u64;- width_px * height_px * if grayscale { 1 } else { 3 }-}-+/// Render the pages in `pages[start..]` that need OCR, stopping once+/// `max_rasters` of them have been rendered (`0` means no limit). Returns the+/// rasters plus the index to resume scanning from, so a caller can process a+/// long document in bounded rounds.+///+/// The bound is on rasters produced, not pages scanned, because pages that+/// need OCR can be sparse: bounding a round by page span would hand the OCR+/// engine only the OCR-needing pages that happen to fall in that span, which+/// on a mostly-native-text document starves the engine's worker pool and+/// serializes recognition that could have run concurrently. Skipping a page+/// costs a complexity check, not a render, so scanning ahead to fill a round+/// is cheap.+///+/// `RenderedPage::idx` indexes `pages` absolutely, so the caller merges+/// against the whole slice regardless of where the round started.+#[allow(clippy::too_many_arguments)]pub(crate) fn render_pages_for_ocr(document: &Document,pages: &[Page],+ start: usize,+ max_rasters: usize,dpi: f32,grayscale: bool,render_form_fields: bool,continue_on_page_error: bool,flatten_page_numbers: &std::collections::HashSet<u32>,-) -> Result<Vec<RenderedPage>, LiteParseError> {+) -> Result<(Vec<RenderedPage>, usize), LiteParseError> {let mut rendered = Vec::new();// With `render_form_fields`, draw form-field appearances into the OCR// raster so filled-in form values are visible to the OCR engine (matches@@ -478,7 +472,8 @@ pub(crate) fn render_pages_for_ocr(if let Some(form) = form.as_ref() {form.run_document_actions();}- for (idx, page) in pages.iter().enumerate() {+ let mut next_start = pages.len();+ for (idx, page) in pages.iter().enumerate().skip(start) {let page_render = (|| -> Result<Option<RenderedPage>, LiteParseError> {let page_index = (page.page_number - 1) as i32;// Text extraction may have flattened THIS page's widget@@ -531,7 +526,13 @@ pub(crate) fn render_pages_for_ocr(}))})();match page_render {- Ok(Some(render)) => rendered.push(render),+ Ok(Some(render)) => {+ rendered.push(render);+ if max_rasters > 0 && rendered.len() >= max_rasters {+ next_start = idx + 1;+ break;+ }+ }Ok(None) => {}// The page already extracted successfully, so a tolerant parse// keeps its native text and only forgoes the OCR enrichment.@@ -542,7 +543,7 @@ pub(crate) fn render_pages_for_ocr(Err(error) => return Err(error),}}- Ok(rendered)+ Ok((rendered, next_start))}/// Run OCR on pre-rendered page bitmaps and merge results into `pages`.@@ -1079,30 +1080,6 @@ mod tests {use super::*;use crate::types::Rect;- #[test]- fn estimate_ocr_raster_bytes_scales_and_clamps() {- // Letter page at 150 dpi: 1275 x 1650 px; RGB is three bytes a pixel,- // grayscale one.- assert_eq!(- estimate_ocr_raster_bytes(612.0, 792.0, 150.0, false),- 1275 * 1650 * 3- );- assert_eq!(- estimate_ocr_raster_bytes(612.0, 792.0, 150.0, true),- 1275 * 1650- );- // A large-format page is clamped to the long-edge pixel cap no matter- // how high the requested dpi is.- let clamped = estimate_ocr_raster_bytes(2592.0, 1728.0, 600.0, false);- let cap = MAX_OCR_RENDER_LONG_EDGE_PX as u64;- assert!(clamped <= cap * cap * 3);- assert_eq!(- clamped,- estimate_ocr_raster_bytes(2592.0, 1728.0, 10_000.0, false),- "beyond the clamp, higher dpi must not grow the estimate"- );- }-fn leaf(n: usize) -> Region {Region {bbox: Rect::default(),diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex cf303ae..b1feef3 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -381,7 +381,6 @@ impl LiteParse {self.glyph_resolver.as_deref(),extract::ExtractionOutputOptions {continue_on_page_error: self.config.continue_on_page_error,- memory_budget_bytes: self.config.memory_budget_bytes(),..Default::default()},)?@@ -668,7 +667,6 @@ impl LiteParse {extract_annotations: self.config.extract_annotations,extract_form_fields: self.config.extract_form_fields,extract_structure_tree: self.config.extract_structure_tree,- memory_budget_bytes: self.config.memory_budget_bytes(),},)?;let extract::ExtractedPages {@@ -678,7 +676,6 @@ impl LiteParse {image_error_count,flattened_form_widgets,flattened_page_numbers,- approx_extracted_bytes,} = extracted;// Reopening the input costs a full parse, so it is confined to the// one consumer that genuinely needs live widget annotations: the@@ -733,10 +730,6 @@ impl LiteParse {.iter().map(|page| page.page_number as u32).collect::<Vec<_>>();- // Screenshot PNGs accumulate alongside the extracted pages,- // so they draw down the same memory budget; the extraction- // bytes already spent are passed as pre-charged so the error- // reports the configured budget and the true running total.render::render_document_pages(analysis_document,Some(&page_numbers),@@ -744,8 +737,6 @@ impl LiteParse {self.config.detect_screenshot_rects,self.config.render_form_fields,self.config.continue_on_page_error,- self.config.memory_budget_bytes(),- approx_extracted_bytes,)?.into_iter().map(|page| ScreenshotResult {@@ -788,16 +779,33 @@ impl LiteParse {// OCR pass, in bounded render→recognize rounds. Rendering every// text-poor page before recognizing any of them holds one raster per// page simultaneously, which grows without bound on long scanned- // documents. Instead, render at most a raster budget's worth of- // pages under a short-lived PDFium lock, recognize and merge that- // round, release the rasters, and repeat. Each round reopens the- // document; that costs one document load per round, which is small- // next to rendering plus OCR for dozens of pages. The reopen is also- // a new failure point: a path input whose file becomes unreadable+ // documents: at 200 dpi a letter page raster is ~11 MB, so a+ // 500-page scan held ~5.6 GB before the first recognition started.+ // Instead, render one round of rasters under a short-lived PDFium+ // lock, recognize and merge that round, release the rasters, and+ // repeat.+ //+ // A round is `num_workers` *rasters* — not pages. That is the most+ // rasters that can be under recognition at once, since+ // `ocr_and_merge_rendered` gates concurrency on a `num_workers`+ // semaphore, so any raster beyond that count is parked memory rather+ // than throughput. Bounding by rasters instead of by page span is+ // what keeps the engine fed: OCR-needing pages are often sparse in a+ // mostly-native-text document, so a page-span round would hand the+ // engine only the few such pages inside that span and serialize work+ // that could have overlapped. `render_pages_for_ocr` therefore scans+ // ahead — skipping a page costs a complexity check, not a render —+ // until it has a full round.+ //+ // Each round reopens the document, costing one document load per+ // round; with rounds sized by rasters there is exactly one round per+ // `num_workers` pages that actually need OCR, so a document with+ // little OCR work pays at most one reopen. The reopen is also a new+ // failure point: a path input whose file becomes unreadable// mid-parse now fails at OCR time (after extraction succeeded)// rather than never reloading.if let Some(engine) = ocr_engine {- let raster_budget_bytes = self.config.ocr_raster_budget_mb as u64 * 1024 * 1024;+ let round_rasters = self.config.num_workers.max(1);// Extraction may have flattened SOME pages' form widgets into// page content in its (now dropped) document instance; re-apply// per round on exactly those pages so the rasters match what@@ -812,19 +820,14 @@ impl LiteParse {let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);let mut round_start = 0usize;while round_start < pages.len() {- let round_end = ocr_round_end(- &pages,- round_start,- raster_budget_bytes,- self.config.dpi,- ocr_grayscale,- );- let rendered = {+ let (rendered, next_start) = {let lib = Library::init();let document = extract::load_document_from_input(&lib, ocr_input, password)?;ocr_merge::render_pages_for_ocr(&document,- &pages[round_start..round_end],+ &pages,+ round_start,+ round_rasters,self.config.dpi,ocr_grayscale,self.config.render_form_fields,@@ -834,8 +837,16 @@ impl LiteParse {// `lib` drops here, releasing the PDFium lock before the// engine's async recognition below.};+ round_start = next_start;+ if rendered.is_empty() {+ // The scan reached the end without finding another page+ // that needs OCR.+ continue;+ }+ // `RenderedPage::idx` is absolute, so the whole slice is+ // passed regardless of where this round started.ocr_merge::ocr_and_merge_rendered(- &mut pages[round_start..round_end],+ &mut pages,rendered,engine.clone(),&self.config.ocr_language,@@ -843,7 +854,6 @@ impl LiteParse {self.config.ocr_failure_fatal,).await?;- round_start = round_end;}}let t_ocr = web_time::Instant::now();@@ -974,9 +984,6 @@ impl LiteParse {/// LibreOffice/ImageMagick on the system). Plain-text formats cannot be/// rendered and return a clear error.#[cfg(not(target_arch = "wasm32"))]- /// Note: this standalone API does not apply `memory_budget_mb`; all- /// requested pages' PNGs are held in memory. Bound the page selection- /// for very long documents.pub async fn screenshot(&self,input: &str,@@ -1085,36 +1092,6 @@ impl LiteParse {}}-/// End index (exclusive) of the next OCR render round starting at-/// `round_start`: admits pages until their estimated raster bytes would cross-/// `raster_budget_bytes`. Always admits at least one page, so a budget-/// smaller than a single raster still makes progress; a budget of `0` means-/// unbounded (one round covering every remaining page).-fn ocr_round_end(- pages: &[Page],- round_start: usize,- raster_budget_bytes: u64,- dpi: f32,- grayscale: bool,-) -> usize {- let mut round_end = round_start;- let mut estimated_bytes = 0u64;- while round_end < pages.len() {- let page = &pages[round_end];- let page_bytes =- ocr_merge::estimate_ocr_raster_bytes(page.page_width, page.page_height, dpi, grayscale);- if round_end > round_start- && raster_budget_bytes > 0- && estimated_bytes + page_bytes > raster_budget_bytes- {- break;- }- estimated_bytes += page_bytes;- round_end += 1;- }- round_end-}-/// A document opened once and parsed in bounded page batches.pub struct ParseSession {parser: LiteParse,@@ -1423,39 +1400,4 @@ mod tests {"dropping the session should clean up the converted temp PDF");}-- fn blank_page(page_number: usize, width: f32, height: f32) -> Page {- Page {- page_number,- page_width: width,- page_height: height,- content_bounds: None,- text_items: vec![],- graphics: vec![],- vector_graphics: None,- struct_nodes: vec![],- image_refs: vec![],- annotations: None,- form_fields: None,- structure_tree: None,- }- }-- #[test]- fn ocr_round_end_zero_budget_admits_every_page() {- let pages: Vec<Page> = (1..=5).map(|n| blank_page(n, 612.0, 792.0)).collect();- assert_eq!(ocr_round_end(&pages, 0, 0, 150.0, false), 5);- }-- #[test]- fn ocr_round_end_splits_on_budget_and_always_progresses() {- let pages: Vec<Page> = (1..=4).map(|n| blank_page(n, 612.0, 792.0)).collect();- let one_page = ocr_merge::estimate_ocr_raster_bytes(612.0, 792.0, 150.0, false);- // Budget of two pages: rounds of two.- assert_eq!(ocr_round_end(&pages, 0, one_page * 2, 150.0, false), 2);- assert_eq!(ocr_round_end(&pages, 2, one_page * 2, 150.0, false), 4);- // Budget below a single raster still admits one page per round.- assert_eq!(ocr_round_end(&pages, 0, 1, 150.0, false), 1);- assert_eq!(ocr_round_end(&pages, 1, 1, 150.0, false), 2);- }}diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rsindex d88cc98..43cb98d 100644--- a/crates/liteparse/src/render.rs+++ b/crates/liteparse/src/render.rs@@ -47,18 +47,9 @@ pub fn render_pages_to_png(detect_rects,render_form_fields,false,- 0,- 0,)}-/// `memory_budget_bytes` bounds the accumulated PNG payload (`0` disables):-/// when `charged_bytes` (memory the caller already attributed to this budget,-/// e.g. extracted text) plus the running PNG total crosses it, the render-/// fails with [`LiteParseError::MemoryBudget`] instead of holding one PNG per-/// page on an arbitrarily long document. The error reports the caller's-/// configured budget and the true combined total.-#[allow(clippy::too_many_arguments)]pub(crate) fn render_document_pages(document: &pdfium::Document,page_numbers: Option<&[u32]>,@@ -66,8 +57,6 @@ pub(crate) fn render_document_pages(detect_rects: bool,render_form_fields: bool,continue_on_page_error: bool,- memory_budget_bytes: u64,- charged_bytes: u64,) -> Result<Vec<RenderedPage>, LiteParseError> {let page_count = document.page_count() as u32;let pages: Vec<u32> = match page_numbers {@@ -83,7 +72,6 @@ pub(crate) fn render_document_pages(}let mut results = Vec::with_capacity(pages.len());- let mut accumulated_png_bytes = 0u64;for page_num in pages {let page_render = (|| -> Result<RenderedPage, LiteParseError> {if page_num < 1 || page_num > page_count {@@ -126,19 +114,7 @@ pub(crate) fn render_document_pages(})();match page_render {- Ok(rendered) => {- accumulated_png_bytes += rendered.png_bytes.len() as u64;- results.push(rendered);- if memory_budget_bytes > 0- && charged_bytes + accumulated_png_bytes > memory_budget_bytes- {- return Err(LiteParseError::MemoryBudget {- used_mib: (charged_bytes + accumulated_png_bytes) / (1024 * 1024),- budget_mib: memory_budget_bytes / (1024 * 1024),- page_number: page_num,- });- }- }+ Ok(rendered) => results.push(rendered),// The page's text is already extracted; a tolerant parse keeps// it and just forgoes this page's screenshot.Err(error) if continue_on_page_error => eprintln!(diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex 7d9ccdf..a69b348 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -741,20 +741,21 @@ impl liteparse::ocr::OcrEngine for ProbeEngine {}}-/// OCR runs in bounded render→recognize rounds when `ocr_raster_budget_mb`-/// is smaller than the document's rasters. Every page must be recognized-/// exactly once, each page's OCR text must land on the page whose raster-/// produced it (distinct page sizes make a misroute visible), and a budget-/// of one page per round must actually serialize recognition (peak-/// concurrent recognitions = 1), which fails if the round loop is removed.+/// OCR runs in render→recognize rounds of `num_workers` pages. Whatever the+/// round size, every page must be recognized exactly once and each page's OCR+/// text must land on the page whose raster produced it — distinct page sizes+/// make a misroute visible, which is the failure mode the per-round document+/// reopen and form-widget re-flatten could introduce. `num_workers: 1` forces+/// one page per round (four rounds over four pages) and must also serialize+/// recognition; `num_workers: 4` covers the whole document in a single round+/// with overlapping recognition. Both must agree page-for-page.#[tokio::test]#[serial]async fn test_ocr_rounds_cover_every_page_once() {use std::sync::Arc;use std::sync::atomic::{AtomicUsize, Ordering};- // Distinct sizes; at 150 dpi the letter page's raster alone (~6 MB RGB)- // exceeds a 1 MiB budget, so every round holds exactly one page.+ // Distinct sizes so a raster delivered to the wrong page is detectable.let page_sizes: [(u32, u32); 4] = [(612, 792), (400, 500), (300, 300), (500, 900)];let expected_texts: Vec<String> = page_sizes.iter()@@ -768,7 +769,7 @@ async fn test_ocr_rounds_cover_every_page_once() {}).collect();- let run = |budget_mb: usize| {+ let run = |num_workers: usize| {let calls = Arc::new(AtomicUsize::new(0));let peak = Arc::new(AtomicUsize::new(0));let engine = ProbeEngine {@@ -778,8 +779,7 @@ async fn test_ocr_rounds_cover_every_page_once() {};let parser = LiteParse::new(LiteParseConfig {ocr_enabled: true,- ocr_raster_budget_mb: budget_mb,- num_workers: 4,+ num_workers,dpi: 150.0,quiet: true,..Default::default()@@ -788,13 +788,29 @@ async fn test_ocr_rounds_cover_every_page_once() {(parser, calls, peak)};- // Bounded rounds: one page per round, recognition fully serialized.+ let assert_pages_carry_own_rasters = |result: &liteparse::ParseResult, label: &str| {+ assert_eq!(result.pages.len(), page_sizes.len(), "{label}: page count");+ for (page, expected) in result.pages.iter().zip(&expected_texts) {+ let text: String = page+ .text_items+ .iter()+ .map(|item| item.text.as_str())+ .collect();+ assert!(+ text.contains(expected.as_str()),+ "{label}: page {} carries {text:?}, expected {expected:?} — OCR text landed on the wrong page",+ page.page_number+ );+ }+ };++ // One page per round: four rounds, recognition fully serialized.let (parser, calls, peak) = run(1);- let result = parser+ let serialized = parser.parse_input(PdfInput::Bytes(blank_pdf(&page_sizes))).await- .expect("bounded-round OCR parse should succeed");- assert_eq!(result.pages.len(), page_sizes.len());+ .expect("single-page-round OCR parse should succeed");+ assert_pages_carry_own_rasters(&serialized, "rounds of 1");assert_eq!(calls.load(Ordering::SeqCst),page_sizes.len(),@@ -803,32 +819,76 @@ async fn test_ocr_rounds_cover_every_page_once() {assert_eq!(peak.load(Ordering::SeqCst),1,- "a one-page round budget must serialize recognition; >1 means the round loop is not bounding"+ "num_workers=1 must serialize recognition");- for (page, expected) in result.pages.iter().zip(&expected_texts) {- let text: String = page- .text_items- .iter()- .map(|item| item.text.as_str())- .collect();- assert!(- text.contains(expected.as_str()),- "page {} carries {text:?}, expected {expected:?} — OCR text landed on the wrong page",- page.page_number- );- }- // Legacy escape hatch: budget 0 renders everything in one round and- // recognition overlaps up to num_workers.- let (parser, calls, peak) = run(0);- let result = parser+ // Round wide enough for the whole document: one round, overlapping+ // recognition, identical routing.+ let (parser, calls, peak) = run(4);+ let overlapped = parser.parse_input(PdfInput::Bytes(blank_pdf(&page_sizes))).await- .expect("legacy single-round OCR parse should succeed");- assert_eq!(result.pages.len(), page_sizes.len());+ .expect("single-round OCR parse should succeed");+ assert_pages_carry_own_rasters(&overlapped, "rounds of 4");assert_eq!(calls.load(Ordering::SeqCst), page_sizes.len());assert!(peak.load(Ordering::SeqCst) > 1,- "budget 0 must restore the single-round path with overlapping recognition"+ "num_workers=4 over 4 pages must overlap recognition within the round"+ );+}++/// A round is bounded by rasters rendered, not by page span, so OCR-needing+/// pages that are sparsely scattered through a mostly-native-text document+/// still fill a round and recognize concurrently.+///+/// This guards a real regression: bounding the round by page span instead+/// made each round contain only the OCR-needing pages that happened to fall+/// inside its span — often one or two — which starved the worker pool and+/// serialized recognition. On this document that was a 2.4x wall-clock loss+/// at realistic OCR latency, with no test failing.+#[tokio::test]+#[serial]+async fn test_ocr_rounds_fill_across_sparse_pages() {+ use std::sync::Arc;+ use std::sync::atomic::{AtomicUsize, Ordering};++ let calls = Arc::new(AtomicUsize::new(0));+ let peak = Arc::new(AtomicUsize::new(0));+ let parser = LiteParse::new(LiteParseConfig {+ ocr_enabled: true,+ num_workers: 8,+ dpi: 72.0, // Small rasters: this test is about scheduling, not pixels.+ quiet: true,+ ..Default::default()+ })+ .with_ocr_engine(std::sync::Arc::new(ProbeEngine {+ calls: calls.clone(),+ in_flight: Arc::new(AtomicUsize::new(0)),+ peak_in_flight: peak.clone(),+ }));++ let result = parser+ .parse("../../demo/docs/apple-10k-2024.pdf")+ .await+ .expect("should parse the 10-K with OCR enabled");++ let recognized = calls.load(Ordering::SeqCst);+ assert!(+ recognized > 8,+ "expected the fixture to need OCR on more than one round's worth of pages, got {recognized}"+ );+ assert!(+ result.pages.len() > recognized,+ "fixture should be mostly native text, so OCR pages ({recognized}) must be sparse \+ among its {} pages",+ result.pages.len()+ );+ // The scan-ahead must gather a full round even though the OCR-needing+ // pages are interleaved with native-text pages it skips.+ assert_eq!(+ peak.load(Ordering::SeqCst),+ 8,+ "rounds must fill to num_workers across skipped pages; a lower peak means \+ rounds are being cut short by page span");}diff --git a/packages/node/native.d.ts b/packages/node/native.d.tsindex bf3702f..3a6f4dd 100644--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@ -71,7 +71,8 @@ export interface JsLiteParseConfig {/*** Emit each page's classified layout blocks (headings, paragraphs, list* items, tables with per-cell boxes, code, rules, figures) with bounding- * boxes as `ParsedPage.blocks`. Default false.+ * boxes as `ParsedPage.blocks`. Default false. Independent of+ * `outputFormat`; enabling it never changes the rendered markdown.*/extractBlocks?: boolean/** Extract AcroForm widget fields and values. */@@ -143,18 +144,6 @@ export interface JsLiteParseConfig {includeComplexity?: boolean/** Expose page-scoped vector path extraction. Default false. */extractVectorGraphics?: boolean- /**- * Approximate memory budget (MiB) for extracted content accumulated- * during a parse; the parse rejects with a clear error instead of- * growing without bound. 0 disables. Default 4096.- */- memoryBudgetMb?: number- /**- * Approximate budget (MiB) for OCR page rasters held in memory at once;- * pages render and recognize in rounds sized to this budget. 0 renders- * all pages up front. Default 1024.- */- ocrRasterBudgetMb?: number}/*** A page sub-region as the fraction cropped from each side (top-left origin,diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex beedfd4..8ef08de 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -119,19 +119,6 @@ export interface LiteParseConfig {includeComplexity: boolean;/** Expose page-scoped vector shapes and merged H/V line segments. Default false. */extractVectorGraphics: boolean;- /**- * Approximate memory budget (MiB) for extracted content accumulated during- * a parse (text items, word boxes, embedded images, screenshot PNGs). The- * parse rejects with a clear error instead of growing without bound on- * pathological inputs. 0 disables the check. Default 4096.- */- memoryBudgetMb: number;- /**- * Approximate budget (MiB) for OCR page rasters held in memory at once;- * pages render and recognize in bounded rounds sized to this budget.- * 0 renders all pages up front. Default 1024.- */- ocrRasterBudgetMb: number;}/**@@ -659,8 +646,6 @@ export class LiteParse {skipDiagonalText: userConfig.skipDiagonalText,includeComplexity: userConfig.includeComplexity,extractVectorGraphics: userConfig.extractVectorGraphics,- memoryBudgetMb: userConfig.memoryBudgetMb,- ocrRasterBudgetMb: userConfig.ocrRasterBudgetMb,};this._native = new native.LiteParse(nativeConfig);@@ -705,8 +690,6 @@ export class LiteParse {skipDiagonalText: resolved.skipDiagonalText ?? false,includeComplexity: resolved.includeComplexity ?? false,extractVectorGraphics: resolved.extractVectorGraphics ?? false,- memoryBudgetMb: resolved.memoryBudgetMb ?? 4096,- ocrRasterBudgetMb: resolved.ocrRasterBudgetMb ?? 1024,};}diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex b94450a..d247d6d 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -57,8 +57,6 @@ export interface LiteParseNativeConfig {skipDiagonalText?: boolean;includeComplexity?: boolean;extractVectorGraphics?: boolean;- memoryBudgetMb?: number;- ocrRasterBudgetMb?: number;}export interface NativeCropBox {diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex 8fdb9db..f48d566 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -489,8 +489,6 @@ class LiteParse:skip_diagonal_text: Optional[bool] = None,include_complexity: Optional[bool] = None,extract_vector_graphics: Optional[bool] = None,- memory_budget_mb: Optional[int] = None,- ocr_raster_budget_mb: Optional[int] = None,):"""Initialize LiteParse parser.@@ -570,14 +568,6 @@ class LiteParse:pass.extract_vector_graphics: Expose page-scoped vector shapes andmerged horizontal/vertical line segments. Default False.- memory_budget_mb: Approximate memory budget (MiB) for extracted- content accumulated during a parse; the parse fails with a- clear error instead of growing without bound. 0 disables.- Default 4096.- ocr_raster_budget_mb: Approximate budget (MiB) for OCR page- rasters held in memory at once; pages render and recognize- in bounded rounds. 0 renders all pages up front. Default- 1024."""kwargs = {}if ocr_enabled is not None:@@ -654,10 +644,6 @@ class LiteParse:kwargs["include_complexity"] = include_complexityif extract_vector_graphics is not None:kwargs["extract_vector_graphics"] = extract_vector_graphics- if memory_budget_mb is not None:- kwargs["memory_budget_mb"] = memory_budget_mb- if ocr_raster_budget_mb is not None:- kwargs["ocr_raster_budget_mb"] = ocr_raster_budget_mbself._native = _NativeLiteParse(**kwargs)@@ -897,8 +883,6 @@ class LiteParse:extract_document_metadata=cfg.extract_document_metadata,extract_content_bounds=cfg.extract_content_bounds,detect_screenshot_rects=cfg.detect_screenshot_rects,- memory_budget_mb=cfg.memory_budget_mb,- ocr_raster_budget_mb=cfg.ocr_raster_budget_mb,)def __repr__(self) -> str:diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.pyindex ffa48f7..c796c75 100644--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@ -488,14 +488,6 @@ class LiteParseConfig:extract_document_metadata: bool = Falsedetect_screenshot_rects: bool = Falseextract_content_bounds: bool = False- #: Approximate memory budget (MiB) for extracted content accumulated- #: during a parse; the parse raises a clear error instead of growing- #: without bound. 0 disables. Default 4096.- memory_budget_mb: int = 4096- #: Approximate budget (MiB) for OCR page rasters held in memory at once;- #: pages render and recognize in bounded rounds. 0 renders all pages up- #: front. Default 1024.- ocr_raster_budget_mb: int = 1024class ParseError(Exception):