Skip to content

A fast, helpful, and open-source document parser

refactor and simplify

Logan Markewichebf87f91a75f
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs
index 351d076..4d7df3d 100644
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@ -119,14 +119,6 @@ pub struct JsLiteParseConfig {
pub include_complexity: Option<bool>,
/// Expose page-scoped vector path extraction. Default false.
pub extract_vector_graphics: Option<bool>,
- /// Approximate memory budget (MiB) for extracted content accumulated
- /// during a parse; the parse rejects with a clear error instead of
- /// growing without bound. 0 disables. Default 4096.
- pub memory_budget_mb: Option<u32>,
- /// Approximate budget (MiB) for OCR page rasters held in memory at once;
- /// pages render and recognize in rounds sized to this budget. 0 renders
- /// all pages up front. Default 1024.
- pub ocr_raster_budget_mb: Option<u32>,
}
/// A page sub-region as the fraction cropped from each side (top-left origin,
@@ -267,12 +259,6 @@ impl JsLiteParseConfig {
if let Some(v) = self.extract_vector_graphics {
cfg.extract_vector_graphics = v;
}
- if let Some(v) = self.memory_budget_mb {
- cfg.memory_budget_mb = v as usize;
- }
- if let Some(v) = self.ocr_raster_budget_mb {
- cfg.ocr_raster_budget_mb = v as usize;
- }
cfg
}
@@ -337,8 +323,6 @@ impl JsLiteParseConfig {
skip_diagonal_text: Some(cfg.skip_diagonal_text),
include_complexity: Some(cfg.include_complexity),
extract_vector_graphics: Some(cfg.extract_vector_graphics),
- memory_budget_mb: Some(cfg.memory_budget_mb as u32),
- ocr_raster_budget_mb: Some(cfg.ocr_raster_budget_mb as u32),
}
}
}
diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs
index db21b9e..5ad832d 100644
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@ -1268,10 +1268,6 @@ struct PyLiteParseConfig {
extract_images: bool,
#[pyo3(get)]
extract_vector_graphics: bool,
- #[pyo3(get)]
- memory_budget_mb: usize,
- #[pyo3(get)]
- ocr_raster_budget_mb: usize,
}
#[pymethods]
@@ -1339,8 +1335,6 @@ impl PyLiteParseConfig {
image_output_dir: cfg.image_output_dir.clone(),
extract_images: cfg.extract_images,
extract_vector_graphics: cfg.extract_vector_graphics,
- memory_budget_mb: cfg.memory_budget_mb,
- ocr_raster_budget_mb: cfg.ocr_raster_budget_mb,
}
}
}
@@ -1468,8 +1462,6 @@ impl LiteParse {
skip_diagonal_text = None,
include_complexity = None,
extract_vector_graphics = None,
- memory_budget_mb = None,
- ocr_raster_budget_mb = None,
))]
fn new(
ocr_language: Option<String>,
@@ -1509,8 +1501,6 @@ impl LiteParse {
skip_diagonal_text: Option<bool>,
include_complexity: Option<bool>,
extract_vector_graphics: Option<bool>,
- memory_budget_mb: Option<usize>,
- ocr_raster_budget_mb: Option<usize>,
) -> PyResult<Self> {
let mut cfg = LiteParseConfig::default();
if let Some(v) = ocr_language {
@@ -1637,12 +1627,6 @@ impl LiteParse {
if let Some(v) = extract_vector_graphics {
cfg.extract_vector_graphics = v;
}
- if let Some(v) = memory_budget_mb {
- cfg.memory_budget_mb = v;
- }
- if let Some(v) = ocr_raster_budget_mb {
- cfg.ocr_raster_budget_mb = v;
- }
let inner = liteparse::parser::LiteParse::new(cfg.clone());
let runtime = std::sync::Arc::new(
diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs
index 527bbea..1341a02 100644
--- a/crates/liteparse-wasm/src/lib.rs
+++ b/crates/liteparse-wasm/src/lib.rs
@@ -48,13 +48,6 @@ pub struct LiteParseConfig {
ocr_server_headers: Option<HashMap<String, String>>,
tessdata_path: Option<String>,
max_pages: Option<usize>,
- /// Approximate memory budget (MiB) for extracted content accumulated
- /// during a parse; the parse rejects with a clear error instead of
- /// growing without bound. 0 disables. Default 4096.
- memory_budget_mb: Option<usize>,
- /// Approximate budget (MiB) for OCR page rasters held in memory at once.
- /// 0 renders all pages up front. Default 1024.
- ocr_raster_budget_mb: Option<usize>,
target_pages: Option<String>,
/// Skip page-level PDF extraction failures and report them in
/// `ParseResult.pageErrors`. Document-level failures remain fatal.
@@ -151,12 +144,6 @@ impl LiteParseConfig {
if let Some(v) = self.max_pages {
cfg.max_pages = v;
}
- if let Some(v) = self.memory_budget_mb {
- cfg.memory_budget_mb = v;
- }
- if let Some(v) = self.ocr_raster_budget_mb {
- cfg.ocr_raster_budget_mb = v;
- }
if self.target_pages.is_some() {
cfg.target_pages = self.target_pages;
}
@@ -279,8 +266,6 @@ impl LiteParseConfig {
},
tessdata_path: cfg.tessdata_path.clone(),
max_pages: Some(cfg.max_pages),
- memory_budget_mb: Some(cfg.memory_budget_mb),
- ocr_raster_budget_mb: Some(cfg.ocr_raster_budget_mb),
target_pages: cfg.target_pages.clone(),
continue_on_page_error: Some(cfg.continue_on_page_error),
extract_screenshots: Some(cfg.extract_screenshots),
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs
index 79b62db..ff3a8f7 100644
--- a/crates/liteparse/src/config.rs
+++ b/crates/liteparse/src/config.rs
@@ -163,47 +163,6 @@ pub struct LiteParseConfig {
/// vertical `lines`) in parse results. Default `false`; path objects are
/// still inspected internally for layout detection when disabled.
pub extract_vector_graphics: bool,
- /// Approximate budget, in MiB, for extracted content accumulated in
- /// memory during a parse (text items, word boxes, embedded image bytes,
- /// screenshot PNGs). When the running estimate crosses the budget the
- /// parse fails with [`crate::LiteParseError::MemoryBudget`] instead of
- /// growing without bound on pathological inputs. `0` disables the check.
- ///
- /// The estimate is a lower bound on real usage (it counts payload bytes
- /// and struct sizes, not allocator or formatting overhead), so treat the
- /// budget as "reject clearly-oversized documents", not as a precise cap.
- /// Not counted: vector-graphics paths, structure-tree nodes, annotations
- /// and form-field records — documents dominated by those can exceed the
- /// budget's view of memory. The standalone `screenshot()` API is exempt.
- /// In batch parsing the budget applies per batch, not across the whole
- /// session — the caller controls cross-batch accumulation.
- /// Default 4096 MiB. For large documents prefer narrowing
- /// `target_pages`/`max_pages` or batch parsing over raising the budget.
- #[serde(default = "default_memory_budget_mb")]
- pub memory_budget_mb: usize,
- /// Approximate budget, in MiB, for OCR page rasters held in memory at
- /// once. Pages are rendered and OCR'd in bounded rounds sized to this
- /// budget, instead of pre-rendering every text-poor page up front — on a
- /// long scanned document the up-front strategy holds every raster
- /// simultaneously. `0` restores the unbounded single-round behavior.
- /// Default 1024 MiB; at least one page is always rendered per round.
- ///
- /// Rounds run serially, so effective OCR concurrency is
- /// `min(num_workers, pages per round)` — a budget small enough to admit
- /// fewer pages than `num_workers` reduces OCR throughput. The default
- /// admits far more pages per round than any realistic worker count.
- /// Each round also reopens the document once (one document load per
- /// round; small next to rendering plus recognition for a round's pages).
- #[serde(default = "default_ocr_raster_budget_mb")]
- pub ocr_raster_budget_mb: usize,
-}
-
-fn default_memory_budget_mb() -> usize {
- 4096
-}
-
-fn default_ocr_raster_budget_mb() -> usize {
- 1024
}
/// A page sub-region expressed as the fraction cropped from each side.
@@ -245,11 +204,6 @@ pub enum OutputFormat {
}
impl LiteParseConfig {
- /// The extracted-content memory budget in bytes; `0` means disabled.
- pub fn memory_budget_bytes(&self) -> u64 {
- self.memory_budget_mb as u64 * 1024 * 1024
- }
-
/// Whether embedded-image extraction should run. True when
/// `extract_images` is set, or when the legacy `ImageMode::Embed` is
/// configured (`Embed` implies byte extraction for backwards
@@ -309,8 +263,6 @@ impl Default for LiteParseConfig {
skip_diagonal_text: false,
include_complexity: false,
extract_vector_graphics: false,
- memory_budget_mb: default_memory_budget_mb(),
- ocr_raster_budget_mb: default_ocr_raster_budget_mb(),
}
}
}
diff --git a/crates/liteparse/src/error.rs b/crates/liteparse/src/error.rs
index d918dbb..bb99e54 100644
--- a/crates/liteparse/src/error.rs
+++ b/crates/liteparse/src/error.rs
@@ -1,7 +1,6 @@
use thiserror::Error;
#[derive(Debug, Error)]
-#[non_exhaustive]
pub enum LiteParseError {
#[error("PDF error: {0}")]
Pdf(#[from] pdfium::PdfiumError),
@@ -24,15 +23,6 @@ pub enum LiteParseError {
#[error("invalid config: {0}")]
Config(String),
- #[error(
- "memory budget exceeded: extracted content reached ~{used_mib} MiB (budget {budget_mib} MiB) at page {page_number}; raise `memory_budget_mb` (0 disables the check), narrow `target_pages`/`max_pages`, or parse in batches"
- )]
- MemoryBudget {
- used_mib: u64,
- budget_mib: u64,
- page_number: u32,
- },
-
#[error("{0}")]
Other(String),
}
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index da90966..693756f 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -81,10 +81,6 @@ pub(crate) struct ExtractedPages {
/// pages — flattening a page extraction never touched hides that page's
/// non-widget annotations from the raster.
pub flattened_page_numbers: Vec<u32>,
- /// Running estimate of the bytes held by `pages` + `images`, as counted
- /// against `ExtractionOutputOptions::memory_budget_bytes`. Lets callers
- /// charge later allocations (e.g. screenshot PNGs) to the same budget.
- pub approx_extracted_bytes: u64,
}
/// Same as `extract_pages_from_document` but optionally also renders every
@@ -107,7 +103,6 @@ pub(crate) fn extract_pages_and_images(
let mut image_error_count = 0u32;
let mut flattened_form_widgets = false;
let mut flattened_page_numbers: Vec<u32> = Vec::new();
- let mut approx_extracted_bytes = 0u64;
// One FFI call keeps the per-page annotation walk off the hot path for
// every document without an AcroForm catalog, which is nearly all of them.
let document_has_form = document.form_type() != 0;
@@ -148,10 +143,6 @@ pub(crate) fn extract_pages_and_images(
&mut page_errors,
)? {
Some(extraction) => {
- approx_extracted_bytes += approx_page_bytes(&extraction.page);
- for image in &extraction.images {
- approx_extracted_bytes += approx_image_bytes(image);
- }
if extraction.flattened_form_widgets {
flattened_page_numbers.push(page_number);
}
@@ -159,15 +150,6 @@ pub(crate) fn extract_pages_and_images(
images.extend(extraction.images);
image_error_count += extraction.image_error_count;
flattened_form_widgets |= extraction.flattened_form_widgets;
- if output_options.memory_budget_bytes > 0
- && approx_extracted_bytes > output_options.memory_budget_bytes
- {
- return Err(LiteParseError::MemoryBudget {
- used_mib: approx_extracted_bytes / (1024 * 1024),
- budget_mib: output_options.memory_budget_bytes / (1024 * 1024),
- page_number,
- });
- }
}
// A failed page's cached renders must not seed dedup for later
// pages: a hit would emit `duplicate_of` pointing at an image id
@@ -183,7 +165,6 @@ pub(crate) fn extract_pages_and_images(
image_error_count,
flattened_form_widgets,
flattened_page_numbers,
- approx_extracted_bytes,
})
}
@@ -468,52 +449,6 @@ pub(crate) struct ExtractionOutputOptions {
pub extract_form_fields: bool,
pub extract_structure_tree: bool,
pub emit_word_boxes: bool,
- /// Approximate byte budget for accumulated pages + images; `0` disables
- /// the check. See `LiteParseConfig::memory_budget_mb`.
- pub memory_budget_bytes: u64,
-}
-
-/// Approximate resident bytes of one extracted page: struct sizes plus the
-/// owned payloads that dominate on text-dense documents (item text, per-word
-/// boxes, char codes, string fields). Deliberately a lower bound — allocator
-/// slack and later formatting copies are not counted — so the budget check
-/// errs toward admitting.
-pub(crate) fn approx_page_bytes(page: &LitePage) -> u64 {
- let mut bytes = std::mem::size_of::<LitePage>() as u64;
- for item in &page.text_items {
- bytes += std::mem::size_of::<TextItem>() as u64;
- bytes += item.text.len() as u64;
- bytes += (item.char_codes.len() * std::mem::size_of::<u32>()) as u64;
- bytes += item.font_name.as_deref().map_or(0, str::len) as u64;
- bytes += item.fill_color.as_deref().map_or(0, str::len) as u64;
- bytes += item.stroke_color.as_deref().map_or(0, str::len) as u64;
- bytes += item.link.as_deref().map_or(0, str::len) as u64;
- for word in &item.words {
- bytes += std::mem::size_of::<WordBox>() as u64 + word.text.len() as u64;
- }
- }
- bytes += (page.graphics.len() * std::mem::size_of::<GraphicPrimitive>()) as u64;
- if let Some(vector_graphics) = &page.vector_graphics {
- bytes += (vector_graphics.shapes.len() * std::mem::size_of::<VectorShape>()) as u64;
- bytes += (vector_graphics.lines.len() * std::mem::size_of::<VectorLine>()) as u64;
- }
- bytes += (page.struct_nodes.len() * std::mem::size_of::<StructNode>()) as u64;
- bytes
-}
-
-/// Approximate resident bytes of one extracted image. Duplicate entries share
-/// the canonical image's `Arc` buffer, so only the first occurrence pays for
-/// the pixel bytes.
-fn approx_image_bytes(image: &ExtractedImage) -> u64 {
- let payload = if image.duplicate_of.is_none() {
- image.bytes.len() as u64
- } else {
- 0
- };
- std::mem::size_of::<ExtractedImage>() as u64
- + payload
- + image.id.len() as u64
- + image.name.len() as u64
}
fn document_annotation(annotation: &pdfium::PdfAnnotation) -> DocumentAnnotation {
@@ -3740,60 +3675,4 @@ mod tests {
Some("p3_1")
);
}
-
- #[test]
- fn memory_budget_rejects_oversized_extraction() {
- let bytes = rotated_text_pdf();
-
- // The document borrows the input buffer, so the input must outlive it.
- let input = PdfInput::Bytes(bytes);
-
- // A budget of one byte trips on the first extracted page.
- let error = {
- let lib = Library::init();
- let document = load_document_from_input(&lib, &input, None).unwrap();
- match extract_pages_and_images(
- &document,
- None,
- usize::MAX,
- false,
- None,
- ExtractionOutputOptions {
- memory_budget_bytes: 1,
- ..Default::default()
- },
- ) {
- Ok(_) => panic!("expected the one-byte budget to reject the extraction"),
- Err(error) => error,
- }
- };
- match error {
- LiteParseError::MemoryBudget {
- budget_mib,
- page_number,
- ..
- } => {
- assert_eq!(budget_mib, 0); // one byte rounds down to 0 MiB
- assert_eq!(page_number, 1);
- }
- other => panic!("expected MemoryBudget, got: {other}"),
- }
-
- // Budget 0 disables the check, and the running estimate is reported.
- let lib = Library::init();
- let document = load_document_from_input(&lib, &input, None).unwrap();
- let extracted = extract_pages_and_images(
- &document,
- None,
- usize::MAX,
- false,
- None,
- ExtractionOutputOptions::default(),
- )
- .unwrap();
- assert_eq!(extracted.pages.len(), 2);
- assert!(extracted.approx_extracted_bytes > 0);
- let per_page_sum: u64 = extracted.pages.iter().map(approx_page_bytes).sum();
- assert_eq!(extracted.approx_extracted_bytes, per_page_sum);
- }
}
diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs
index 080ad7a..b0dd380 100644
--- a/crates/liteparse/src/main.rs
+++ b/crates/liteparse/src/main.rs
@@ -75,18 +75,6 @@ struct ParseCommand {
#[arg(long, default_value = "1000")]
max_pages: usize,
- /// Approximate memory budget (MiB) for extracted content held in memory;
- /// the parse fails with a clear error instead of growing without bound.
- /// 0 disables the check.
- #[arg(long, default_value = "4096")]
- memory_budget_mb: usize,
-
- /// Approximate budget (MiB) for OCR page rasters held in memory at once;
- /// pages are rendered and recognized in rounds sized to this budget.
- /// 0 renders all pages up front.
- #[arg(long, default_value = "1024")]
- ocr_raster_budget_mb: usize,
-
/// Target pages (e.g., "1-5,10,15-20")
#[arg(long)]
target_pages: Option<String>,
@@ -247,18 +235,6 @@ struct BatchParseCommand {
#[arg(long, default_value = "1000")]
max_pages: usize,
- /// Approximate memory budget (MiB) for extracted content held in memory;
- /// the parse fails with a clear error instead of growing without bound.
- /// 0 disables the check.
- #[arg(long, default_value = "4096")]
- memory_budget_mb: usize,
-
- /// Approximate budget (MiB) for OCR page rasters held in memory at once;
- /// pages are rendered and recognized in rounds sized to this budget.
- /// 0 renders all pages up front.
- #[arg(long, default_value = "1024")]
- ocr_raster_budget_mb: usize,
-
/// Continue after page-level extraction errors and report them in JSON.
#[arg(long)]
continue_on_page_error: bool,
@@ -470,8 +446,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
include_complexity: cmd.complexity,
extract_text_metadata: cmd.extract_text_metadata,
extract_vector_graphics: cmd.extract_vector_graphics,
- memory_budget_mb: cmd.memory_budget_mb,
- ocr_raster_budget_mb: cmd.ocr_raster_budget_mb,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -574,8 +548,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
extract_blocks: cmd.extract_blocks,
extract_xfa_packets: cmd.extract_xfa_packets,
extract_content_bounds: cmd.extract_content_bounds,
- memory_budget_mb: cmd.memory_budget_mb,
- ocr_raster_budget_mb: cmd.ocr_raster_budget_mb,
..Default::default()
};
if let Some(n) = cmd.num_workers {
@@ -879,48 +851,6 @@ mod tests {
));
}
- #[test]
- fn memory_budget_flags_are_available_for_parse_and_batch() {
- let cli = Cli::try_parse_from([
- "lit",
- "parse",
- "document.pdf",
- "--memory-budget-mb",
- "512",
- "--ocr-raster-budget-mb",
- "0",
- ])
- .unwrap();
- assert!(matches!(
- cli.command,
- Commands::Parse(ParseCommand {
- memory_budget_mb: 512,
- ocr_raster_budget_mb: 0,
- ..
- })
- ));
-
- let cli = Cli::try_parse_from([
- "lit",
- "batch-parse",
- "input",
- "output",
- "--memory-budget-mb",
- "512",
- "--ocr-raster-budget-mb",
- "0",
- ])
- .unwrap();
- assert!(matches!(
- cli.command,
- Commands::BatchParse(BatchParseCommand {
- memory_budget_mb: 512,
- ocr_raster_budget_mb: 0,
- ..
- })
- ));
- }
-
#[test]
fn extract_structure_tree_flag_is_available_for_parse_and_batch() {
let cli = Cli::try_parse_from(["lit", "parse", "document.pdf", "--extract-structure-tree"])
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index 1873ccf..1df6962 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -434,39 +434,33 @@ fn count_columns(region: &Region, total_items: usize) -> usize {
/// to avoid excessive memory usage.
pub(crate) const MAX_OCR_RENDER_LONG_EDGE_PX: f32 = 4096.0;
-/// Upper-bound estimate of one page's OCR raster output bytes at `dpi`,
-/// mirroring the long-edge DPI clamp applied in [`render_pages_for_ocr`].
-/// Used to size bounded render→recognize rounds before anything is rendered.
-/// Models the retained output buffer only; the render itself transiently
-/// holds one additional 4-bytes-per-pixel bitmap for the page in flight.
-pub(crate) fn estimate_ocr_raster_bytes(
- page_width_pt: f32,
- page_height_pt: f32,
- dpi: f32,
- grayscale: bool,
-) -> u64 {
- let long_edge_pt = page_width_pt.max(page_height_pt);
- let mut eff_dpi = dpi;
- if long_edge_pt > 0.0 {
- let max_dpi = MAX_OCR_RENDER_LONG_EDGE_PX * 72.0 / long_edge_pt;
- if eff_dpi > max_dpi {
- eff_dpi = max_dpi;
- }
- }
- let width_px = (page_width_pt / 72.0 * eff_dpi).max(1.0) as u64;
- let height_px = (page_height_pt / 72.0 * eff_dpi).max(1.0) as u64;
- width_px * height_px * if grayscale { 1 } else { 3 }
-}
-
+/// Render the pages in `pages[start..]` that need OCR, stopping once
+/// `max_rasters` of them have been rendered (`0` means no limit). Returns the
+/// rasters plus the index to resume scanning from, so a caller can process a
+/// long document in bounded rounds.
+///
+/// The bound is on rasters produced, not pages scanned, because pages that
+/// need OCR can be sparse: bounding a round by page span would hand the OCR
+/// engine only the OCR-needing pages that happen to fall in that span, which
+/// on a mostly-native-text document starves the engine's worker pool and
+/// serializes recognition that could have run concurrently. Skipping a page
+/// costs a complexity check, not a render, so scanning ahead to fill a round
+/// is cheap.
+///
+/// `RenderedPage::idx` indexes `pages` absolutely, so the caller merges
+/// against the whole slice regardless of where the round started.
+#[allow(clippy::too_many_arguments)]
pub(crate) fn render_pages_for_ocr(
document: &Document,
pages: &[Page],
+ start: usize,
+ max_rasters: usize,
dpi: f32,
grayscale: bool,
render_form_fields: bool,
continue_on_page_error: bool,
flatten_page_numbers: &std::collections::HashSet<u32>,
-) -> Result<Vec<RenderedPage>, LiteParseError> {
+) -> Result<(Vec<RenderedPage>, usize), LiteParseError> {
let mut rendered = Vec::new();
// With `render_form_fields`, draw form-field appearances into the OCR
// raster so filled-in form values are visible to the OCR engine (matches
@@ -478,7 +472,8 @@ pub(crate) fn render_pages_for_ocr(
if let Some(form) = form.as_ref() {
form.run_document_actions();
}
- for (idx, page) in pages.iter().enumerate() {
+ let mut next_start = pages.len();
+ for (idx, page) in pages.iter().enumerate().skip(start) {
let page_render = (|| -> Result<Option<RenderedPage>, LiteParseError> {
let page_index = (page.page_number - 1) as i32;
// Text extraction may have flattened THIS page's widget
@@ -531,7 +526,13 @@ pub(crate) fn render_pages_for_ocr(
}))
})();
match page_render {
- Ok(Some(render)) => rendered.push(render),
+ Ok(Some(render)) => {
+ rendered.push(render);
+ if max_rasters > 0 && rendered.len() >= max_rasters {
+ next_start = idx + 1;
+ break;
+ }
+ }
Ok(None) => {}
// The page already extracted successfully, so a tolerant parse
// keeps its native text and only forgoes the OCR enrichment.
@@ -542,7 +543,7 @@ pub(crate) fn render_pages_for_ocr(
Err(error) => return Err(error),
}
}
- Ok(rendered)
+ Ok((rendered, next_start))
}
/// Run OCR on pre-rendered page bitmaps and merge results into `pages`.
@@ -1079,30 +1080,6 @@ mod tests {
use super::*;
use crate::types::Rect;
- #[test]
- fn estimate_ocr_raster_bytes_scales_and_clamps() {
- // Letter page at 150 dpi: 1275 x 1650 px; RGB is three bytes a pixel,
- // grayscale one.
- assert_eq!(
- estimate_ocr_raster_bytes(612.0, 792.0, 150.0, false),
- 1275 * 1650 * 3
- );
- assert_eq!(
- estimate_ocr_raster_bytes(612.0, 792.0, 150.0, true),
- 1275 * 1650
- );
- // A large-format page is clamped to the long-edge pixel cap no matter
- // how high the requested dpi is.
- let clamped = estimate_ocr_raster_bytes(2592.0, 1728.0, 600.0, false);
- let cap = MAX_OCR_RENDER_LONG_EDGE_PX as u64;
- assert!(clamped <= cap * cap * 3);
- assert_eq!(
- clamped,
- estimate_ocr_raster_bytes(2592.0, 1728.0, 10_000.0, false),
- "beyond the clamp, higher dpi must not grow the estimate"
- );
- }
-
fn leaf(n: usize) -> Region {
Region {
bbox: Rect::default(),
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index cf303ae..b1feef3 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -381,7 +381,6 @@ impl LiteParse {
self.glyph_resolver.as_deref(),
extract::ExtractionOutputOptions {
continue_on_page_error: self.config.continue_on_page_error,
- memory_budget_bytes: self.config.memory_budget_bytes(),
..Default::default()
},
)?
@@ -668,7 +667,6 @@ impl LiteParse {
extract_annotations: self.config.extract_annotations,
extract_form_fields: self.config.extract_form_fields,
extract_structure_tree: self.config.extract_structure_tree,
- memory_budget_bytes: self.config.memory_budget_bytes(),
},
)?;
let extract::ExtractedPages {
@@ -678,7 +676,6 @@ impl LiteParse {
image_error_count,
flattened_form_widgets,
flattened_page_numbers,
- approx_extracted_bytes,
} = extracted;
// Reopening the input costs a full parse, so it is confined to the
// one consumer that genuinely needs live widget annotations: the
@@ -733,10 +730,6 @@ impl LiteParse {
.iter()
.map(|page| page.page_number as u32)
.collect::<Vec<_>>();
- // Screenshot PNGs accumulate alongside the extracted pages,
- // so they draw down the same memory budget; the extraction
- // bytes already spent are passed as pre-charged so the error
- // reports the configured budget and the true running total.
render::render_document_pages(
analysis_document,
Some(&page_numbers),
@@ -744,8 +737,6 @@ impl LiteParse {
self.config.detect_screenshot_rects,
self.config.render_form_fields,
self.config.continue_on_page_error,
- self.config.memory_budget_bytes(),
- approx_extracted_bytes,
)?
.into_iter()
.map(|page| ScreenshotResult {
@@ -788,16 +779,33 @@ impl LiteParse {
// OCR pass, in bounded render→recognize rounds. Rendering every
// text-poor page before recognizing any of them holds one raster per
// page simultaneously, which grows without bound on long scanned
- // documents. Instead, render at most a raster budget's worth of
- // pages under a short-lived PDFium lock, recognize and merge that
- // round, release the rasters, and repeat. Each round reopens the
- // document; that costs one document load per round, which is small
- // next to rendering plus OCR for dozens of pages. The reopen is also
- // a new failure point: a path input whose file becomes unreadable
+ // documents: at 200 dpi a letter page raster is ~11 MB, so a
+ // 500-page scan held ~5.6 GB before the first recognition started.
+ // Instead, render one round of rasters under a short-lived PDFium
+ // lock, recognize and merge that round, release the rasters, and
+ // repeat.
+ //
+ // A round is `num_workers` *rasters* — not pages. That is the most
+ // rasters that can be under recognition at once, since
+ // `ocr_and_merge_rendered` gates concurrency on a `num_workers`
+ // semaphore, so any raster beyond that count is parked memory rather
+ // than throughput. Bounding by rasters instead of by page span is
+ // what keeps the engine fed: OCR-needing pages are often sparse in a
+ // mostly-native-text document, so a page-span round would hand the
+ // engine only the few such pages inside that span and serialize work
+ // that could have overlapped. `render_pages_for_ocr` therefore scans
+ // ahead — skipping a page costs a complexity check, not a render —
+ // until it has a full round.
+ //
+ // Each round reopens the document, costing one document load per
+ // round; with rounds sized by rasters there is exactly one round per
+ // `num_workers` pages that actually need OCR, so a document with
+ // little OCR work pays at most one reopen. The reopen is also a new
+ // failure point: a path input whose file becomes unreadable
// mid-parse now fails at OCR time (after extraction succeeded)
// rather than never reloading.
if let Some(engine) = ocr_engine {
- let raster_budget_bytes = self.config.ocr_raster_budget_mb as u64 * 1024 * 1024;
+ let round_rasters = self.config.num_workers.max(1);
// Extraction may have flattened SOME pages' form widgets into
// page content in its (now dropped) document instance; re-apply
// per round on exactly those pages so the rasters match what
@@ -812,19 +820,14 @@ impl LiteParse {
let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);
let mut round_start = 0usize;
while round_start < pages.len() {
- let round_end = ocr_round_end(
- &pages,
- round_start,
- raster_budget_bytes,
- self.config.dpi,
- ocr_grayscale,
- );
- let rendered = {
+ let (rendered, next_start) = {
let lib = Library::init();
let document = extract::load_document_from_input(&lib, ocr_input, password)?;
ocr_merge::render_pages_for_ocr(
&document,
- &pages[round_start..round_end],
+ &pages,
+ round_start,
+ round_rasters,
self.config.dpi,
ocr_grayscale,
self.config.render_form_fields,
@@ -834,8 +837,16 @@ impl LiteParse {
// `lib` drops here, releasing the PDFium lock before the
// engine's async recognition below.
};
+ round_start = next_start;
+ if rendered.is_empty() {
+ // The scan reached the end without finding another page
+ // that needs OCR.
+ continue;
+ }
+ // `RenderedPage::idx` is absolute, so the whole slice is
+ // passed regardless of where this round started.
ocr_merge::ocr_and_merge_rendered(
- &mut pages[round_start..round_end],
+ &mut pages,
rendered,
engine.clone(),
&self.config.ocr_language,
@@ -843,7 +854,6 @@ impl LiteParse {
self.config.ocr_failure_fatal,
)
.await?;
- round_start = round_end;
}
}
let t_ocr = web_time::Instant::now();
@@ -974,9 +984,6 @@ impl LiteParse {
/// LibreOffice/ImageMagick on the system). Plain-text formats cannot be
/// rendered and return a clear error.
#[cfg(not(target_arch = "wasm32"))]
- /// Note: this standalone API does not apply `memory_budget_mb`; all
- /// requested pages' PNGs are held in memory. Bound the page selection
- /// for very long documents.
pub async fn screenshot(
&self,
input: &str,
@@ -1085,36 +1092,6 @@ impl LiteParse {
}
}
-/// End index (exclusive) of the next OCR render round starting at
-/// `round_start`: admits pages until their estimated raster bytes would cross
-/// `raster_budget_bytes`. Always admits at least one page, so a budget
-/// smaller than a single raster still makes progress; a budget of `0` means
-/// unbounded (one round covering every remaining page).
-fn ocr_round_end(
- pages: &[Page],
- round_start: usize,
- raster_budget_bytes: u64,
- dpi: f32,
- grayscale: bool,
-) -> usize {
- let mut round_end = round_start;
- let mut estimated_bytes = 0u64;
- while round_end < pages.len() {
- let page = &pages[round_end];
- let page_bytes =
- ocr_merge::estimate_ocr_raster_bytes(page.page_width, page.page_height, dpi, grayscale);
- if round_end > round_start
- && raster_budget_bytes > 0
- && estimated_bytes + page_bytes > raster_budget_bytes
- {
- break;
- }
- estimated_bytes += page_bytes;
- round_end += 1;
- }
- round_end
-}
-
/// A document opened once and parsed in bounded page batches.
pub struct ParseSession {
parser: LiteParse,
@@ -1423,39 +1400,4 @@ mod tests {
"dropping the session should clean up the converted temp PDF"
);
}
-
- fn blank_page(page_number: usize, width: f32, height: f32) -> Page {
- Page {
- page_number,
- page_width: width,
- page_height: height,
- content_bounds: None,
- text_items: vec![],
- graphics: vec![],
- vector_graphics: None,
- struct_nodes: vec![],
- image_refs: vec![],
- annotations: None,
- form_fields: None,
- structure_tree: None,
- }
- }
-
- #[test]
- fn ocr_round_end_zero_budget_admits_every_page() {
- let pages: Vec<Page> = (1..=5).map(|n| blank_page(n, 612.0, 792.0)).collect();
- assert_eq!(ocr_round_end(&pages, 0, 0, 150.0, false), 5);
- }
-
- #[test]
- fn ocr_round_end_splits_on_budget_and_always_progresses() {
- let pages: Vec<Page> = (1..=4).map(|n| blank_page(n, 612.0, 792.0)).collect();
- let one_page = ocr_merge::estimate_ocr_raster_bytes(612.0, 792.0, 150.0, false);
- // Budget of two pages: rounds of two.
- assert_eq!(ocr_round_end(&pages, 0, one_page * 2, 150.0, false), 2);
- assert_eq!(ocr_round_end(&pages, 2, one_page * 2, 150.0, false), 4);
- // Budget below a single raster still admits one page per round.
- assert_eq!(ocr_round_end(&pages, 0, 1, 150.0, false), 1);
- assert_eq!(ocr_round_end(&pages, 1, 1, 150.0, false), 2);
- }
}
diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rs
index d88cc98..43cb98d 100644
--- a/crates/liteparse/src/render.rs
+++ b/crates/liteparse/src/render.rs
@@ -47,18 +47,9 @@ pub fn render_pages_to_png(
detect_rects,
render_form_fields,
false,
- 0,
- 0,
)
}
-/// `memory_budget_bytes` bounds the accumulated PNG payload (`0` disables):
-/// when `charged_bytes` (memory the caller already attributed to this budget,
-/// e.g. extracted text) plus the running PNG total crosses it, the render
-/// fails with [`LiteParseError::MemoryBudget`] instead of holding one PNG per
-/// page on an arbitrarily long document. The error reports the caller's
-/// configured budget and the true combined total.
-#[allow(clippy::too_many_arguments)]
pub(crate) fn render_document_pages(
document: &pdfium::Document,
page_numbers: Option<&[u32]>,
@@ -66,8 +57,6 @@ pub(crate) fn render_document_pages(
detect_rects: bool,
render_form_fields: bool,
continue_on_page_error: bool,
- memory_budget_bytes: u64,
- charged_bytes: u64,
) -> Result<Vec<RenderedPage>, LiteParseError> {
let page_count = document.page_count() as u32;
let pages: Vec<u32> = match page_numbers {
@@ -83,7 +72,6 @@ pub(crate) fn render_document_pages(
}
let mut results = Vec::with_capacity(pages.len());
- let mut accumulated_png_bytes = 0u64;
for page_num in pages {
let page_render = (|| -> Result<RenderedPage, LiteParseError> {
if page_num < 1 || page_num > page_count {
@@ -126,19 +114,7 @@ pub(crate) fn render_document_pages(
})();
match page_render {
- Ok(rendered) => {
- accumulated_png_bytes += rendered.png_bytes.len() as u64;
- results.push(rendered);
- if memory_budget_bytes > 0
- && charged_bytes + accumulated_png_bytes > memory_budget_bytes
- {
- return Err(LiteParseError::MemoryBudget {
- used_mib: (charged_bytes + accumulated_png_bytes) / (1024 * 1024),
- budget_mib: memory_budget_bytes / (1024 * 1024),
- page_number: page_num,
- });
- }
- }
+ Ok(rendered) => results.push(rendered),
// The page's text is already extracted; a tolerant parse keeps
// it and just forgoes this page's screenshot.
Err(error) if continue_on_page_error => eprintln!(
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
index 7d9ccdf..a69b348 100644
--- a/crates/liteparse/tests/integration_test.rs
+++ b/crates/liteparse/tests/integration_test.rs
@@ -741,20 +741,21 @@ impl liteparse::ocr::OcrEngine for ProbeEngine {
}
}
-/// OCR runs in bounded render→recognize rounds when `ocr_raster_budget_mb`
-/// is smaller than the document's rasters. Every page must be recognized
-/// exactly once, each page's OCR text must land on the page whose raster
-/// produced it (distinct page sizes make a misroute visible), and a budget
-/// of one page per round must actually serialize recognition (peak
-/// concurrent recognitions = 1), which fails if the round loop is removed.
+/// OCR runs in render→recognize rounds of `num_workers` pages. Whatever the
+/// round size, every page must be recognized exactly once and each page's OCR
+/// text must land on the page whose raster produced it — distinct page sizes
+/// make a misroute visible, which is the failure mode the per-round document
+/// reopen and form-widget re-flatten could introduce. `num_workers: 1` forces
+/// one page per round (four rounds over four pages) and must also serialize
+/// recognition; `num_workers: 4` covers the whole document in a single round
+/// with overlapping recognition. Both must agree page-for-page.
#[tokio::test]
#[serial]
async fn test_ocr_rounds_cover_every_page_once() {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
- // Distinct sizes; at 150 dpi the letter page's raster alone (~6 MB RGB)
- // exceeds a 1 MiB budget, so every round holds exactly one page.
+ // Distinct sizes so a raster delivered to the wrong page is detectable.
let page_sizes: [(u32, u32); 4] = [(612, 792), (400, 500), (300, 300), (500, 900)];
let expected_texts: Vec<String> = page_sizes
.iter()
@@ -768,7 +769,7 @@ async fn test_ocr_rounds_cover_every_page_once() {
})
.collect();
- let run = |budget_mb: usize| {
+ let run = |num_workers: usize| {
let calls = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let engine = ProbeEngine {
@@ -778,8 +779,7 @@ async fn test_ocr_rounds_cover_every_page_once() {
};
let parser = LiteParse::new(LiteParseConfig {
ocr_enabled: true,
- ocr_raster_budget_mb: budget_mb,
- num_workers: 4,
+ num_workers,
dpi: 150.0,
quiet: true,
..Default::default()
@@ -788,13 +788,29 @@ async fn test_ocr_rounds_cover_every_page_once() {
(parser, calls, peak)
};
- // Bounded rounds: one page per round, recognition fully serialized.
+ let assert_pages_carry_own_rasters = |result: &liteparse::ParseResult, label: &str| {
+ assert_eq!(result.pages.len(), page_sizes.len(), "{label}: page count");
+ for (page, expected) in result.pages.iter().zip(&expected_texts) {
+ let text: String = page
+ .text_items
+ .iter()
+ .map(|item| item.text.as_str())
+ .collect();
+ assert!(
+ text.contains(expected.as_str()),
+ "{label}: page {} carries {text:?}, expected {expected:?} — OCR text landed on the wrong page",
+ page.page_number
+ );
+ }
+ };
+
+ // One page per round: four rounds, recognition fully serialized.
let (parser, calls, peak) = run(1);
- let result = parser
+ let serialized = parser
.parse_input(PdfInput::Bytes(blank_pdf(&page_sizes)))
.await
- .expect("bounded-round OCR parse should succeed");
- assert_eq!(result.pages.len(), page_sizes.len());
+ .expect("single-page-round OCR parse should succeed");
+ assert_pages_carry_own_rasters(&serialized, "rounds of 1");
assert_eq!(
calls.load(Ordering::SeqCst),
page_sizes.len(),
@@ -803,32 +819,76 @@ async fn test_ocr_rounds_cover_every_page_once() {
assert_eq!(
peak.load(Ordering::SeqCst),
1,
- "a one-page round budget must serialize recognition; >1 means the round loop is not bounding"
+ "num_workers=1 must serialize recognition"
);
- for (page, expected) in result.pages.iter().zip(&expected_texts) {
- let text: String = page
- .text_items
- .iter()
- .map(|item| item.text.as_str())
- .collect();
- assert!(
- text.contains(expected.as_str()),
- "page {} carries {text:?}, expected {expected:?} — OCR text landed on the wrong page",
- page.page_number
- );
- }
- // Legacy escape hatch: budget 0 renders everything in one round and
- // recognition overlaps up to num_workers.
- let (parser, calls, peak) = run(0);
- let result = parser
+ // Round wide enough for the whole document: one round, overlapping
+ // recognition, identical routing.
+ let (parser, calls, peak) = run(4);
+ let overlapped = parser
.parse_input(PdfInput::Bytes(blank_pdf(&page_sizes)))
.await
- .expect("legacy single-round OCR parse should succeed");
- assert_eq!(result.pages.len(), page_sizes.len());
+ .expect("single-round OCR parse should succeed");
+ assert_pages_carry_own_rasters(&overlapped, "rounds of 4");
assert_eq!(calls.load(Ordering::SeqCst), page_sizes.len());
assert!(
peak.load(Ordering::SeqCst) > 1,
- "budget 0 must restore the single-round path with overlapping recognition"
+ "num_workers=4 over 4 pages must overlap recognition within the round"
+ );
+}
+
+/// A round is bounded by rasters rendered, not by page span, so OCR-needing
+/// pages that are sparsely scattered through a mostly-native-text document
+/// still fill a round and recognize concurrently.
+///
+/// This guards a real regression: bounding the round by page span instead
+/// made each round contain only the OCR-needing pages that happened to fall
+/// inside its span — often one or two — which starved the worker pool and
+/// serialized recognition. On this document that was a 2.4x wall-clock loss
+/// at realistic OCR latency, with no test failing.
+#[tokio::test]
+#[serial]
+async fn test_ocr_rounds_fill_across_sparse_pages() {
+ use std::sync::Arc;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ let calls = Arc::new(AtomicUsize::new(0));
+ let peak = Arc::new(AtomicUsize::new(0));
+ let parser = LiteParse::new(LiteParseConfig {
+ ocr_enabled: true,
+ num_workers: 8,
+ dpi: 72.0, // Small rasters: this test is about scheduling, not pixels.
+ quiet: true,
+ ..Default::default()
+ })
+ .with_ocr_engine(std::sync::Arc::new(ProbeEngine {
+ calls: calls.clone(),
+ in_flight: Arc::new(AtomicUsize::new(0)),
+ peak_in_flight: peak.clone(),
+ }));
+
+ let result = parser
+ .parse("../../demo/docs/apple-10k-2024.pdf")
+ .await
+ .expect("should parse the 10-K with OCR enabled");
+
+ let recognized = calls.load(Ordering::SeqCst);
+ assert!(
+ recognized > 8,
+ "expected the fixture to need OCR on more than one round's worth of pages, got {recognized}"
+ );
+ assert!(
+ result.pages.len() > recognized,
+ "fixture should be mostly native text, so OCR pages ({recognized}) must be sparse \
+ among its {} pages",
+ result.pages.len()
+ );
+ // The scan-ahead must gather a full round even though the OCR-needing
+ // pages are interleaved with native-text pages it skips.
+ assert_eq!(
+ peak.load(Ordering::SeqCst),
+ 8,
+ "rounds must fill to num_workers across skipped pages; a lower peak means \
+ rounds are being cut short by page span"
);
}
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index bf3702f..3a6f4dd 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -71,7 +71,8 @@ export interface JsLiteParseConfig {
/**
* Emit each page's classified layout blocks (headings, paragraphs, list
* items, tables with per-cell boxes, code, rules, figures) with bounding
- * boxes as `ParsedPage.blocks`. Default false.
+ * boxes as `ParsedPage.blocks`. Default false. Independent of
+ * `outputFormat`; enabling it never changes the rendered markdown.
*/
extractBlocks?: boolean
/** Extract AcroForm widget fields and values. */
@@ -143,18 +144,6 @@ export interface JsLiteParseConfig {
includeComplexity?: boolean
/** Expose page-scoped vector path extraction. Default false. */
extractVectorGraphics?: boolean
- /**
- * Approximate memory budget (MiB) for extracted content accumulated
- * during a parse; the parse rejects with a clear error instead of
- * growing without bound. 0 disables. Default 4096.
- */
- memoryBudgetMb?: number
- /**
- * Approximate budget (MiB) for OCR page rasters held in memory at once;
- * pages render and recognize in rounds sized to this budget. 0 renders
- * all pages up front. Default 1024.
- */
- ocrRasterBudgetMb?: number
}
/**
* A page sub-region as the fraction cropped from each side (top-left origin,
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index beedfd4..8ef08de 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -119,19 +119,6 @@ export interface LiteParseConfig {
includeComplexity: boolean;
/** Expose page-scoped vector shapes and merged H/V line segments. Default false. */
extractVectorGraphics: boolean;
- /**
- * Approximate memory budget (MiB) for extracted content accumulated during
- * a parse (text items, word boxes, embedded images, screenshot PNGs). The
- * parse rejects with a clear error instead of growing without bound on
- * pathological inputs. 0 disables the check. Default 4096.
- */
- memoryBudgetMb: number;
- /**
- * Approximate budget (MiB) for OCR page rasters held in memory at once;
- * pages render and recognize in bounded rounds sized to this budget.
- * 0 renders all pages up front. Default 1024.
- */
- ocrRasterBudgetMb: number;
}
/**
@@ -659,8 +646,6 @@ export class LiteParse {
skipDiagonalText: userConfig.skipDiagonalText,
includeComplexity: userConfig.includeComplexity,
extractVectorGraphics: userConfig.extractVectorGraphics,
- memoryBudgetMb: userConfig.memoryBudgetMb,
- ocrRasterBudgetMb: userConfig.ocrRasterBudgetMb,
};
this._native = new native.LiteParse(nativeConfig);
@@ -705,8 +690,6 @@ export class LiteParse {
skipDiagonalText: resolved.skipDiagonalText ?? false,
includeComplexity: resolved.includeComplexity ?? false,
extractVectorGraphics: resolved.extractVectorGraphics ?? false,
- memoryBudgetMb: resolved.memoryBudgetMb ?? 4096,
- ocrRasterBudgetMb: resolved.ocrRasterBudgetMb ?? 1024,
};
}
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index b94450a..d247d6d 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -57,8 +57,6 @@ export interface LiteParseNativeConfig {
skipDiagonalText?: boolean;
includeComplexity?: boolean;
extractVectorGraphics?: boolean;
- memoryBudgetMb?: number;
- ocrRasterBudgetMb?: number;
}
export interface NativeCropBox {
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index 8fdb9db..f48d566 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -489,8 +489,6 @@ class LiteParse:
skip_diagonal_text: Optional[bool] = None,
include_complexity: Optional[bool] = None,
extract_vector_graphics: Optional[bool] = None,
- memory_budget_mb: Optional[int] = None,
- ocr_raster_budget_mb: Optional[int] = None,
):
"""
Initialize LiteParse parser.
@@ -570,14 +568,6 @@ class LiteParse:
pass.
extract_vector_graphics: Expose page-scoped vector shapes and
merged horizontal/vertical line segments. Default False.
- memory_budget_mb: Approximate memory budget (MiB) for extracted
- content accumulated during a parse; the parse fails with a
- clear error instead of growing without bound. 0 disables.
- Default 4096.
- ocr_raster_budget_mb: Approximate budget (MiB) for OCR page
- rasters held in memory at once; pages render and recognize
- in bounded rounds. 0 renders all pages up front. Default
- 1024.
"""
kwargs = {}
if ocr_enabled is not None:
@@ -654,10 +644,6 @@ class LiteParse:
kwargs["include_complexity"] = include_complexity
if extract_vector_graphics is not None:
kwargs["extract_vector_graphics"] = extract_vector_graphics
- if memory_budget_mb is not None:
- kwargs["memory_budget_mb"] = memory_budget_mb
- if ocr_raster_budget_mb is not None:
- kwargs["ocr_raster_budget_mb"] = ocr_raster_budget_mb
self._native = _NativeLiteParse(**kwargs)
@@ -897,8 +883,6 @@ class LiteParse:
extract_document_metadata=cfg.extract_document_metadata,
extract_content_bounds=cfg.extract_content_bounds,
detect_screenshot_rects=cfg.detect_screenshot_rects,
- memory_budget_mb=cfg.memory_budget_mb,
- ocr_raster_budget_mb=cfg.ocr_raster_budget_mb,
)
def __repr__(self) -> str:
diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py
index ffa48f7..c796c75 100644
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@ -488,14 +488,6 @@ class LiteParseConfig:
extract_document_metadata: bool = False
detect_screenshot_rects: bool = False
extract_content_bounds: bool = False
- #: Approximate memory budget (MiB) for extracted content accumulated
- #: during a parse; the parse raises a clear error instead of growing
- #: without bound. 0 disables. Default 4096.
- memory_budget_mb: int = 4096
- #: Approximate budget (MiB) for OCR page rasters held in memory at once;
- #: pages render and recognize in bounded rounds. 0 renders all pages up
- #: front. Default 1024.
- ocr_raster_budget_mb: int = 1024
class ParseError(Exception):