A fast, helpful, and open-source document parser
clean up implementation
Logan Markewichfa0dd80a719c
diff --git a/Cargo.lock b/Cargo.lockindex 73750c2..50608b4 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2018,6 +2018,7 @@ checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"name = "liteparse"version = "2.14.6"dependencies = [+ "base64 0.22.1","blake3","clap","crc32fast",diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.tomlindex 3548037..833a6cb 100644--- a/crates/liteparse/Cargo.toml+++ b/crates/liteparse/Cargo.toml@@ -27,6 +27,7 @@ oar-ocr-openvino = ["oar-ocr", "oar-ocr/openvino"]tesseract = ["dep:tesseract-rs"][dependencies]+base64 = "0.22"blake3 = "1"clap = { version = "4.5.55", features = ["derive"] }file-format = { version = "0.29.0", features = ["reader"] }diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex c56386d..8cf1b7f 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -275,7 +275,7 @@ impl Default for LiteParseConfig {target_pages: None,extract_screenshots: false,continue_on_page_error: false,- dpi: 150.0,+ dpi: DEFAULT_DPI,output_format: OutputFormat::Json,preserve_very_small_text: false,password: None,@@ -330,6 +330,10 @@ const MAX_TARGET_PAGES: u64 = 100_000;/// roughly 13% at this size on a 457-page document, ~4% at 50).pub const DEFAULT_PAGE_BATCH_SIZE: usize = 25;+/// Default render resolution for OCR rasters and screenshots+/// (`LiteParseConfig::dpi`).+pub const DEFAULT_DPI: f32 = 150.0;+#[doc(hidden)]pub fn parse_target_pages(s: &str) -> Result<Vec<u32>, String> {let mut pages = Vec::new();@@ -426,7 +430,7 @@ mod tests {assert_eq!(c.max_pages, 1000);assert!(!c.extract_screenshots);assert!(!c.continue_on_page_error);- assert_eq!(c.dpi, 150.0);+ assert_eq!(c.dpi, DEFAULT_DPI);assert_eq!(c.output_format, OutputFormat::Json);assert!(!c.preserve_very_small_text);assert!(!c.quiet);diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex b258867..acd8b3a 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -12,6 +12,7 @@ use pdfium::{Document, Font, FontType, FormEnvironment, Library, Page, PathObject, PdfLink, RectF,SegmentKind, TextPage,};+use serde::{Deserialize, Serialize};/// Dedup spatial-grid cell size bounds (pt). The cell tracks the typical item/// footprint so a cell holds O(1) non-overlapping items.@@ -105,6 +106,11 @@ pub(crate) fn extract_pages_from_document(}/// Output of [`extract_pages_and_images`].+///+/// Serializable so the whole result can cross a stage boundary (see+/// `stages`); image payloads (`ExtractedImage::bytes`) are the one thing+/// that does not, and travel separately keyed by image id.+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ExtractedPages {pub pages: Vec<LitePage>,pub page_errors: Vec<PageError>,diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex 14aa440..4650f12 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -44,12 +44,13 @@ const DENSE_GRAPHICS_MIN_COVERAGE: f32 = 0.2;/// the document again.#[derive(Debug, Clone, Serialize, Deserialize)]pub struct OcrRaster {- /// Position in the `pages` slice the raster was rendered from.- pub page_idx: usize,- /// 1-based source page number, for logs and for callers that key by- /// page rather than by slice position.+ /// 1-based source page number. Outcomes are matched back to pages by+ /// this number, so a raster can be merged into any page slice that+ /// contains its page.pub page_number: usize,- /// Tightly-packed pixels: 1 byte/px (grayscale) or 3 (RGB).+ /// Tightly-packed pixels: 1 byte/px (grayscale) or 3 (RGB). Serialized+ /// as a base64 string, since a JSON array of numbers is ~3.5× the size.+ #[serde(with = "base64_bytes")]pub pixels: Vec<u8>,pub width: u32,pub height: u32,@@ -67,7 +68,11 @@ pub struct OcrRaster {}/// How [`render_pages_for_ocr`] picks and rasterizes pages.-#[derive(Debug, Clone, Default)]+///+/// `Default` renders every page that needs OCR at+/// [`DEFAULT_DPI`](crate::config::DEFAULT_DPI) in RGB, in one round, with+/// no form rendering or re-flattening.+#[derive(Debug, Clone)]pub struct OcrRenderOptions {/// Stop after this many rasters (`0` = no limit), so a long document can/// be processed in bounded rounds; the function returns where to resume.@@ -91,6 +96,36 @@ pub struct OcrRenderOptions {pub selection: Option<HashSet<u32>>,}+impl Default for OcrRenderOptions {+ fn default() -> Self {+ Self {+ max_rasters: 0,+ dpi: crate::config::DEFAULT_DPI,+ grayscale: false,+ render_form_fields: false,+ continue_on_page_error: false,+ reflatten_pages: HashSet::new(),+ selection: None,+ }+ }+}++/// Base64 (de)serialization for raster pixel buffers.+mod base64_bytes {+ use base64::Engine as _;+ use base64::engine::general_purpose::STANDARD;+ use serde::{Deserialize, Deserializer, Serializer};++ pub fn serialize<S: Serializer>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> {+ serializer.serialize_str(&STANDARD.encode(bytes))+ }++ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<u8>, D::Error> {+ let encoded = String::deserialize(deserializer)?;+ STANDARD.decode(encoded).map_err(serde::de::Error::custom)+ }+}+/// Why a page was flagged as needing more than the cheap text-only path./// Multiple reasons can apply to one page (e.g. a sparse page whose little/// text is also garbled). Empty exactly when `needs_ocr` is false.@@ -251,7 +286,7 @@ pub(crate) fn calculate_page_complexity(let is_garbled = page_is_garbled(page);let mut reasons = Vec::new();- if text_length < 20 {+ if text_length < MIN_NATIVE_TEXT_LEN {// Too little text to be the page's content. A full-page raster behind// it means a scan; otherwise it's effectively blank.reasons.push(if full_page_image {@@ -399,7 +434,7 @@ pub fn calculate_native_page_complexity(let is_garbled = page_is_garbled(page);let mut reasons = Vec::new();- if text_length < 20 {+ if text_length < MIN_NATIVE_TEXT_LEN {reasons.push(if full_page_image {ComplexityReason::Scanned} else {@@ -690,14 +725,32 @@ pub fn render_pages_for_ocr(} else {document.page(page_index)?};- let page_complexity = calculate_page_complexity(page, &page_obj)?;-- // An explicit selection overrides the scoring's OCR gate; the- // scoring still decides `has_native_text`, which drives the- // merge's artifact filters.- if selection.is_none() && !page_complexity.needs_ocr {- return Ok(None);- }+ // `has_native_text` drives the merge's artifact filters: true+ // when the page was not sent to OCR as scanned, blank, garbled or+ // annotation-only. With an explicit selection the OCR gate is the+ // caller's decision, so the page-object walk in+ // `calculate_page_complexity` is skipped and the signal is derived+ // from the text items alone (which is all those reasons read).+ let has_native_text = match selection {+ Some(_) => {+ native_text_length(page) >= MIN_NATIVE_TEXT_LEN && !page_is_garbled(page)+ }+ None => {+ let page_complexity = calculate_page_complexity(page, &page_obj)?;+ if !page_complexity.needs_ocr {+ return Ok(None);+ }+ !page_complexity.reasons.iter().any(|r| {+ matches!(+ r,+ ComplexityReason::Scanned+ | ComplexityReason::NoText+ | ComplexityReason::Garbled+ | ComplexityReason::AnnotationText+ )+ })+ }+ };// Clamp the render DPI so the long edge stays within the raster budget.let long_edge_pt = page.page_width.max(page.page_height);@@ -719,15 +772,6 @@ pub fn render_pages_for_ocr(bitmap.to_rgb()};- let has_native_text = !page_complexity.reasons.iter().any(|r| {- matches!(- r,- ComplexityReason::Scanned- | ComplexityReason::NoText- | ComplexityReason::Garbled- | ComplexityReason::AnnotationText- )- });let image_rects = if has_native_text {let pw = page.page_width;let ph = page.page_height;@@ -750,7 +794,6 @@ pub fn render_pages_for_ocr(};Ok(Some(OcrRaster {- page_idx: idx,page_number: page.page_number,pixels,width,@@ -786,20 +829,24 @@ pub fn render_pages_for_ocr(/// function of `(pages, outcomes)`. The pixels are gone — this is the small/// payload that crosses a stage boundary, not the bitmap.///-/// `result` is `Err(message)` when the engine failed on this page; the merge-/// counts failures to tell a systemic engine problem from a flaky page.+/// `error` is set when the engine failed on this page (`results` is then+/// empty); the merge counts failures to tell a systemic engine problem from+/// a flaky page.#[derive(Debug, Clone, Serialize, Deserialize)]pub struct PageOcrOutcome {- /// Position in the `pages` slice the raster came from.- pub page_idx: usize,- /// 1-based source page number.+ /// 1-based source page number the raster came from.pub page_number: usize,/// DPI the raster was rendered at; pixel → point scale is `72 / dpi`.pub dpi: f32,pub has_native_text: bool,pub image_rects: Vec<Rect>,- /// Engine output in raster pixel coordinates.- pub result: Result<Vec<OcrResult>, String>,+ /// Engine output in raster pixel coordinates. Empty when the engine+ /// failed (see `error`) or found nothing.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ pub results: Vec<OcrResult>,+ /// The engine's error message when recognition failed on this page.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,}/// Run `ocr_engine` over pre-rendered page bitmaps, at most `num_workers`@@ -818,11 +865,10 @@ pub async fn recognize_rasters(// blocking thread pool. Run each JavaScript OCR callback directly so the// returned Promise can make progress on the browser event loop.#[cfg(target_arch = "wasm32")]- let task_results: Vec<(usize, usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {+ let task_results: Vec<(usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {let _ = num_workers;let mut results = Vec::with_capacity(rendered.len());for r in rendered {- let idx = r.page_idx;let page_number = r.page_number;let page_dpi = r.dpi;let native = (r.has_native_text, r.image_rects.clone());@@ -833,7 +879,7 @@ pub async fn recognize_rasters(let result = ocr_engine.recognize(&r.pixels, r.width, r.height, &options).await;- results.push((idx, page_number, page_dpi, native, result));+ results.push((page_number, page_dpi, native, result));}results};@@ -852,7 +898,7 @@ pub async fn recognize_rasters(// pass deadlocks. Acquiring the permit asynchronously parks the lightweight// task instead, so only `num_workers` blocking threads are ever consumed.#[cfg(not(target_arch = "wasm32"))]- let task_results: Vec<(usize, usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {+ let task_results: Vec<(usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {let num_workers = num_workers.max(1);let semaphore = Arc::new(tokio::sync::Semaphore::new(num_workers));let mut handles = Vec::with_capacity(rendered.len());@@ -867,7 +913,6 @@ pub async fn recognize_rasters(let rt_handle = handle.clone();handles.push((- r.page_idx,page_number,r.dpi,(r.has_native_text, r.image_rects.clone()),@@ -898,14 +943,14 @@ pub async fn recognize_rasters(}let mut results = Vec::with_capacity(handles.len());- for (idx, page_number, page_dpi, native, handle) in handles {+ for (page_number, page_dpi, native, handle) in handles {let result = match handle.await {Ok(result) => result,Err(join_err) => {Err(Box::new(join_err) as Box<dyn std::error::Error + Send + Sync>)}};- results.push((idx, page_number, page_dpi, native, result));+ results.push((page_number, page_dpi, native, result));}results};@@ -913,13 +958,19 @@ pub async fn recognize_rasters(task_results.into_iter().map(- |(page_idx, page_number, dpi, (has_native_text, image_rects), result)| PageOcrOutcome {- page_idx,- page_number,- dpi,- has_native_text,- image_rects,- result: result.map_err(|e| e.to_string()),+ |(page_number, dpi, (has_native_text, image_rects), result)| {+ let (results, error) = match result {+ Ok(results) => (results, None),+ Err(e) => (Vec::new(), Some(e.to_string())),+ };+ PageOcrOutcome {+ page_number,+ dpi,+ has_native_text,+ image_rects,+ results,+ error,+ }},).collect()@@ -933,6 +984,10 @@ pub async fn recognize_rasters(/// first so OCR can replace it; then each result is filtered against rule/// artifacts, chart figures and overlap with surviving native text before/// being appended as an `OCR`-font [`TextItem`] in viewport points.+///+/// Outcomes are matched to pages by `page_number`; an outcome for a page not+/// in `pages` is an error rather than a panic, so a caller can merge a+/// subset of pages or hand-built outcomes safely.pub fn merge_ocr_results(pages: &mut [Page],outcomes: Vec<PageOcrOutcome>,@@ -955,27 +1010,36 @@ pub fn merge_ocr_results(let mut failed_sparse_text_page = false;let mut first_error: Option<String> = None;+ let index_by_page_number: HashMap<usize, usize> = pages+ .iter()+ .enumerate()+ .map(|(idx, page)| (page.page_number, idx))+ .collect();+for outcome in outcomes {let PageOcrOutcome {- page_idx: idx,page_number,dpi: page_dpi,has_native_text,image_rects,- result,+ results: ocr_results,+ error,} = outcome;- let ocr_results: Vec<OcrResult> = match result {- Ok(results) => results,- Err(msg) => {- failed_tasks += 1;- failed_sparse_text_page |= page_has_sparse_native_text(&pages[idx]);- if first_error.is_none() {- eprintln!("[ocr] failed for page {}: {}", page_number, msg);- first_error = Some(msg);- }- continue;- }+ let Some(&idx) = index_by_page_number.get(&page_number) else {+ return Err(LiteParseError::Other(format!(+ "OCR outcome for page {page_number} has no matching page among the {} page(s) being merged",+ pages.len()+ )));};+ if let Some(msg) = error {+ failed_tasks += 1;+ failed_sparse_text_page |= page_has_sparse_native_text(&pages[idx]);+ if first_error.is_none() {+ eprintln!("[ocr] failed for page {}: {}", page_number, msg);+ first_error = Some(msg);+ }+ continue;+ }if ocr_results.is_empty() {continue;@@ -1189,12 +1253,7 @@ pub fn merge_ocr_results(/// systemic-failure guard matches the same pages that were rendered because/// their native text was insufficient.fn page_has_sparse_native_text(page: &Page) -> bool {- let text_length: usize = page- .text_items- .iter()- .filter(|item| !is_unusable_native(item))- .map(|item| item.text.len())- .sum();+ let text_length = native_text_length(page);let page_area = page.page_width * page.page_height;let text_bbox_area: f32 = page.text_items@@ -1208,7 +1267,21 @@ fn page_has_sparse_native_text(page: &Page) -> bool {0.0};- text_length < 20 || text_coverage < 0.15+ text_length < MIN_NATIVE_TEXT_LEN || text_coverage < 0.15+}++/// Below this many bytes of usable native text a page counts as having no+/// text of its own (`NoText` / `Scanned`), so OCR is its primary source.+const MIN_NATIVE_TEXT_LEN: usize = 20;++/// Bytes of native text on the page, excluding items OCR would replace+/// anyway (see `is_unusable_native`).+fn native_text_length(page: &Page) -> usize {+ page.text_items+ .iter()+ .filter(|item| !is_unusable_native(item))+ .map(|item| item.text.len())+ .sum()}/// A native text item that cannot be trusted as a text source: either its@@ -2113,7 +2186,6 @@ mod tests {OcrRaster {has_native_text: false,image_rects: Vec::new(),- page_idx: idx,page_number: idx + 1,// 1x1 grayscale pixel; the engine never inspects it.pixels: vec![0u8],diff --git a/crates/liteparse/src/output/mod.rs b/crates/liteparse/src/output/mod.rsindex c2d2804..1c593c8 100644--- a/crates/liteparse/src/output/mod.rs+++ b/crates/liteparse/src/output/mod.rs@@ -1,3 +1,2 @@pub mod json;-pub mod markdown;pub mod text;diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 6d0b14c..4fb0d5f 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -68,7 +68,7 @@ pub struct ParseResult {}/// Result of rendering a single page screenshot.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct ScreenshotResult {pub page_num: u32,pub width: u32,@@ -101,7 +101,7 @@ fn write_extracted_images(// pipeline is built around): only the canonical file is written; every// duplicate placement keeps its own `name` but points `path` at the// canonical file. Markdown figure references are rewritten to the- // canonical name (`rewrite_duplicate_image_refs`) so they only ever+ // canonical name (`stages::canonicalize_image_refs`) so they only ever// reference files that exist.let mut written: HashMap<String, String> = HashMap::new();for image in images {@@ -121,51 +121,6 @@ fn write_extracted_images(Ok(())}-/// Rewrite markdown figure references for deduplicated images to the-/// canonical entry's file name. The markdown emitter references each figure-/// by its own placement id (``), but only the canonical-/// file is written to disk (see `write_extracted_images`), so duplicate-/// placements must reference the canonical name, matching the resolution the-/// LlamaParse worker applies via `resolveOutputImageName`. Public as-/// [`stages::canonicalize_image_refs`].-pub(crate) fn rewrite_duplicate_image_refs(- pages: &mut [ParsedPage],- full_text: &mut String,- images: &[ExtractedImage],-) {- use std::collections::HashMap;-- let by_id: HashMap<&str, &ExtractedImage> = images- .iter()- .map(|image| (image.id.as_str(), image))- .collect();- let renames: Vec<(String, String)> = images- .iter()- .filter_map(|image| {- let canonical = by_id.get(image.duplicate_of.as_ref()?.as_str())?;- Some((- format!("", image.id, image.format),- format!("", canonical.name),- ))- })- .collect();- if renames.is_empty() {- return;- }-- for markdown in pages- .iter_mut()- .map(|page| &mut page.markdown)- .chain(std::iter::once(full_text))- {- for (from, to) in &renames {- if markdown.contains(from.as_str()) {- *markdown = markdown.replace(from.as_str(), to);- }- }- }-}-/// Build the default glyph resolver from the environment, if configured.#[cfg(not(target_arch = "wasm32"))]fn default_glyph_resolver() -> Option<std::sync::Arc<dyn crate::GlyphResolver>> {@@ -246,20 +201,17 @@ pub struct LiteParse {/// consumers share a single pass rather than each triggering their own. Returns/// the joined document markdown when that is the output format.fn apply_layout(- config: &crate::config::LiteParseConfig,+ parser: &LiteParse,parsed_pages: &mut [ParsedPage],outline: &[OutlineTarget],) -> Option<String> {+ let config = &parser.config;let wants_markdown = config.output_format == crate::config::OutputFormat::Markdown;if !wants_markdown && !config.extract_blocks {return None;}let signals = stages::document_signals(parsed_pages, config.keep_headers_footers);- let block_options = stages::BlockOptions {- outline,- image_mode: config.image_mode,- keep_headers_footers: config.keep_headers_footers,- };+ let block_options = parser.block_options(outline);let mut page_md = Vec::with_capacity(parsed_pages.len());for page in parsed_pages.iter_mut() {let blocks = stages::extract_blocks(page, &signals, &block_options);@@ -384,7 +336,8 @@ impl LiteParse {}}- /// The OCR render stage's options for a full parse, derived from config.+ /// The OCR render stage's options for a full parse, derived from config+ /// and from what extraction did to the document.////// Extraction may have flattened SOME pages' form widgets into page/// content in its (since dropped) document instance; those pages are@@ -394,11 +347,11 @@ impl LiteParse {pub fn ocr_render_options(&self,grayscale: bool,- flattened_form_widgets: bool,- flattened_page_numbers: &[u32],+ extracted: &stages::ExtractedPages,) -> stages::OcrRenderOptions {- let reflatten_pages = if flattened_form_widgets && !self.config.render_form_fields {- flattened_page_numbers.iter().copied().collect()+ let reflatten_pages = if extracted.flattened_form_widgets && !self.config.render_form_fields+ {+ extracted.flattened_page_numbers.iter().copied().collect()} else {std::collections::HashSet::new()};@@ -414,6 +367,26 @@ impl LiteParse {}}+ /// The screenshot stage's options, derived from config. `parse()` passes+ /// `config.continue_on_page_error`; `screenshot()` is always fail-fast.+ pub fn screenshot_options(&self, continue_on_page_error: bool) -> stages::ScreenshotOptions {+ stages::ScreenshotOptions {+ dpi: self.config.dpi,+ detect_rects: self.config.detect_screenshot_rects,+ render_form_fields: self.config.render_form_fields,+ continue_on_page_error,+ }+ }++ /// The block-classification stage's options, derived from config.+ pub fn block_options<'a>(&self, outline: &'a [OutlineTarget]) -> stages::BlockOptions<'a> {+ stages::BlockOptions {+ outline,+ image_mode: self.config.image_mode,+ keep_headers_footers: self.config.keep_headers_footers,+ }+ }+/// The content-filter stage's options, derived from config.pub fn content_filters(&self) -> stages::ContentFilters<'_> {stages::ContentFilters {@@ -686,8 +659,7 @@ impl LiteParse {producer,doc_meta,xfa_packets,- flattened_form_widgets,- flattened_page_numbers,+ ocr_render_options,repaired_input,) = {let lib = Library::init();@@ -733,14 +705,10 @@ impl LiteParse {let outline = outline.unwrap_or_else(|| stages::outline(&document));let extracted =stages::extract(&document, &self.extract_request(target_pages, max_pages))?;- let stages::ExtractedPages {- pages,- page_errors,- images,- image_error_count,- flattened_form_widgets,- flattened_page_numbers,- } = extracted;+ // Derived here, before the document is dropped, so the OCR rounds+ // below reproduce exactly what extraction did to it.+ let ocr_render_options = self.ocr_render_options(ocr_grayscale, &extracted);+ let screenshot_options = self.screenshot_options(self.config.continue_on_page_error);// Reopening the input costs a full parse, so it is confined to the// one consumer that genuinely needs live widget annotations: the// opt-in form renderer, which initializes the form environment to@@ -753,13 +721,19 @@ impl LiteParse {// flattened document by design (see `is_complex`). OCR rasters// render in bounded rounds after this section, each against a// freshly reopened (hence pristine) document.- let needs_pristine_document = flattened_form_widgets- && self.config.extract_screenshots- && self.config.render_form_fields;+ let needs_pristine_document = self.config.extract_screenshots+ && stages::screenshots_need_pristine_document(&extracted, &screenshot_options);let pristine_document = needs_pristine_document.then(|| self.open_document(&lib, document_input, password)).transpose()?;let analysis_document = pristine_document.as_ref().unwrap_or(&document);+ let stages::ExtractedPages {+ pages,+ page_errors,+ images,+ image_error_count,+ ..+ } = extracted;let t_extract = web_time::Instant::now();log(&format!("[liteparse] extract: {:.1}ms ({} pages)",@@ -790,16 +764,7 @@ impl LiteParse {.iter().map(|page| page.page_number as u32).collect::<Vec<_>>();- stages::screenshots(- analysis_document,- Some(&page_numbers),- &stages::ScreenshotOptions {- dpi: self.config.dpi,- detect_rects: self.config.detect_screenshot_rects,- render_form_fields: self.config.render_form_fields,- continue_on_page_error: self.config.continue_on_page_error,- },- )?+ stages::screenshots(analysis_document, Some(&page_numbers), &screenshot_options)?} else {Vec::new()};@@ -820,8 +785,7 @@ impl LiteParse {producer,doc_meta,xfa_packets,- flattened_form_widgets,- flattened_page_numbers,+ ocr_render_options,repaired_input,)};@@ -829,11 +793,7 @@ impl LiteParse {let t1 = web_time::Instant::now();if let Some(engine) = ocr_engine {- let render_options = self.ocr_render_options(- ocr_grayscale,- flattened_form_widgets,- &flattened_page_numbers,- );+ let render_options = ocr_render_options;let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);let mut round_start = 0usize;while round_start < pages.len() {@@ -895,7 +855,7 @@ impl LiteParse {t2.duration_since(t_ocr).as_secs_f64() * 1000.0));- let laid_out = apply_layout(&self.config, &mut parsed_pages, &outline);+ let laid_out = apply_layout(self, &mut parsed_pages, &outline);let mut full_text = if let Some(md) = laid_out {let t3 = web_time::Instant::now();log(&format!(@@ -953,7 +913,7 @@ impl LiteParse {let total_pages = pages.len().min(u32::MAX as usize) as u32;let mut parsed_pages = stages::project(pages);- let full_text = if let Some(md) = apply_layout(&self.config, &mut parsed_pages, &outline) {+ let full_text = if let Some(md) = apply_layout(self, &mut parsed_pages, &outline) {md} else {parsed_pages@@ -1137,12 +1097,7 @@ impl LiteParse {stages::screenshots(&document,page_numbers.as_deref(),- &stages::ScreenshotOptions {- dpi: self.config.dpi,- detect_rects: self.config.detect_screenshot_rects,- render_form_fields: self.config.render_form_fields,- continue_on_page_error: false,- },+ &self.screenshot_options(false),)}@@ -1524,7 +1479,7 @@ mod tests {image("p2_1", "jpg", Some("p1_1")),];- rewrite_duplicate_image_refs(&mut pages, &mut full_text, &images);+ stages::canonicalize_image_refs(&mut pages, &mut full_text, &images);// The duplicate's ref now points at the canonical file; canonical// refs and surrounding text are untouched.diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rsindex dfce1f8..79e2e4c 100644--- a/crates/liteparse/src/render.rs+++ b/crates/liteparse/src/render.rs@@ -1,6 +1,5 @@-use crate::config::PageOrientationCorrection;use crate::error::LiteParseError;-use crate::extract::{apply_page_orientation_corrections, encode_png, load_document_from_input};+use crate::extract::{encode_png, load_document_from_input};use crate::types::{PdfInput, ScreenshotRect};use pdfium::Library;use serde::Serialize;@@ -27,39 +26,6 @@ pub struct RenderedPage {pub rects: Vec<ScreenshotRect>,}-/// Render selected pages from a PDF input to PNG bytes.-///-/// With `render_form_fields`, form-field appearances (filled values, checkbox-/// states) are drawn on top of the page raster; this runs the document's-/// open/JS actions, so it is opt-in. With `detect_rects`, each page's raster-/// is also scanned for solid rectangles and lines.-///-/// Acquires the process-global PDFium lock for the entire render. The lock-/// is held until this function returns — PNG encoding happens inside the-/// critical section, which is fine because it is pure CPU work with no-/// `.await` points.-pub fn render_pages_to_png(- input: &PdfInput,- page_numbers: Option<&[u32]>,- dpi: f32,- password: Option<&str>,- detect_rects: bool,- render_form_fields: bool,- page_orientation_corrections: &[PageOrientationCorrection],-) -> Result<Vec<RenderedPage>, LiteParseError> {- let lib = Library::init();- let document = load_document_from_input(&lib, input, password)?;- apply_page_orientation_corrections(&document, page_orientation_corrections)?;- render_document_pages(- &document,- page_numbers,- dpi,- detect_rects,- render_form_fields,- false,- )-}-pub(crate) fn render_document_pages(document: &pdfium::Document,page_numbers: Option<&[u32]>,@@ -272,31 +238,6 @@ pub(crate) fn find_solid_rects_rgba(out}-/// Render a single page to a PNG file.-pub fn screenshot(- pdf_path: &str,- page_num: u32,- dpi: f32,- output_path: &str,- password: Option<&str>,-) -> Result<(), LiteParseError> {- let input = PdfInput::Path(pdf_path.to_string());- let pages = render_pages_to_png(&input, Some(&[page_num]), dpi, password, false, false, &[])?;- let page = pages- .into_iter()- .next()- .ok_or_else(|| LiteParseError::Other("no page rendered".into()))?;-- std::fs::write(output_path, &page.png_bytes)?;-- eprintln!(- "[rust-bin] rendered page {} at {dpi} DPI → {output_path} ({}×{})",- page_num, page.width, page.height- );-- Ok(())-}-#[derive(Debug, Serialize)]struct ImageBoundsOutput {x: f32,@@ -358,18 +299,6 @@ mod tests {assert!(s.contains("\"width\":3"));}- #[test]- fn test_screenshot_missing_file_errors() {- let r = screenshot(- "/nonexistent/path/does_not_exist.pdf",- 1,- 72.0,- "/tmp/out.png",- None,- );- assert!(r.is_err());- }-#[test]fn test_image_bounds_missing_file_errors() {let r = image_bounds("/nonexistent/path/does_not_exist.pdf", None);diff --git a/crates/liteparse/src/stages.rs b/crates/liteparse/src/stages.rsindex 42a96c4..b9006b5 100644--- a/crates/liteparse/src/stages.rs+++ b/crates/liteparse/src/stages.rs@@ -16,7 +16,9 @@//! becoming a public stage.//! * Every type crossing a stage boundary is `Clone + Serialize +//! Deserialize`, and the serialization is lossless for everything a later-//! stage reads.+//! stage reads. The two exceptions are payloads that are better carried+//! out of band: `ExtractedImage::bytes` is skipped (key it by image id),+//! and `OcrRaster::pixels` is base64 so JSON stays ~1.3× the raw size.//!//! Stages split into two kinds. **pdfium-bound** stages take an open//! [`Document`] and must run while the [`Library`] that opened it is alive@@ -36,9 +38,12 @@//! | markdown | [`document_signals`] → [`extract_blocks`] → [`render_page_markdown`] | pure |//! | screenshots | [`screenshots`] | pdfium |//!+//! [`Document`] and [`Library`] are the `pdfium` crate's types re-exported+//! verbatim, so that crate's handle API is part of this module's contract.+//!//! Everything else in the crate that is not re-exported here is unstable.-use std::collections::HashSet;+use std::collections::{HashMap, HashSet};use std::sync::Arc;use serde::{Deserialize, Serialize};@@ -176,8 +181,17 @@ impl Default for ExtractRequest<'_> {}/// Extract text items, graphics, structure nodes and (optionally) images-/// from an open document. This is the only stage that reads page content+/// from an open document. This is the only stage that reads *text* content/// through PDFium; everything downstream works on the returned [`Page`]s.+///+/// With `extract_form_fields`, extraction may flatten some pages' widget+/// annotations into page content to recover their text. That mutates the+/// open document: a later [`screenshots`] call with `render_form_fields` on+/// this same document would paint those widgets twice, and an OCR raster+/// from a reopened document must re-flatten the same pages to match. The+/// returned `flattened_form_widgets` / `flattened_page_numbers` say what+/// happened; [`screenshots_need_pristine_document`] and+/// `LiteParse::ocr_render_options` turn them into the right follow-up.pub fn extract(document: &Document,request: &ExtractRequest<'_>,@@ -380,7 +394,7 @@ pub fn extract_blocks(if !has_content {blocks.retain(|b| !matches!(b.block, Block::HorizontalRule));}- crate::output::markdown::dedupe_rules(&mut blocks);+ dedupe_rules(&mut blocks);Some(markdown_layout::splice_soft_hyphens(blocks))}@@ -417,7 +431,52 @@ pub fn canonicalize_image_refs(full_text: &mut String,images: &[ExtractedImage],) {- crate::parser::rewrite_duplicate_image_refs(pages, full_text, images)+ let by_id: HashMap<&str, &ExtractedImage> = images+ .iter()+ .map(|image| (image.id.as_str(), image))+ .collect();+ let renames: Vec<(String, String)> = images+ .iter()+ .filter_map(|image| {+ let canonical = by_id.get(image.duplicate_of.as_ref()?.as_str())?;+ Some((+ format!("", image.id, image.format),+ format!("", canonical.name),+ ))+ })+ .collect();+ if renames.is_empty() {+ return;+ }++ for markdown in pages+ .iter_mut()+ .map(|page| &mut page.markdown)+ .chain(std::iter::once(full_text))+ {+ for (from, to) in &renames {+ if markdown.contains(from.as_str()) {+ *markdown = markdown.replace(from.as_str(), to);+ }+ }+ }+}++/// Collapse cosmetic horizontal-rule noise on a single page's block stream:+/// drop leading/trailing rules (which would otherwise abut the `-----` page+/// separator) and collapse runs of consecutive rules to one. Rules come from+/// two sources — vector-graphics detection and decorative divider text — and+/// doubling up reads as sloppy output to a human, while carrying no extra+/// structure for an LLM.+fn dedupe_rules(blocks: &mut Vec<PositionedBlock>) {+ use Block::HorizontalRule;+ while matches!(blocks.first().map(|b| &b.block), Some(HorizontalRule)) {+ blocks.remove(0);+ }+ while matches!(blocks.last().map(|b| &b.block), Some(HorizontalRule)) {+ blocks.pop();+ }+ blocks.dedup_by(|a, b| matches!((&a.block, &b.block), (HorizontalRule, HorizontalRule)));}// ── Screenshots ────────────────────────────────────────────────────────@@ -434,6 +493,19 @@ pub struct ScreenshotOptions {pub continue_on_page_error: bool,}+/// Whether [`screenshots`] must run on a freshly opened document rather+/// than the one [`extract`] ran on. True when extraction flattened widget+/// annotations into page content *and* the screenshots will paint form+/// fields through the form environment: on the mutated document the widgets+/// would be drawn twice. Plain screenshots are unaffected, since flattening+/// promotes the same appearance streams the renderer would paint.+pub fn screenshots_need_pristine_document(+ extracted: &ExtractedPages,+ options: &ScreenshotOptions,+) -> bool {+ extracted.flattened_form_widgets && options.render_form_fields+}+/// Render pages to PNG. `page_numbers` is 1-based; `None` renders every/// page.pub fn screenshots(@@ -460,3 +532,196 @@ pub fn screenshots(}).collect())}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::types::{Anchor, ProjectedLine, Rect, TextItem};++ fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine {+ ProjectedLine {+ text: text.into(),+ rtl: crate::bidi::is_rtl_text(text),+ bbox: Rect {+ x,+ y,+ width: text.chars().count() as f32 * (size * 0.5),+ height: h,+ },+ anchor: Anchor::Left,+ indent_x: x,+ dominant_font_size: size,+ font_size_is_estimated: false,+ heading_font_size: None,+ dominant_font_name: Some("Arial".into()),+ all_bold: false,+ all_italic: false,+ all_mono: false,+ all_strike: false,+ spans: vec![TextItem::default()],+ region_path: Vec::new(),+ mcid: None,+ in_figure: false,+ }+ }++ fn page_with(n: usize, lines: Vec<ProjectedLine>) -> ParsedPage {+ ParsedPage {+ page_number: n,+ page_label: None,+ page_width: 612.0,+ page_height: 792.0,+ content_bounds: None,+ text: "fallback".into(),+ markdown: String::new(),+ text_items: vec![],+ projected_lines: lines,+ regions: crate::types::Region::default(),+ graphics: vec![],+ vector_graphics: None,+ figures: vec![],+ projected_item_frames: vec![],+ struct_nodes: vec![],+ image_refs: vec![],+ complexity: None,+ annotations: None,+ form_fields: None,+ structure_tree: None,+ blocks: None,+ }+ }++ /// The markdown stages composed as `parse()` composes them, one string+ /// per page.+ fn render_pages(pages: &[ParsedPage]) -> Vec<String> {+ let signals = document_signals(pages, false);+ let options = BlockOptions {+ outline: &[],+ image_mode: ImageMode::Placeholder,+ keep_headers_footers: false,+ };+ pages+ .iter()+ .map(|page| {+ let blocks = extract_blocks(page, &signals, &options);+ render_page_markdown(page, blocks.as_deref())+ })+ .collect()+ }++ fn render_document(pages: &[ParsedPage]) -> String {+ render_pages(pages).join("\n\n-----\n\n")+ }++ #[test]+ fn empty_document_renders_empty() {+ assert_eq!(render_document(&[]), "");+ }++ #[test]+ fn dedupe_rules_drops_edges_and_collapses_runs() {+ use Block::{HorizontalRule, Paragraph};+ let p = |t: &str| Paragraph {+ text: t.into(),+ bold: false,+ italic: false,+ };+ let mut blocks: Vec<PositionedBlock> = [+ HorizontalRule,+ p("a"),+ HorizontalRule,+ HorizontalRule,+ p("b"),+ HorizontalRule,+ ]+ .into_iter()+ .map(PositionedBlock::unlocated)+ .collect();+ dedupe_rules(&mut blocks);+ let kinds: Vec<bool> = blocks+ .iter()+ .map(|b| matches!(b.block, Block::HorizontalRule))+ .collect();+ // Leading + trailing rules gone; the doubled interior run collapsed to one.+ assert_eq!(kinds, vec![false, true, false]);+ }++ #[test]+ fn page_without_projected_lines_falls_back_to_fenced_text() {+ let mut p = page_with(1, vec![]);+ p.text = "hello".into();+ let out = render_document(&[p]);+ assert!(out.contains("```text"));+ assert!(out.contains("hello"));+ }++ #[test]+ fn heading_and_paragraph() {+ let p = page_with(+ 1,+ vec![+ line("My Title For This Test Document", 50.0, 50.0, 18.0, 18.0),+ // Enough body text to dominate the char-weighted body-size+ // mode so the title at 18pt registers as larger-than-body.+ line("First sentence of body prose here.", 50.0, 80.0, 10.0, 10.0),+ line(+ "Second sentence of body prose here.",+ 50.0,+ 92.0,+ 10.0,+ 10.0,+ ),+ line(+ "Third sentence of body prose here.",+ 50.0,+ 104.0,+ 10.0,+ 10.0,+ ),+ ],+ );+ let out = render_document(&[p]);+ assert!(out.contains("# My Title For This Test Document"));+ assert!(out.contains("First sentence of body prose here."));+ }++ #[test]+ fn per_page_markdown_joins_with_separator() {+ let a = page_with(1, vec![line("A page.", 50.0, 80.0, 10.0, 10.0)]);+ let b = page_with(2, vec![line("B page.", 50.0, 80.0, 10.0, 10.0)]);+ let pages = [a, b];+ let per_page = render_pages(&pages);+ assert_eq!(per_page.len(), 2);+ assert!(per_page[0].contains("A page."));+ assert!(per_page[1].contains("B page."));+ // The per-page strings carry no separator on their own.+ assert!(!per_page[0].contains("-----"));+ let out = render_document(&pages);+ assert!(out.contains("-----"));+ assert!(out.find("A page.").unwrap() < out.find("B page.").unwrap());+ }++ #[test]+ fn canonicalize_image_refs_points_duplicates_at_canonical_file() {+ let image = |id: &str, dup: Option<&str>| ExtractedImage {+ id: id.into(),+ name: format!("img_{id}.jpg"),+ page: 1,+ bbox: Rect::default(),+ width: 1,+ height: 1,+ rotation: 0.0,+ format: "jpg".into(),+ path: None,+ duplicate_of: dup.map(Into::into),+ bytes: Arc::new(Vec::new()),+ };+ let mut pages = vec![page_with(1, vec![])];+ pages[0].markdown = "intro\n\n\n\noutro".into();+ let mut full_text = pages[0].markdown.clone();+ let images = vec![image("p1_1", None), image("p2_1", Some("p1_1"))];+ canonicalize_image_refs(&mut pages, &mut full_text, &images);+ assert_eq!(pages[0].markdown, "intro\n\n\n\noutro");+ assert_eq!(full_text, pages[0].markdown);+ }+}diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rsindex abe88d0..c3ffb32 100644--- a/crates/liteparse/src/types.rs+++ b/crates/liteparse/src/types.rs@@ -59,7 +59,6 @@ pub struct DocumentMetadata {/// Represents a single text item extracted from a PDF page,/// including its content, position, size, rotation, and font metadata.#[derive(Debug, Clone, Default, Serialize, Deserialize)]-#[serde(default)]pub struct TextItem {pub text: String,/// Viewport-space coordinates (top-left origin, 72 DPI).@@ -72,56 +71,56 @@ pub struct TextItem {pub font_name: Option<String>,pub font_size: Option<f32>,/// Font size * scale_y from the text matrix — accounts for CTM scaling.- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub font_height: Option<f32>,- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub font_ascent: Option<f32>,- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub font_descent: Option<f32>,- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub font_weight: Option<i32>,- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub font_flags: Option<i32>,/// Sum of glyph widths (using charcode-based lookup when possible).- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub text_width: Option<f32>,/// Whether the font has buggy encoding (private-use codepoints, TT subset, etc.)- #[serde(skip_serializing_if = "std::ops::Not::not")]+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]pub font_is_buggy: bool,/// Whether most characters in this item could not be mapped to Unicode/// (e.g. a Type3 font with no ToUnicode map). The text content is/// PDFium's char-code fallback and does not reflect the rendered glyphs.- #[serde(skip_serializing_if = "std::ops::Not::not")]+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]pub has_unicode_map_error: bool,/// Marked content ID from the PDF structure tree.- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub mcid: Option<i32>,/// Fill color as ARGB hex string (e.g. "ff000000").- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub fill_color: Option<String>,/// Stroke color as ARGB hex string.- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub stroke_color: Option<String>,/// Raw character codes from the PDF content stream. These correspond to/// source glyphs rather than Unicode scalar values, so ligature expansion/// can produce more text characters than entries in this array.- #[serde(skip_serializing_if = "Vec::is_empty")]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub char_codes: Vec<u32>,/// Whether the trailing source space was synthesized by PDFium rather than/// represented by a real space glyph in the PDF content stream.- #[serde(skip_serializing_if = "std::ops::Not::not")]+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]pub trailing_space_generated: bool,/// OCR confidence score (0.0–1.0). None for native PDF text.- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub confidence: Option<f32>,/// Target URI when this item falls inside a hyperlink annotation's/// rectangle. Populated in `extract.rs`; consumed by the markdown emitter.- #[serde(skip_serializing_if = "Option::is_none")]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub link: Option<String>,/// Whether a thin horizontal stroke/rect crosses this item's vertical middle/// band (a strikethrough line). Populated in `extract.rs`; consumed by the/// markdown emitter to wrap the text in `~~…~~`.- #[serde(skip_serializing_if = "std::ops::Not::not")]+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]pub strike: bool,/// Per-word sub-boxes within this item, split on the inter-word spaces seen/// during segment building. A segment groups several words together (it only@@ -131,7 +130,7 @@ pub struct TextItem {/// use only — the public JSON output (`output/json.rs`) builds its own/// item view without it, but it is marshalled across the napi boundary/// and must survive a stage boundary (see `stages`), so it serializes.- #[serde(skip_serializing_if = "Vec::is_empty")]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub words: Vec<WordBox>,}diff --git a/crates/liteparse/tests/stages_compose.rs b/crates/liteparse/tests/stages_compose.rsindex ff49a27..b85a819 100644--- a/crates/liteparse/tests/stages_compose.rs+++ b/crates/liteparse/tests/stages_compose.rs@@ -215,7 +215,8 @@ async fn compose(parser: &LiteParse, input: PdfInput) -> ParseResult {let producer = document.meta_text("Producer").filter(|v| !v.is_empty());let doc_meta = want_doc_meta.then(|| {if repaired.is_some() {- let source = stages::open(&lib, &input, password, &[]).unwrap();+ let source =+ stages::open(&lib, &input, password, &config.page_orientation_corrections).unwrap();return stages::document_metadata(&input, &source);}stages::document_metadata(&input, &document)@@ -229,49 +230,65 @@ async fn compose(parser: &LiteParse, input: PdfInput) -> ParseResult {&parser.extract_request(target_pages.as_deref(), config.max_pages),).unwrap();- let stages::ExtractedPages {- pages,- page_errors,- mut images,- image_error_count,- flattened_form_widgets,- flattened_page_numbers,- } = extracted;+ let ocr_render_options = parser.ocr_render_options(grayscale, &extracted);+ let screenshot_options = parser.screenshot_options(config.continue_on_page_error);+ // Extraction may have flattened form widgets into the open document;+ // screenshots that paint form fields need a document it did not touch.+ let pristine = (config.extract_screenshots+ && stages::screenshots_need_pristine_document(&extracted, &screenshot_options))+ .then(|| {+ stages::open(+ &lib,+ document_input,+ password,+ &config.page_orientation_corrections,+ )+ .unwrap()+ });+ let analysis_document = pristine.as_ref().unwrap_or(&document);let complexity: Vec<stages::PageComplexityStats> = if config.include_complexity {- pages+ extracted+ .pages.iter()- .map(|page| stages::page_complexity(&document, page).unwrap())+ .map(|page| stages::page_complexity(analysis_document, page).unwrap()).collect()} else {Vec::new()};let screenshots = if config.extract_screenshots {- let page_numbers: Vec<u32> = pages.iter().map(|p| p.page_number as u32).collect();- stages::screenshots(- &document,- Some(&page_numbers),- &stages::ScreenshotOptions {- dpi: config.dpi,- detect_rects: config.detect_screenshot_rects,- render_form_fields: config.render_form_fields,- continue_on_page_error: config.continue_on_page_error,- },- )- .unwrap()+ let page_numbers: Vec<u32> = extracted+ .pages+ .iter()+ .map(|p| p.page_number as u32)+ .collect();+ stages::screenshots(analysis_document, Some(&page_numbers), &screenshot_options).unwrap()} else {Vec::new()};+ drop(pristine);drop(document);drop(lib);- // Boundary 1: the extracted page set crosses a (simulated) process- // boundary before OCR.- let mut pages: Vec<Page> = round_trip(pages, "Vec<Page> after extract");+ // Boundary 1: the whole extraction result crosses a (simulated) process+ // boundary before OCR. Image payloads are the documented exception (they+ // travel out of band, keyed by id), so they are carried across by hand.+ let image_bytes: Vec<_> = extracted.images.iter().map(|i| i.bytes.clone()).collect();+ let extracted = round_trip(extracted, "ExtractedPages after extract");+ let stages::ExtractedPages {+ pages,+ page_errors,+ mut images,+ image_error_count,+ ..+ } = extracted;+ for (image, bytes) in images.iter_mut().zip(image_bytes) {+ image.bytes = bytes;+ }+ let mut pages: Vec<Page> = pages;// ── OCR: render rounds (pdfium) → recognize (async) → merge (pure)if let Some(engine) = engine {- let options =- parser.ocr_render_options(grayscale, flattened_form_widgets, &flattened_page_numbers);+ let options = ocr_render_options;let ocr_input = repaired.as_ref().unwrap_or(&input);let mut start = 0;while start < pages.len() {@@ -329,11 +346,7 @@ async fn compose(parser: &LiteParse, input: PdfInput) -> ParseResult {assert_eq!(back.heading_map, signals.heading_map);assert_eq!(back.header_footer, signals.header_footer);let signals = back;- let options = stages::BlockOptions {- outline: &outline,- image_mode: config.image_mode,- keep_headers_footers: config.keep_headers_footers,- };+ let options = parser.block_options(&outline);for page in parsed.iter_mut() {let blocks = stages::extract_blocks(page, &signals, &options);let blocks = blocks.map(|b| round_trip(b, "Vec<PositionedBlock>"));@@ -575,6 +588,87 @@ async fn parse_equals_composed_stages_with_form_fields() {assert_parse_equals_composition(config, &fixture("filled_acroform.pdf"), true).await;}+/// Form fields plus screenshots that paint form fields: the one combination+/// where `parse()` reopens a pristine document after extraction flattened+/// widgets, so the composed pipeline must do the same.+#[tokio::test]+#[serial]+async fn parse_equals_composed_stages_with_form_fields_and_rendered_screenshots() {+ let config = LiteParseConfig {+ ocr_enabled: false,+ quiet: true,+ output_format: OutputFormat::Markdown,+ extract_form_fields: true,+ extract_screenshots: true,+ render_form_fields: true,+ include_complexity: true,+ ..LiteParseConfig::default()+ };+ let result =+ assert_parse_equals_composition(config, &fixture("filled_acroform.pdf"), false).await;+ assert!(+ !result.screenshots.is_empty(),+ "fixture should produce screenshots"+ );+}++/// An explicit OCR page selection bypasses the complexity gate: a text-dense+/// page that scoring would never send to OCR is rastered anyway, and pages+/// outside the selection are skipped.+#[tokio::test]+#[serial]+async fn render_for_ocr_selection_overrides_complexity_gate() {+ let parser = LiteParse::new(everything_config());+ let lib = Library::init();+ let document = stages::open(&lib, &PdfInput::Path(fixture("sample.pdf")), None, &[]).unwrap();+ let pages = stages::extract(&document, &parser.extract_request(None, usize::MAX))+ .unwrap()+ .pages;+ let first = pages[0].page_number as u32;+ let gated = stages::render_for_ocr(&document, &pages, 0, &stages::OcrRenderOptions::default())+ .unwrap()+ .0;+ assert!(+ gated.iter().all(|r| r.page_number as u32 != first),+ "sample.pdf page 1 has native text and should not be rastered by the scoring gate"+ );+ let options = stages::OcrRenderOptions {+ selection: Some([first].into_iter().collect()),+ ..stages::OcrRenderOptions::default()+ };+ let (rasters, next) = stages::render_for_ocr(&document, &pages, 0, &options).unwrap();+ assert_eq!(next, pages.len());+ assert_eq!(rasters.len(), 1);+ assert_eq!(rasters[0].page_number as u32, first);+ assert!(rasters[0].has_native_text);+ assert!(rasters[0].width > 0 && rasters[0].height > 0);+ // The raster's pixels survive the boundary as base64, not a number array.+ let json = serde_json::to_value(&rasters[0]).unwrap();+ assert!(json["pixels"].is_string());+}++/// A hand-built outcome for a page that is not being merged is an error,+/// not a panic.+#[test]+fn merge_ocr_rejects_outcome_for_unknown_page() {+ let mut pages: Vec<Page> = Vec::new();+ let outcome = stages::PageOcrOutcome {+ page_number: 7,+ dpi: 150.0,+ has_native_text: false,+ image_rects: Vec::new(),+ results: vec![OcrResult {+ text: "x".into(),+ bbox: [0.0, 0.0, 10.0, 10.0],+ confidence: 1.0,+ polygon: None,+ }],+ error: None,+ };+ let err = stages::merge_ocr(&mut pages, vec![outcome], false).unwrap_err();+ assert!(err.to_string().contains("page 7"), "{err}");+}+/// Sanity check on the round-trip helper itself: a page with every optional/// field populated survives serde unchanged, including the fields the public/// JSON output omits (word boxes, graphics, structure nodes, image refs).