A fast, helpful, and open-source document parser
Expose the parse pipeline as public stage functions
Logan Markewich3bc9bcac85f2
diff --git a/crates/liteparse/src/layout.rs b/crates/liteparse/src/layout.rsindex 790e456..13b1d04 100644--- a/crates/liteparse/src/layout.rs+++ b/crates/liteparse/src/layout.rs@@ -12,7 +12,7 @@//! during serialization, so a heading serializes as `{kind, text, level, bbox}`//! rather than a wall of nulls.-use serde::Serialize;+use serde::{Deserialize, Serialize};use crate::markdown_layout::{Block, Cell, PositionedBlock, SpanCell};use crate::types::{ParsedPage, Rect};@@ -22,7 +22,7 @@ use crate::types::{ParsedPage, Rect};/// `bbox` is `None` for cells with no ink behind them — padding inserted to/// square off a ragged grid, or halves of a merged run split at an estimated/// position rather than an observed boundary.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct LayoutCell {pub text: String,#[serde(skip_serializing_if = "Option::is_none")]@@ -63,11 +63,11 @@ impl From<&SpanCell> for LayoutCell {/// `kind` discriminates the block; see each field for which kinds populate it./// Blocks appear in reading order, matching the order the markdown renderer/// emits them, so the Nth block here is the Nth block of that page's markdown.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct LayoutBlock {/// One of `heading`, `paragraph`, `list_item`, `code`, `table`,/// `merged_table`, `grid_fallback`, `rule`, `figure`.- pub kind: &'static str,+ pub kind: String,/// Rendered text for the text-bearing kinds (`heading`, `paragraph`,/// `list_item`). Table text lives in `header`/`rows`; code and grid text in/// `lines`.@@ -78,9 +78,9 @@ pub struct LayoutBlock {pub level: Option<u8>,/// Whether the block's text is uniformly bold / italic. `paragraph` and/// `list_item` only; omitted when false.- #[serde(skip_serializing_if = "std::ops::Not::not")]+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]pub bold: bool,- #[serde(skip_serializing_if = "std::ops::Not::not")]+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]pub italic: bool,/// `list_item`: whether the list is ordered, and the original marker as it/// appeared on the page (`138.`, `iii)`, `•`).@@ -123,7 +123,7 @@ impl LayoutBlock {/// Base value with every variant-specific field cleared.fn of(kind: &'static str, bbox: Option<Rect>) -> Self {LayoutBlock {- kind,+ kind: kind.to_string(),text: None,level: None,bold: false,@@ -236,7 +236,7 @@ pub(crate) fn blocks_for_page(page: &ParsedPage, blocks: &[PositionedBlock]) ->/// map it through).fn remap_to_page_frame(blocks: &mut [LayoutBlock], frames: &[(Rect, Rect)]) {for block in blocks.iter_mut() {- if matches!(block.kind, "figure" | "rule") {+ if matches!(block.kind.as_str(), "figure" | "rule") {continue;}remap_opt(&mut block.bbox, frames);diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rsindex 106f713..2a4f499 100644--- a/crates/liteparse/src/lib.rs+++ b/crates/liteparse/src/lib.rs@@ -3,6 +3,10 @@//! This crate is the core Rust library. Language bindings for Node.js, Python,//! and WebAssembly re-export the same types with language-idiomatic wrappers.//!+//! [`LiteParse::parse`] runs a fixed pipeline; the [`stages`] module exposes+//! each step of it as a public function for callers that want to sequence+//! the pipeline themselves.+//!// ── Public API re-exports ──────────────────────────────────────────────pub use config::{DEFAULT_PAGE_BATCH_SIZE, LiteParseConfig, OutputFormat};@@ -23,6 +27,7 @@ pub mod layout;pub mod parser;pub mod raw_text;pub mod search;+pub mod stages;pub mod types;// ── Internal modules (available for binding crates, hidden from docs) ──diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rsindex 59c483f..89ab502 100644--- a/crates/liteparse/src/markdown_layout/blocks.rs+++ b/crates/liteparse/src/markdown_layout/blocks.rs@@ -2,13 +2,14 @@ use super::inline::escape_inline;use super::paragraphs::{ParaAccum, is_soft_hyphen_break};use super::tables::escape_table_cell;use crate::types::Rect;+use serde::{Deserialize, Serialize};/// One table cell: rendered text plus, when the cell came from real page/// content, the region it occupied. `bbox` is `None` for cells that exist only/// to square off a ragged grid (padding inserted when rows disagree on column/// count) — those occupy no ink on the page, and reporting a rect for them/// would invent geometry the classifier never saw.-#[derive(Debug, Clone, Default)]+#[derive(Debug, Clone, Default, Serialize, Deserialize)]pub struct Cell {pub text: String,pub bbox: Option<Rect>,@@ -84,7 +85,7 @@ impl PartialEq<String> for Cell {/// merge structure a producer *declared* — but both carry the region the cell/// occupies when the producer knows it, so merged tables ground exactly like/// plain ones.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct SpanCell {pub text: String,pub colspan: u16,@@ -142,7 +143,7 @@ impl Eq for SpanCell {}/// Coarse block representation: the output of page classification, consumed by/// `render_blocks` to produce the final markdown string.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub enum Block {Heading {level: u8,@@ -250,7 +251,7 @@ fn wrap_emphasis(text: &str, bold: bool, italic: bool) -> String {/// a wrapped heading or a multi-line paragraph reports the whole band it/// occupies. It is `None` only for blocks with no page geometry behind them —/// synthesized content, or sources that never carried coordinates.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct PositionedBlock {pub block: Block,pub bbox: Option<Rect>,diff --git a/crates/liteparse/src/ocr/mod.rs b/crates/liteparse/src/ocr/mod.rsindex a65685b..dcfb2e5 100644--- a/crates/liteparse/src/ocr/mod.rs+++ b/crates/liteparse/src/ocr/mod.rs@@ -1,5 +1,7 @@use std::pin::Pin;+use serde::{Deserialize, Serialize};+#[cfg(not(target_arch = "wasm32"))]pub mod http_simple;#[cfg(all(feature = "oar-ocr", not(target_arch = "wasm32")))]@@ -8,7 +10,7 @@ pub mod oar;pub mod tesseract;/// A single word-level OCR result with bounding box and confidence.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct OcrResult {pub text: String,/// Bounding box in pixel coordinates: [x1, y1, x2, y2] (left, top, right, bottom).diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex 6ea2cbd..14aa440 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -3,9 +3,9 @@ use std::sync::Arc;use crate::error::LiteParseError;use crate::ocr::{OcrEngine, OcrOptions, OcrResult};-use crate::types::{CutAxis, Page, ParsedPage, Region, RegionKind, TextItem};+use crate::types::{CutAxis, Page, ParsedPage, Rect, Region, RegionKind, TextItem};use pdfium::{Document, ImageBounds};-use serde::Serialize;+use serde::{Deserialize, Serialize};/// Minimum dark filled-path area (pt², ~72 DPI page space) not covered by/// native text before a page is sent to OCR. 400 pt² is roughly one word at@@ -37,14 +37,24 @@ const MIN_COLUMN_ITEM_FRACTION: f32 = 0.15;/// `DenseGraphics`.const DENSE_GRAPHICS_MIN_COVERAGE: f32 = 0.2;-/// Owned page bitmap prepared for OCR. Indices refer to positions in the `pages` slice.-pub(crate) struct RenderedPage {- pub idx: usize,+/// Owned page bitmap prepared for OCR, plus the page facts the merge needs+/// to judge the engine's output. Produced by [`render_pages_for_ocr`],+/// consumed by [`recognize_rasters`]; the non-pixel facts are carried into+/// the [`PageOcrOutcome`] so [`merge_ocr_results`] never needs the raster or+/// the document again.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct OcrRaster {+ /// Position in the `pages` slice the raster was rendered from.+ pub page_idx: usize,+ /// 1-based source page number, for logs and for callers that key by+ /// page rather than by slice position.+ pub page_number: usize,/// Tightly-packed pixels: 1 byte/px (grayscale) or 3 (RGB).pub pixels: Vec<u8>,pub width: u32,pub height: u32,- /// DPI this page was actually rendered at.+ /// DPI this page was actually rendered at (the long-edge cap can lower it+ /// below the requested DPI). Pixel → point scale is `72 / dpi`.pub dpi: f32,/// The page has real native text (it was not sent to OCR as scanned,/// blank, or garbled). OCR is then an enrichment pass over native text,@@ -53,13 +63,38 @@ pub(crate) struct RenderedPage {/// Embedded raster figure rects (viewport pt, excluding full-page/// backgrounds), so the merge can judge each figure's OCR output as a/// group — a chart's axis/legend labels are dropped together.- pub image_rects: Vec<ImageBounds>,+ pub image_rects: Vec<Rect>,+}++/// How [`render_pages_for_ocr`] picks and rasterizes pages.+#[derive(Debug, Clone, Default)]+pub struct OcrRenderOptions {+ /// Stop after this many rasters (`0` = no limit), so a long document can+ /// be processed in bounded rounds; the function returns where to resume.+ pub max_rasters: usize,+ /// Requested render resolution; clamped per page by the long-edge cap.+ pub dpi: f32,+ /// Emit 1 byte/px luma instead of RGB (see `OcrEngine::prefers_grayscale`).+ pub grayscale: bool,+ /// Paint form-field appearances into the raster (runs document actions).+ pub render_form_fields: bool,+ /// Keep going when one page fails to render (it keeps its native text).+ pub continue_on_page_error: bool,+ /// Pages extraction flattened widget annotations on, which must be+ /// flattened again on this (reopened) document so the raster shows what+ /// extraction saw. See `ExtractedPages::flattened_page_numbers`.+ pub reflatten_pages: HashSet<u32>,+ /// Explicit 1-based page selection. `None` lets the page-complexity+ /// scoring decide which pages need OCR (the default parse behaviour);+ /// `Some` renders exactly these pages, whatever the scoring says, so a+ /// caller with its own routing can override it.+ pub selection: Option<HashSet<u32>>,}/// Why a page was flagged as needing more than the cheap text-only path./// Multiple reasons can apply to one page (e.g. a sparse page whose little/// text is also garbled). Empty exactly when `needs_ocr` is false.-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]#[serde(rename_all = "kebab-case")]pub enum ComplexityReason {/// A single raster covers essentially the whole page and there is little or@@ -104,7 +139,7 @@ impl ComplexityReason {}}-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct PageComplexityStats {pub page_number: usize,pub text_length: usize,@@ -438,7 +473,7 @@ pub fn calculate_native_page_complexity(/// that the text-only path may mangle reading order or structure, so a caller/// can route the page to a higher-accuracy pipeline. Same leniency contract/// as `ComplexityReason`: new variants will be added, treat unknowns kindly.-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]#[serde(rename_all = "kebab-case")]pub enum LayoutComplexityReason {/// The XY-cut layout tree splits the page into two or more side-by-side@@ -464,7 +499,7 @@ impl LayoutComplexityReason {}}-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct LayoutComplexityStats {/// Side-by-side text columns found by the XY-cut layout pass (1 = single/// column). Only subtrees holding a substantial share of the page's text@@ -587,31 +622,39 @@ fn count_columns(region: &Region, total_items: usize) -> usize {}}-/// Render pages that need OCR from an already-open document.-///-/// The pdfium `Document` holds raw pointers that are not `Send`, so callers must-/// drop it before awaiting the OCR engine.-////// Cap on the rendered long edge (in pixels) for OCR page rasters/// to avoid excessive memory usage.pub(crate) const MAX_OCR_RENDER_LONG_EDGE_PX: f32 = 4096.0;-/// Render the pages in `pages[start..]` that need OCR, stopping once-/// `max_rasters` of them have been rendered (`0` means no limit). Returns the-/// rasters plus the index to resume scanning from, so a caller can process a-/// long document in bounded rounds.-#[allow(clippy::too_many_arguments)]-pub(crate) fn render_pages_for_ocr(+/// Render the pages in `pages[start..]` that need OCR from an already-open+/// document, stopping once `options.max_rasters` of them have been rendered.+/// Returns the rasters plus the index to resume scanning from, so a caller+/// can process a long document in bounded rounds.+///+/// The pdfium `Document` holds raw pointers that are not `Send`, so callers+/// must drop it before awaiting the OCR engine.+pub fn render_pages_for_ocr(document: &Document,pages: &[Page],start: usize,- max_rasters: usize,- dpi: f32,- grayscale: bool,- render_form_fields: bool,- continue_on_page_error: bool,- flatten_page_numbers: &std::collections::HashSet<u32>,-) -> Result<(Vec<RenderedPage>, usize), LiteParseError> {+ options: &OcrRenderOptions,+) -> Result<(Vec<OcrRaster>, usize), LiteParseError> {+ let OcrRenderOptions {+ max_rasters,+ dpi,+ grayscale,+ render_form_fields,+ continue_on_page_error,+ reflatten_pages: flatten_page_numbers,+ selection,+ } = options;+ let (max_rasters, dpi, grayscale, render_form_fields, continue_on_page_error) = (+ *max_rasters,+ *dpi,+ *grayscale,+ *render_form_fields,+ *continue_on_page_error,+ );let mut rendered = Vec::new();// With `render_form_fields`, draw form-field appearances into the OCR// raster so filled-in form values are visible to the OCR engine (matches@@ -625,7 +668,12 @@ pub(crate) fn render_pages_for_ocr(}let mut next_start = pages.len();for (idx, page) in pages.iter().enumerate().skip(start) {- let page_render = (|| -> Result<Option<RenderedPage>, LiteParseError> {+ if let Some(selection) = selection+ && !selection.contains(&(page.page_number as u32))+ {+ continue;+ }+ let page_render = (|| -> Result<Option<OcrRaster>, LiteParseError> {let page_index = (page.page_number - 1) as i32;// Text extraction may have flattened THIS page's widget// annotations into page content. When the caller hands us a@@ -644,7 +692,10 @@ pub(crate) fn render_pages_for_ocr(};let page_complexity = calculate_page_complexity(page, &page_obj)?;- if !page_complexity.needs_ocr {+ // An explicit selection overrides the scoring's OCR gate; the+ // scoring still decides `has_native_text`, which drives the+ // merge's artifact filters.+ if selection.is_none() && !page_complexity.needs_ocr {return Ok(None);}@@ -687,13 +738,20 @@ pub(crate) fn render_pages_for_ocr(!(b.width > pw * MAX_IMAGE_PAGE_COVERAGE&& b.height > ph * MAX_IMAGE_PAGE_COVERAGE)})+ .map(|b| Rect {+ x: b.x,+ y: b.y,+ width: b.width,+ height: b.height,+ }).collect()} else {Vec::new()};- Ok(Some(RenderedPage {- idx,+ Ok(Some(OcrRaster {+ page_idx: idx,+ page_number: page.page_number,pixels,width,height,@@ -723,27 +781,49 @@ pub(crate) fn render_pages_for_ocr(Ok((rendered, next_start))}-/// Run OCR on pre-rendered page bitmaps and merge results into `pages`.-pub(crate) async fn ocr_and_merge_rendered(- pages: &mut [Page],- rendered: Vec<RenderedPage>,+/// One page's recognition outcome: what the engine returned for one+/// [`OcrRaster`], plus the raster's page facts so the merge is a pure+/// function of `(pages, outcomes)`. The pixels are gone — this is the small+/// payload that crosses a stage boundary, not the bitmap.+///+/// `result` is `Err(message)` when the engine failed on this page; the merge+/// counts failures to tell a systemic engine problem from a flaky page.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct PageOcrOutcome {+ /// Position in the `pages` slice the raster came from.+ pub page_idx: usize,+ /// 1-based source page number.+ pub page_number: usize,+ /// DPI the raster was rendered at; pixel → point scale is `72 / dpi`.+ pub dpi: f32,+ pub has_native_text: bool,+ pub image_rects: Vec<Rect>,+ /// Engine output in raster pixel coordinates.+ pub result: Result<Vec<OcrResult>, String>,+}++/// Run `ocr_engine` over pre-rendered page bitmaps, at most `num_workers`+/// pages at a time, returning one outcome per raster in the order given.+/// Pure recognition: nothing is merged, and a failed page is reported as an+/// `Err` outcome rather than aborting the batch.+pub async fn recognize_rasters(+ rendered: Vec<OcrRaster>,ocr_engine: Arc<dyn OcrEngine>,ocr_language: &str,num_workers: usize,- ocr_failure_fatal: bool,-) -> Result<(), LiteParseError> {+) -> Vec<PageOcrOutcome> {type OcrTaskResult = Result<Vec<OcrResult>, Box<dyn std::error::Error + Send + Sync>>;// Browser WASM uses the JavaScript event loop. It has no Tokio runtime or// blocking thread pool. Run each JavaScript OCR callback directly so the// returned Promise can make progress on the browser event loop.#[cfg(target_arch = "wasm32")]- let task_results: Vec<(usize, usize, f32, (bool, Vec<ImageBounds>), OcrTaskResult)> = {+ let task_results: Vec<(usize, usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {let _ = num_workers;let mut results = Vec::with_capacity(rendered.len());for r in rendered {- let idx = r.idx;- let page_number = pages[idx].page_number;+ let idx = r.page_idx;+ let page_number = r.page_number;let page_dpi = r.dpi;let native = (r.has_native_text, r.image_rects.clone());let options = OcrOptions {@@ -772,7 +852,7 @@ pub(crate) async fn ocr_and_merge_rendered(// pass deadlocks. Acquiring the permit asynchronously parks the lightweight// task instead, so only `num_workers` blocking threads are ever consumed.#[cfg(not(target_arch = "wasm32"))]- let task_results: Vec<(usize, usize, f32, (bool, Vec<ImageBounds>), OcrTaskResult)> = {+ let task_results: Vec<(usize, usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {let num_workers = num_workers.max(1);let semaphore = Arc::new(tokio::sync::Semaphore::new(num_workers));let mut handles = Vec::with_capacity(rendered.len());@@ -783,11 +863,11 @@ pub(crate) async fn ocr_and_merge_rendered(let engine = ocr_engine.clone();let sem = semaphore.clone();let language = ocr_language.to_string();- let page_number = pages[r.idx].page_number;+ let page_number = r.page_number;let rt_handle = handle.clone();handles.push((- r.idx,+ r.page_idx,page_number,r.dpi,(r.has_native_text, r.image_rects.clone()),@@ -830,8 +910,34 @@ pub(crate) async fn ocr_and_merge_rendered(results};- // Phase 3: collect results and merge into pages.+ task_results+ .into_iter()+ .map(+ |(page_idx, page_number, dpi, (has_native_text, image_rects), result)| PageOcrOutcome {+ page_idx,+ page_number,+ dpi,+ has_native_text,+ image_rects,+ result: result.map_err(|e| e.to_string()),+ },+ )+ .collect()+}+/// Merge recognition outcomes into `pages`, in place. Pure: needs neither the+/// document nor the rasters, only the outcomes [`recognize_rasters`] produced+/// (or an equivalent a caller built from its own OCR engine's output).+///+/// Per page, unusable native text (garbled fonts, unmappable Type3) is dropped+/// first so OCR can replace it; then each result is filtered against rule+/// artifacts, chart figures and overlap with surviving native text before+/// being appended as an `OCR`-font [`TextItem`] in viewport points.+pub fn merge_ocr_results(+ pages: &mut [Page],+ outcomes: Vec<PageOcrOutcome>,+ ocr_failure_fatal: bool,+) -> Result<(), LiteParseError> {// Track OCR task outcomes so we can distinguish a systemic failure (e.g.// missing Tesseract language data, which fails identically on every page)// from incidental per-page failures. Without this, every page logs the same@@ -844,19 +950,26 @@ pub(crate) async fn ocr_and_merge_rendered(// enrichment but already has all its text. We must only fail loud when OCR// failure destroyed a sparse page's likely primary text source — otherwise// a broken OCR setup would abort perfectly good native-text documents.- let total_tasks = task_results.len();+ let total_tasks = outcomes.len();let mut failed_tasks = 0usize;let mut failed_sparse_text_page = false;let mut first_error: Option<String> = None;- for (idx, page_number, page_dpi, (has_native_text, image_rects), result) in task_results {+ for outcome in outcomes {+ let PageOcrOutcome {+ page_idx: idx,+ page_number,+ dpi: page_dpi,+ has_native_text,+ image_rects,+ result,+ } = outcome;let ocr_results: Vec<OcrResult> = match result {Ok(results) => results,- Err(e) => {+ Err(msg) => {failed_tasks += 1;failed_sparse_text_page |= page_has_sparse_native_text(&pages[idx]);if first_error.is_none() {- let msg = e.to_string();eprintln!("[ocr] failed for page {}: {}", page_number, msg);first_error = Some(msg);}@@ -1334,7 +1447,7 @@ const CHART_MIN_RESULTS: usize = 4;/// tokens for the figure to count as a chart.const CHART_LABEL_FRACTION: f32 = 0.5;-fn box_center_in(rect: &ImageBounds, x: f32, y: f32, w: f32, h: f32) -> bool {+fn box_center_in(rect: &Rect, x: f32, y: f32, w: f32, h: f32) -> bool {let cx = x + w / 2.0;let cy = y + h / 2.0;cx >= rect.x && cx <= rect.x + rect.width && cy >= rect.y && cy <= rect.y + rect.height@@ -1358,7 +1471,7 @@ fn is_chart_label(text: &str) -> bool {/// For each figure rect, whether its OCR results look like a chart's/// axes/ticks/legend rather than text worth keeping.-fn chart_like_images(rects: &[ImageBounds], results: &[OcrResult], scale_factor: f32) -> Vec<bool> {+fn chart_like_images(rects: &[Rect], results: &[OcrResult], scale_factor: f32) -> Vec<bool> {rects.iter().map(|rect| {@@ -1469,7 +1582,7 @@ mod tests {assert!(!is_chart_label("Year"));assert!(!is_chart_label("Fruit Production in British Columbia"));- let rect = ImageBounds {+ let rect = Rect {x: 50.0,y: 100.0,width: 200.0,@@ -1996,11 +2109,12 @@ mod tests {assert!(matches!(garbled_scope(&page), GarbledScope::WholePage));}- fn make_rendered(idx: usize) -> RenderedPage {- RenderedPage {+ fn make_rendered(idx: usize) -> OcrRaster {+ OcrRaster {has_native_text: false,image_rects: Vec::new(),- idx,+ page_idx: idx,+ page_number: idx + 1,// 1x1 grayscale pixel; the engine never inspects it.pixels: vec![0u8],width: 1,@@ -2009,6 +2123,20 @@ mod tests {}}+ /// The pre-split entry point, kept for these tests: recognition followed+ /// by the merge, exactly as `parse()` composes the two stages.+ async fn ocr_and_merge_rendered(+ pages: &mut [Page],+ rendered: Vec<OcrRaster>,+ engine: Arc<dyn OcrEngine>,+ language: &str,+ num_workers: usize,+ ocr_failure_fatal: bool,+ ) -> Result<(), LiteParseError> {+ let outcomes = recognize_rasters(rendered, engine, language, num_workers).await;+ merge_ocr_results(pages, outcomes, ocr_failure_fatal)+ }+// A page that already has substantial native text coverage, as would be the// case for a native-text PDF page that was only rendered for OCR because it// also contains an image.diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rsindex c31d229..2224bef 100644--- a/crates/liteparse/src/output/markdown.rs+++ b/crates/liteparse/src/output/markdown.rs@@ -1,8 +1,6 @@use crate::config::ImageMode;-use crate::markdown_layout::{- PositionedBlock, build_heading_map, classify_page_with_filters, compute_body_size,- compute_header_footer_set, detect_single_page_chrome, render_blocks, splice_soft_hyphens,-};+use crate::markdown_layout::PositionedBlock;+use crate::stages;use crate::types::{OutlineTarget, ParsedPage};/// Format parsed pages as markdown.@@ -67,59 +65,15 @@ pub fn classify_document(return Vec::new();}- let body_size = compute_body_size(pages);- let heading_map = build_heading_map(pages, body_size);- let header_footer = if keep_headers_footers {- Default::default()- } else {- compute_header_footer_set(pages)+ let signals = stages::document_signals(pages, keep_headers_footers);+ let options = stages::BlockOptions {+ outline,+ image_mode,+ keep_headers_footers,};-pages.iter()- .map(|page| {- if page.projected_lines.is_empty() {- return None;- }-- // Filter outline entries to this page so the classifier's y/title- // match is a O(entries_on_page) scan per line, not O(whole doc).- let target_index = (page.page_number as i32).saturating_sub(1);- let page_outline: Vec<OutlineTarget> = outline- .iter()- .filter(|e| e.page_index == target_index)- .cloned()- .collect();- let chrome_indices = if keep_headers_footers {- Default::default()- } else {- detect_single_page_chrome(page, body_size)- };- let mut blocks = classify_page_with_filters(- page,- &heading_map,- &header_footer,- &page_outline,- image_mode,- &chrome_indices,- );- // A page whose only surviving blocks are horizontal rules (all its- // text was stripped as chrome) should render empty, not as a stack- // of bare `---` separators.- let has_content = blocks.iter().any(|b| {- !matches!(- b.block,- crate::markdown_layout::Block::HorizontalRule- | crate::markdown_layout::Block::Figure { .. }- )- });- if !has_content {- blocks- .retain(|b| !matches!(b.block, crate::markdown_layout::Block::HorizontalRule));- }- dedupe_rules(&mut blocks);- Some(splice_soft_hyphens(blocks))- })+ .map(|page| stages::extract_blocks(page, &signals, &options)).collect()}@@ -133,18 +87,7 @@ pub fn render_classified(pages.iter().zip(classified)- .map(|(page, blocks)| match blocks {- Some(blocks) => render_blocks(blocks),- None => {- let mut out = String::from("```text\n");- out.push_str(&page.text);- if !page.text.ends_with('\n') {- out.push('\n');- }- out.push_str("```");- out- }- })+ .map(|(page, blocks)| stages::render_page_markdown(page, blocks.as_deref())).collect()}@@ -154,7 +97,7 @@ pub fn render_classified(/// two sources — vector-graphics detection and decorative divider text — and/// doubling up reads as sloppy output to a human, while carrying no extra/// structure for an LLM.-fn dedupe_rules(blocks: &mut Vec<crate::markdown_layout::PositionedBlock>) {+pub(crate) fn dedupe_rules(blocks: &mut Vec<crate::markdown_layout::PositionedBlock>) {use crate::markdown_layout::Block::HorizontalRule;while matches!(blocks.first().map(|b| &b.block), Some(HorizontalRule)) {blocks.remove(0);diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex ed364ce..6d0b14c 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -1,17 +1,23 @@+//! The `LiteParse` orchestrator: input resolution, configuration, and the+//! fixed parse sequence.+//!+//! Invariant: every step of `parse()` goes through [`crate::stages`]. This+//! file owns the *sequence* and the config-to-stage plumbing; it must not+//! reach into `extract`, `ocr_merge`, `projection` or `markdown_layout`+//! directly, or a caller sequencing the stages themselves could no longer+//! reproduce `parse()`. `tests/stages_compose.rs` asserts that equivalence.+use crate::config::{LiteParseConfig, parse_target_pages};#[cfg(not(target_arch = "wasm32"))]use crate::conversion;use crate::error::LiteParseError;-use crate::extract;use crate::ocr::OcrEngine;#[cfg(not(target_arch = "wasm32"))]use crate::ocr::http_simple::HttpOcrEngine;#[cfg(feature = "tesseract")]use crate::ocr::tesseract::TesseractOcrEngine;-use crate::ocr_merge;-use crate::output::markdown;-use crate::projection;-use crate::render;++use crate::stages;use crate::types::{DocumentMetadata, ExtractedImage, OutlineTarget, Page, PageError, ParsedPage, PdfInput,ScreenshotRect, XfaPacket,@@ -120,8 +126,9 @@ fn write_extracted_images(/// by its own placement id (``), but only the canonical/// file is written to disk (see `write_extracted_images`), so duplicate/// placements must reference the canonical name, matching the resolution the-/// LlamaParse worker applies via `resolveOutputImageName`.-fn rewrite_duplicate_image_refs(+/// LlamaParse worker applies via `resolveOutputImageName`. Public as+/// [`stages::canonicalize_image_refs`].+pub(crate) fn rewrite_duplicate_image_refs(pages: &mut [ParsedPage],full_text: &mut String,images: &[ExtractedImage],@@ -247,27 +254,31 @@ fn apply_layout(if !wants_markdown && !config.extract_blocks {return None;}- let classified = markdown::classify_document(- parsed_pages,+ let signals = stages::document_signals(parsed_pages, config.keep_headers_footers);+ let block_options = stages::BlockOptions {outline,- config.image_mode,- config.keep_headers_footers,- );- if config.extract_blocks {- for (page, blocks) in parsed_pages.iter_mut().zip(&classified) {+ image_mode: config.image_mode,+ keep_headers_footers: config.keep_headers_footers,+ };+ let mut page_md = Vec::with_capacity(parsed_pages.len());+ for page in parsed_pages.iter_mut() {+ let blocks = stages::extract_blocks(page, &signals, &block_options);+ if config.extract_blocks {// A page with no structural decomposition reports an empty list,// not `None` — extraction *was* enabled, there was just nothing to// decompose.- page.blocks = Some(crate::layout::blocks_for_page(+ page.blocks = Some(stages::layout_blocks(page,blocks.as_deref().unwrap_or_default(),));}+ if wants_markdown {+ page_md.push(stages::render_page_markdown(page, blocks.as_deref()));+ }}if !wants_markdown {return None;}- let page_md = markdown::render_classified(parsed_pages, &classified);let md = page_md.join("\n\n-----\n\n");for (page, page_md) in parsed_pages.iter_mut().zip(page_md) {page.markdown = page_md;@@ -324,12 +335,12 @@ impl LiteParse {input: &PdfInput,password: Option<&str>,) -> Result<pdfium::Document<'lib>, LiteParseError> {- let document = extract::load_document_from_input(lib, input, password)?;- extract::apply_page_orientation_corrections(- &document,+ stages::open(+ lib,+ input,+ password,&self.config.page_orientation_corrections,- )?;- Ok(document)+ )}fn validate_output_config(&self) -> Result<(), LiteParseError> {@@ -342,6 +353,118 @@ impl LiteParse {Ok(())}+ /// The extract stage's request for a full parse, derived from config.+ /// One place, so a caller composing the stages can build the same+ /// request from the same config (`tests/stages_compose.rs` does exactly that).+ pub fn extract_request<'a>(+ &'a self,+ target_pages: Option<&'a [u32]>,+ max_pages: usize,+ ) -> stages::ExtractRequest<'a> {+ let markdown = self.config.output_format == crate::config::OutputFormat::Markdown;+ stages::ExtractRequest {+ target_pages,+ max_pages,+ extract_links: self.config.extract_links && markdown,+ glyph_resolver: self.glyph_resolver.as_deref(),+ output: stages::ExtractionOutputOptions {+ continue_on_page_error: self.config.continue_on_page_error,+ extract_content_bounds: self.config.extract_content_bounds,+ extract_images: self.config.effective_extract_images(),+ // The markdown table detector splits PDFium's merged+ // multi-cell runs on real word geometry, so it needs word+ // boxes even when the caller didn't ask for them.+ emit_word_boxes: self.config.emit_word_boxes || markdown,+ extract_text_metadata: self.config.extract_text_metadata,+ extract_vector_graphics: self.config.extract_vector_graphics,+ extract_annotations: self.config.extract_annotations,+ extract_form_fields: self.config.extract_form_fields,+ extract_structure_tree: self.config.extract_structure_tree,+ },+ }+ }++ /// The OCR render stage's options for a full parse, derived from config.+ ///+ /// Extraction may have flattened SOME pages' form widgets into page+ /// content in its (since dropped) document instance; those pages are+ /// re-flattened on the reopened OCR document so the rasters match what+ /// extraction saw. With `render_form_fields` the form environment paints+ /// the widgets instead, so no re-flatten is needed.+ pub fn ocr_render_options(+ &self,+ grayscale: bool,+ flattened_form_widgets: bool,+ flattened_page_numbers: &[u32],+ ) -> stages::OcrRenderOptions {+ let reflatten_pages = if flattened_form_widgets && !self.config.render_form_fields {+ flattened_page_numbers.iter().copied().collect()+ } else {+ std::collections::HashSet::new()+ };+ stages::OcrRenderOptions {+ // One round per `num_workers` pages bounds raster memory.+ max_rasters: self.config.num_workers.max(1),+ dpi: self.config.dpi,+ grayscale,+ render_form_fields: self.config.render_form_fields,+ continue_on_page_error: self.config.continue_on_page_error,+ reflatten_pages,+ selection: None,+ }+ }++ /// The content-filter stage's options, derived from config.+ pub fn content_filters(&self) -> stages::ContentFilters<'_> {+ stages::ContentFilters {+ crop_box: self.config.crop_box.as_ref(),+ skip_diagonal_text: self.config.skip_diagonal_text,+ }+ }++ /// The OCR engine a parse would use: the override when set, else the+ /// HTTP engine for `ocr_server_url`, else built-in Tesseract. `None` when+ /// OCR is disabled; an error when it is enabled but nothing can serve it.+ pub fn ocr_engine(&self) -> Result<Option<std::sync::Arc<dyn OcrEngine>>, LiteParseError> {+ if !self.config.ocr_enabled {+ return Ok(None);+ }+ if let Some(engine) = self.ocr_engine_override.clone() {+ return Ok(Some(engine));+ }+ #[cfg(not(target_arch = "wasm32"))]+ {+ if let Some(ref url) = self.config.ocr_server_url {+ return Ok(Some(std::sync::Arc::new(+ HttpOcrEngine::with_headers(+ url.clone(),+ self.config.ocr_server_headers.clone(),+ )+ .with_retry(crate::ocr::http_simple::OcrRetryConfig {+ hedge_delays_ms: self.config.ocr_hedge_delays_ms.clone(),+ ..Default::default()+ }),+ )));+ }+ #[cfg(feature = "tesseract")]+ {+ Ok(Some(std::sync::Arc::new(TesseractOcrEngine::new(+ self.config.tessdata_path.clone(),+ ))))+ }+ #[cfg(not(feature = "tesseract"))]+ {+ Err("OCR enabled but no --ocr-server-url provided and tesseract feature is disabled".into())+ }+ }+ #[cfg(target_arch = "wasm32")]+ {+ Err(+ "OCR enabled but no `ocrEngine` callback was provided (WASM builds have no built-in OCR engine)".into(),+ )+ }+ }+/// Determine the complexity of each page in a document, returning a vector/// of `PageComplexityStats` for each page. This is useful for deciding/// whether to enable OCR on a per-page basis, or for other heuristics.@@ -353,7 +476,7 @@ impl LiteParse {pub async fn is_complex(&self,input: PdfInput,- ) -> Result<Vec<ocr_merge::PageComplexityStats>, LiteParseError> {+ ) -> Result<Vec<stages::PageComplexityStats>, LiteParseError> {let log = |msg: &str| {if !self.config.quiet {eprintln!("{}", msg);@@ -388,15 +511,17 @@ impl LiteParse {// instead of routing the page to OCR to recover text we already// have. `AnnotationText` still fires for the non-widget appearance// text it was introduced for.- let pages = extract::extract_pages_and_images(+ let pages = stages::extract(&document,- target_pages.as_deref(),- self.config.max_pages,- false, // extract_links: irrelevant for complexity stats- self.glyph_resolver.as_deref(),- extract::ExtractionOutputOptions {- continue_on_page_error: self.config.continue_on_page_error,- ..Default::default()+ &stages::ExtractRequest {+ target_pages: target_pages.as_deref(),+ max_pages: self.config.max_pages,+ extract_links: false, // irrelevant for complexity stats+ glyph_resolver: self.glyph_resolver.as_deref(),+ output: stages::ExtractionOutputOptions {+ continue_on_page_error: self.config.continue_on_page_error,+ ..Default::default()+ },},)?.pages;@@ -413,11 +538,7 @@ impl LiteParse {let mut kept_pages = Vec::with_capacity(pages.len());let mut page_complexities = Vec::with_capacity(pages.len());for page in pages {- let stats = document- .page((page.page_number - 1) as i32)- .map_err(LiteParseError::from)- .and_then(|page_obj| ocr_merge::calculate_page_complexity(&page, &page_obj));- match stats {+ match stages::page_complexity(&document, &page) {Ok(stats) => {page_complexities.push(stats);kept_pages.push(page);@@ -445,9 +566,9 @@ impl LiteParse {// Layout signals come from the real projection pass so they match// what a full parse will decide.let t_layout = web_time::Instant::now();- let parsed_pages = projection::project_pages_to_grid(pages);+ let parsed_pages = stages::project(pages);for (stats, page) in page_complexities.iter_mut().zip(&parsed_pages) {- stats.layout = Some(ocr_merge::calculate_layout_complexity(page));+ stats.layout = Some(stages::layout_complexity(page));}log(&format!("[liteparse] layout: {:.1}ms",@@ -547,48 +668,7 @@ impl LiteParse {let password = self.config.password.as_deref();// Build the OCR engine up front so the renderer knows whether to emit a// grayscale buffer (cheaper, for engines that binarize internally) or RGB.- let ocr_engine: Option<std::sync::Arc<dyn OcrEngine>> = if self.config.ocr_enabled {- Some(if let Some(e) = self.ocr_engine_override.clone() {- e- } else {- #[cfg(not(target_arch = "wasm32"))]- {- if let Some(ref url) = self.config.ocr_server_url {- std::sync::Arc::new(- HttpOcrEngine::with_headers(- url.clone(),- self.config.ocr_server_headers.clone(),- )- .with_retry(- crate::ocr::http_simple::OcrRetryConfig {- hedge_delays_ms: self.config.ocr_hedge_delays_ms.clone(),- ..Default::default()- },- ),- )- } else {- #[cfg(feature = "tesseract")]- {- std::sync::Arc::new(TesseractOcrEngine::new(- self.config.tessdata_path.clone(),- ))- }- #[cfg(not(feature = "tesseract"))]- {- return Err("OCR enabled but no --ocr-server-url provided and tesseract feature is disabled".into());- }- }- }- #[cfg(target_arch = "wasm32")]- {- return Err(- "OCR enabled but no `ocrEngine` callback was provided (WASM builds have no built-in OCR engine)".into(),- );- }- })- } else {- None- };+ let ocr_engine = self.ocr_engine()?;let ocr_grayscale = ocr_engine.as_ref().is_some_and(|e| e.prefers_grayscale());#[allow(unused_mut)] // mutated only by the native image-output writer@@ -615,9 +695,7 @@ impl LiteParse {let repaired_input = self.config.extract_form_fields- .then(|| {- crate::acroform_repair::repair_orphaned_widgets(&lib, validated_input, password)- })+ .then(|| stages::repair_acroform(&lib, validated_input, password)).flatten();#[cfg(not(target_arch = "wasm32"))]let document_input = repaired_input.as_ref().unwrap_or(validated_input);@@ -642,55 +720,20 @@ impl LiteParse {// from the original document; fall back if it no longer loads.#[cfg(not(target_arch = "wasm32"))]if repaired_input.is_some()- && let Ok(source) =- extract::load_document_from_input(&lib, validated_input, password)+ && let Ok(source) = self.open_document(&lib, validated_input, password){- return crate::document_metadata::extract(validated_input, &source);+ return stages::document_metadata(validated_input, &source);}- crate::document_metadata::extract(validated_input, &document)- });- let xfa_packets = self.config.extract_xfa_packets.then(|| {- document- .xfa_packets()- .into_iter()- .map(|packet| XfaPacket {- index: packet.index.max(0) as u32,- name: packet.name,- content_length: packet- .content- .as_ref()- .map_or(0, |content| content.len() as u32),- content: packet- .content- .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()),- })- .collect::<Vec<_>>()+ stages::document_metadata(validated_input, &document)});- let outline = outline.unwrap_or_else(|| extract::extract_outline(&document));- let extracted = extract::extract_pages_and_images(- &document,- target_pages,- max_pages,- self.config.extract_links- && self.config.output_format == crate::config::OutputFormat::Markdown,- self.glyph_resolver.as_deref(),- extract::ExtractionOutputOptions {- continue_on_page_error: self.config.continue_on_page_error,- extract_content_bounds: self.config.extract_content_bounds,- extract_images: self.config.effective_extract_images(),- // The markdown table detector splits PDFium's merged- // multi-cell runs on real word geometry, so it needs word- // boxes even when the caller didn't ask for them.- emit_word_boxes: self.config.emit_word_boxes- || self.config.output_format == crate::config::OutputFormat::Markdown,- extract_text_metadata: self.config.extract_text_metadata,- extract_vector_graphics: self.config.extract_vector_graphics,- extract_annotations: self.config.extract_annotations,- extract_form_fields: self.config.extract_form_fields,- extract_structure_tree: self.config.extract_structure_tree,- },- )?;- let extract::ExtractedPages {+ let xfa_packets = self+ .config+ .extract_xfa_packets+ .then(|| stages::xfa_packets(&document));+ let outline = outline.unwrap_or_else(|| stages::outline(&document));+ let extracted =+ stages::extract(&document, &self.extract_request(target_pages, max_pages))?;+ let stages::ExtractedPages {pages,page_errors,images,@@ -726,11 +769,7 @@ impl LiteParse {let complexity = if self.config.include_complexity {let mut complexity = Vec::with_capacity(pages.len());for page in &pages {- let stats = analysis_document- .page((page.page_number - 1) as i32)- .map_err(LiteParseError::from)- .and_then(|page_obj| ocr_merge::calculate_page_complexity(page, &page_obj));- match stats {+ match stages::page_complexity(analysis_document, page) {Ok(stats) => complexity.push(stats),// The page's text is already extracted; a tolerant// parse keeps it and just leaves `complexity` unset@@ -751,24 +790,16 @@ impl LiteParse {.iter().map(|page| page.page_number as u32).collect::<Vec<_>>();- render::render_document_pages(+ stages::screenshots(analysis_document,Some(&page_numbers),- self.config.dpi,- self.config.detect_screenshot_rects,- self.config.render_form_fields,- self.config.continue_on_page_error,+ &stages::ScreenshotOptions {+ dpi: self.config.dpi,+ detect_rects: self.config.detect_screenshot_rects,+ render_form_fields: self.config.render_form_fields,+ continue_on_page_error: self.config.continue_on_page_error,+ },)?- .into_iter()- .map(|page| ScreenshotResult {- page_num: page.page_num,- width: page.width,- height: page.height,- image_bytes: page.png_bytes,- is_solid_fill: page.is_solid_fill,- rects: page.rects,- })- .collect()} else {Vec::new()};@@ -798,35 +829,18 @@ impl LiteParse {let t1 = web_time::Instant::now();if let Some(engine) = ocr_engine {- let round_rasters = self.config.num_workers.max(1);- // Extraction may have flattened SOME pages' form widgets into- // page content in its (now dropped) document instance; re-apply- // per round on exactly those pages so the rasters match what- // extraction saw. With `render_form_fields` the form environment- // paints the widgets instead, so no re-flatten is needed.- let reflatten_pages: std::collections::HashSet<u32> =- if flattened_form_widgets && !self.config.render_form_fields {- flattened_page_numbers.iter().copied().collect()- } else {- std::collections::HashSet::new()- };+ let render_options = self.ocr_render_options(+ ocr_grayscale,+ flattened_form_widgets,+ &flattened_page_numbers,+ );let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);let mut round_start = 0usize;while round_start < pages.len() {let (rendered, next_start) = {let lib = Library::init();let document = self.open_document(&lib, ocr_input, password)?;- ocr_merge::render_pages_for_ocr(- &document,- &pages,- round_start,- round_rasters,- self.config.dpi,- ocr_grayscale,- self.config.render_form_fields,- self.config.continue_on_page_error,- &reflatten_pages,- )?+ stages::render_for_ocr(&document, &pages, round_start, &render_options)?// `lib` drops here, releasing the PDFium lock before the// engine's async recognition below.};@@ -836,17 +850,16 @@ impl LiteParse {// that needs OCR.continue;}- // `RenderedPage::idx` is absolute, so the whole slice is+ // `OcrRaster::page_idx` is absolute, so the whole slice is// passed regardless of where this round started.- ocr_merge::ocr_and_merge_rendered(- &mut pages,+ let outcomes = stages::recognize(rendered,engine.clone(),&self.config.ocr_language,self.config.num_workers,- self.config.ocr_failure_fatal,)- .await?;+ .await;+ stages::merge_ocr(&mut pages, outcomes, self.config.ocr_failure_fatal)?;}}let t_ocr = web_time::Instant::now();@@ -858,14 +871,10 @@ impl LiteParse {// Caller-requested content filters (page-region crop, diagonal-text// removal). Runs after OCR merge so it also drops OCR text outside the// crop region, and before projection so filtered items never surface.- extract::apply_content_filters(- &mut pages,- self.config.crop_box.as_ref(),- self.config.skip_diagonal_text,- );+ stages::apply_content_filters(&mut pages, &self.content_filters());// Grid projection- let mut parsed_pages = projection::project_pages_to_grid(pages);+ let mut parsed_pages = stages::project(pages);// Attach per-page complexity signals, including the layout signals// that need the projected page (same as `is_complex()` reports).@@ -877,7 +886,7 @@ impl LiteParse {else {continue;};- stats.layout = Some(ocr_merge::calculate_layout_complexity(page));+ stats.layout = Some(stages::layout_complexity(page));page.complexity = Some(stats);}let t2 = web_time::Instant::now();@@ -902,7 +911,7 @@ impl LiteParse {.join("\n\n")};if self.config.output_format == crate::config::OutputFormat::Markdown {- rewrite_duplicate_image_refs(&mut parsed_pages, &mut full_text, &images);+ stages::canonicalize_image_refs(&mut parsed_pages, &mut full_text, &images);}let total = web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;@@ -942,7 +951,7 @@ impl LiteParse {/// own font-recovery pipeline) owns text extraction.pub fn parse_from_pages(&self, pages: Vec<Page>, outline: Vec<OutlineTarget>) -> ParseResult {let total_pages = pages.len().min(u32::MAX as usize) as u32;- let mut parsed_pages = projection::project_pages_to_grid(pages);+ let mut parsed_pages = stages::project(pages);let full_text = if let Some(md) = apply_layout(&self.config, &mut parsed_pages, &outline) {md@@ -978,18 +987,18 @@ impl LiteParse {fn select_block_pages(&self,pages: Vec<Page>,- page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,- complexity: Vec<Option<crate::ocr_merge::PageComplexityStats>>,+ page_blocks: Vec<Vec<stages::PositionedBlock>>,+ complexity: Vec<Option<stages::PageComplexityStats>>,) -> Result<(Vec<Page>,- Vec<Vec<crate::markdown_layout::PositionedBlock>>,- Vec<Option<crate::ocr_merge::PageComplexityStats>>,+ Vec<Vec<stages::PositionedBlock>>,+ Vec<Option<stages::PageComplexityStats>>,bool,),LiteParseError,> {- let mut selected: Vec<((Page, Vec<crate::markdown_layout::PositionedBlock>), _)> =+ let mut selected: Vec<((Page, Vec<stages::PositionedBlock>), _)> =pages.into_iter().zip(page_blocks).zip(complexity).collect();let mut page_filtered = false;if let Some(targets) = self.resolve_target_pages()? {@@ -1021,11 +1030,11 @@ impl LiteParse {pub fn parse_from_blocks(&self,pages: Vec<Page>,- page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,- all_blocks: Option<Vec<crate::markdown_layout::PositionedBlock>>,+ page_blocks: Vec<Vec<stages::PositionedBlock>>,+ all_blocks: Option<Vec<stages::PositionedBlock>>,outline: Vec<OutlineTarget>,images: Vec<crate::types::ExtractedImage>,- page_stats: Vec<Option<crate::ocr_merge::PageComplexityStats>>,+ page_stats: Vec<Option<stages::PageComplexityStats>>,) -> Result<ParseResult, LiteParseError> {// Reported before page filtering, matching every other path.let total_pages = pages.len().min(u32::MAX as usize) as u32;@@ -1034,13 +1043,13 @@ impl LiteParse {let all_blocks = if page_filtered { None } else { all_blocks };let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;- let mut parsed_pages = projection::project_pages_to_grid(pages);+ let mut parsed_pages = stages::project(pages);for ((page, blocks), stats) in parsed_pages.iter_mut().zip(&page_blocks).zip(page_stats) {if markdown_out {- page.markdown = crate::markdown_layout::render_blocks(blocks);+ page.markdown = stages::render_blocks(blocks);}if self.config.extract_blocks {- page.blocks = Some(crate::layout::blocks_for_page(page, blocks));+ page.blocks = Some(stages::layout_blocks(page, blocks));}page.complexity = stats;if !self.config.extract_content_bounds {@@ -1050,7 +1059,7 @@ impl LiteParse {let full_text = if markdown_out {match all_blocks {- Some(blocks) => crate::markdown_layout::render_blocks(&blocks),+ Some(blocks) => stages::render_blocks(&blocks),None => parsed_pages.iter().map(|p| p.markdown.as_str())@@ -1122,27 +1131,19 @@ impl LiteParse {log("[liteparse] converted input to PDF for screenshot rendering");}- let rendered = render::render_pages_to_png(- &validated_input,+ let lib = Library::init();+ let document =+ self.open_document(&lib, &validated_input, self.config.password.as_deref())?;+ stages::screenshots(+ &document,page_numbers.as_deref(),- self.config.dpi,- self.config.password.as_deref(),- self.config.detect_screenshot_rects,- self.config.render_form_fields,- &self.config.page_orientation_corrections,- )?;-- Ok(rendered- .into_iter()- .map(|page| ScreenshotResult {- page_num: page.page_num,- width: page.width,- height: page.height,- image_bytes: page.png_bytes,- is_solid_fill: page.is_solid_fill,- rects: page.rects,- })- .collect())+ &stages::ScreenshotOptions {+ dpi: self.config.dpi,+ detect_rects: self.config.detect_screenshot_rects,+ render_form_fields: self.config.render_form_fields,+ continue_on_page_error: false,+ },+ )}pub fn config(&self) -> &LiteParseConfig {@@ -1177,14 +1178,11 @@ impl LiteParse {// document-level bookmark walk is paid once instead of per batch.let (total_pages, outline) = {let lib = Library::init();- let document = extract::load_document_from_input(- &lib,- &input.input,- self.config.password.as_deref(),- )?;+ let document =+ self.open_document(&lib, &input.input, self.config.password.as_deref())?;(document.page_count().max(0) as u32,- extract::extract_outline(&document),+ stages::outline(&document),)};diff --git a/crates/liteparse/src/stages.rs b/crates/liteparse/src/stages.rsnew file mode 100644index 0000000..42a96c4--- /dev/null+++ b/crates/liteparse/src/stages.rs@@ -0,0 +1,462 @@+//! The parse pipeline as public stage functions.+//!+//! [`LiteParse::parse`](crate::LiteParse::parse) is a fixed sequence of+//! stages: open → orientation → extract → complexity → OCR render →+//! recognition → OCR merge → content filters → grid projection → block+//! classification → markdown rendering. This module exposes each stage as a+//! function over serializable types so a caller can run the same sequence+//! itself and insert its own inputs between stages — its own OCR engine, a+//! page selection, extra blocks — or put a process boundary between any two.+//!+//! Two invariants hold this together:+//!+//! * `parse()` is implemented *only* through these functions (see+//! `parser.rs`), and an integration test composes them by hand and asserts+//! the result equals `parse()`. A step cannot be added to `parse()` without+//! becoming a public stage.+//! * Every type crossing a stage boundary is `Clone + Serialize ++//! Deserialize`, and the serialization is lossless for everything a later+//! stage reads.+//!+//! Stages split into two kinds. **pdfium-bound** stages take an open+//! [`Document`] and must run while the [`Library`] that opened it is alive+//! (PDFium is single-threaded behind a process-global lock, so keep these+//! sections short and never hold a `Document` across an `.await`).+//! **Pure** stages take and return plain data and can run anywhere.+//!+//! | Stage | Function | Kind |+//! | --- | --- | --- |+//! | open + orientation | [`open`], [`apply_orientation`] | pdfium |+//! | document facts | [`outline`], [`xfa_packets`], [`document_metadata`] | pdfium |+//! | extract | [`extract`] | pdfium |+//! | complexity | [`page_complexity`], [`layout_complexity`] | pdfium / pure |+//! | OCR | [`render_for_ocr`] → [`recognize`] → [`merge_ocr`] | pdfium / any / pure |+//! | content filters | [`apply_content_filters`] | pure |+//! | projection | [`project`] | pure |+//! | markdown | [`document_signals`] → [`extract_blocks`] → [`render_page_markdown`] | pure |+//! | screenshots | [`screenshots`] | pdfium |+//!+//! Everything else in the crate that is not re-exported here is unstable.++use std::collections::HashSet;+use std::sync::Arc;++use serde::{Deserialize, Serialize};++use crate::config::{CropBox, ImageMode, PageOrientationCorrection};+use crate::error::LiteParseError;+use crate::extract;+use crate::markdown_layout;+use crate::ocr::OcrEngine;+use crate::ocr_merge;+use crate::projection;+use crate::render;+use crate::types::{+ DocumentMetadata, ExtractedImage, OutlineTarget, Page, ParsedPage, PdfInput, XfaPacket,+};++pub use crate::extract::{ExtractedPages, ExtractionOutputOptions};+pub use crate::layout::{LayoutBlock, LayoutCell};+pub use crate::markdown_layout::{Block, Cell, PositionedBlock, SpanCell, render_blocks};+pub use crate::ocr::OcrResult;+pub use crate::ocr_merge::{+ ComplexityReason, LayoutComplexityReason, LayoutComplexityStats, OcrRaster, OcrRenderOptions,+ PageComplexityStats, PageOcrOutcome,+};+pub use crate::parser::ScreenshotResult;+/// The PDFium handle types the pdfium-bound stages take. Re-exported so a+/// caller needs no direct dependency on the pdfium crate.+pub use pdfium::{Document, Library};++// ── Open ───────────────────────────────────────────────────────────────++/// Open `input` and apply per-page orientation corrections.+///+/// `/Rotate` rewrites live in the open document only, so every reopen of the+/// same input (extraction, OCR rounds, screenshots) must go through this+/// function with the same `corrections` to see the same page geometry.+pub fn open<'lib>(+ lib: &'lib Library,+ input: &PdfInput,+ password: Option<&str>,+ corrections: &[PageOrientationCorrection],+) -> Result<Document<'lib>, LiteParseError> {+ let document = extract::load_document_from_input(lib, input, password)?;+ apply_orientation(&document, corrections)?;+ Ok(document)+}++/// Counter-rotate the pages named in `corrections` so their content reads+/// upright. [`open`] already does this; call it directly only on a document+/// opened some other way.+pub fn apply_orientation(+ document: &Document,+ corrections: &[PageOrientationCorrection],+) -> Result<(), LiteParseError> {+ extract::apply_page_orientation_corrections(document, corrections)+}++/// Rewrite an AcroForm whose widgets are orphaned from their pages so+/// form-field extraction can see them. Returns the repaired input when a+/// rewrite was needed, `None` when the document is fine as is. The result is+/// what to [`open`] for extraction; provenance facts should still come from+/// the original input.+#[cfg(not(target_arch = "wasm32"))]+pub fn repair_acroform(+ lib: &Library,+ input: &PdfInput,+ password: Option<&str>,+) -> Option<PdfInput> {+ crate::acroform_repair::repair_orphaned_widgets(lib, input, password)+}++// ── Document facts ─────────────────────────────────────────────────────++/// Walk the document outline (bookmarks). Document-level, and several times+/// more expensive than opening the document, so a caller parsing in page+/// batches should do it once.+pub fn outline(document: &Document) -> Vec<OutlineTarget> {+ extract::extract_outline(document)+}++/// Raw XFA packets from the document's `/XFA` array. Empty for a non-XFA+/// document.+pub fn xfa_packets(document: &Document) -> Vec<XfaPacket> {+ document+ .xfa_packets()+ .into_iter()+ .map(|packet| XfaPacket {+ index: packet.index.max(0) as u32,+ name: packet.name,+ content_length: packet+ .content+ .as_ref()+ .map_or(0, |content| content.len() as u32),+ content: packet+ .content+ .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()),+ })+ .collect()+}++/// Provenance metadata (dates, security, signatures, incremental-save+/// markers, XMP). `input` must be the file the facts describe: pass the+/// original input, not an AcroForm-repaired or format-converted copy.+pub fn document_metadata(input: &PdfInput, document: &Document) -> DocumentMetadata {+ crate::document_metadata::extract(input, document)+}++// ── Extract ────────────────────────────────────────────────────────────++/// What [`extract`] pulls out of the document.+#[derive(Clone, Copy)]+pub struct ExtractRequest<'a> {+ /// 1-based pages to extract; `None` means every page.+ pub target_pages: Option<&'a [u32]>,+ /// Upper bound on extracted pages, applied after `target_pages`.+ pub max_pages: usize,+ /// Resolve link annotations onto the text items they cover.+ pub extract_links: bool,+ /// Last-resort glyph recovery for buggy fonts (see [`crate::GlyphResolver`]).+ pub glyph_resolver: Option<&'a dyn crate::GlyphResolver>,+ /// Which optional outputs to produce (images, word boxes, form fields…).+ pub output: ExtractionOutputOptions,+}++impl Default for ExtractRequest<'_> {+ fn default() -> Self {+ Self {+ target_pages: None,+ max_pages: usize::MAX,+ extract_links: false,+ glyph_resolver: None,+ output: ExtractionOutputOptions::default(),+ }+ }+}++/// Extract text items, graphics, structure nodes and (optionally) images+/// from an open document. This is the only stage that reads page content+/// through PDFium; everything downstream works on the returned [`Page`]s.+pub fn extract(+ document: &Document,+ request: &ExtractRequest<'_>,+) -> Result<ExtractedPages, LiteParseError> {+ extract::extract_pages_and_images(+ document,+ request.target_pages,+ request.max_pages,+ request.extract_links,+ request.glyph_resolver,+ request.output,+ )+}++// ── Complexity ─────────────────────────────────────────────────────────++/// Score one extracted page: does it need OCR, and why. Reads page objects+/// (images, filled paths) through PDFium, so it runs against the document+/// the page was extracted from. `layout` is left unset; see+/// [`layout_complexity`] for the post-projection half.+pub fn page_complexity(+ document: &Document,+ page: &Page,+) -> Result<PageComplexityStats, LiteParseError> {+ let page_obj = document.page((page.page_number - 1) as i32)?;+ ocr_merge::calculate_page_complexity(page, &page_obj)+}++/// Layout-difficulty signals (columns, tables, dense graphics) for a+/// projected page. Pure; fills `PageComplexityStats::layout`.+pub fn layout_complexity(page: &ParsedPage) -> LayoutComplexityStats {+ ocr_merge::calculate_layout_complexity(page)+}++// ── OCR ────────────────────────────────────────────────────────────────++/// Rasterize the pages in `pages[start..]` that need OCR (or that+/// `options.selection` names), stopping after `options.max_rasters`. Returns+/// the rasters and the index to resume from, so a long document runs in+/// bounded rounds. Drop the [`Document`] before awaiting recognition.+pub fn render_for_ocr(+ document: &Document,+ pages: &[Page],+ start: usize,+ options: &OcrRenderOptions,+) -> Result<(Vec<OcrRaster>, usize), LiteParseError> {+ ocr_merge::render_pages_for_ocr(document, pages, start, options)+}++/// Run an [`OcrEngine`] over rasters, at most `num_workers` at a time. Pure+/// recognition; failures are per-outcome, not errors. A caller with its own+/// OCR service can skip this and build [`PageOcrOutcome`]s directly from+/// the [`OcrRaster`] facts and its engine's word boxes (raster pixel space).+pub async fn recognize(+ rasters: Vec<OcrRaster>,+ engine: Arc<dyn OcrEngine>,+ language: &str,+ num_workers: usize,+) -> Vec<PageOcrOutcome> {+ ocr_merge::recognize_rasters(rasters, engine, language, num_workers).await+}++/// Merge recognition outcomes into `pages` in place: drop unusable native+/// text, filter engine artifacts, append the surviving results as `OCR`+/// text items in viewport points. Errors only when every outcome failed and+/// one of the failed pages had no usable native text (`ocr_failure_fatal`).+pub fn merge_ocr(+ pages: &mut [Page],+ outcomes: Vec<PageOcrOutcome>,+ ocr_failure_fatal: bool,+) -> Result<(), LiteParseError> {+ ocr_merge::merge_ocr_results(pages, outcomes, ocr_failure_fatal)+}++// ── Content filters ────────────────────────────────────────────────────++/// Caller-requested item filters applied between OCR merge and projection.+#[derive(Debug, Clone, Copy, Default)]+pub struct ContentFilters<'a> {+ /// Keep only items lying entirely inside the surviving page region.+ pub crop_box: Option<&'a CropBox>,+ /// Drop skewed text (watermarks, rotated stamps).+ pub skip_diagonal_text: bool,+}++/// Apply [`ContentFilters`] to `pages` in place. Runs after [`merge_ocr`] so+/// OCR text is filtered too, and before [`project`] so removed items never+/// reach the output. No-op when no filter is requested.+pub fn apply_content_filters(pages: &mut [Page], filters: &ContentFilters<'_>) {+ extract::apply_content_filters(pages, filters.crop_box, filters.skip_diagonal_text);+}++// ── Projection ─────────────────────────────────────────────────────────++/// Grid projection: turn each page's items into plain text with spatial+/// layout, plus the per-line structure the markdown stages read. Pure and+/// per page, so it can run over any page subset.+pub fn project(pages: Vec<Page>) -> Vec<ParsedPage> {+ projection::project_pages_to_grid(pages)+}++// ── Markdown ───────────────────────────────────────────────────────────++/// Whole-document signals the block classifier needs: computed once over+/// every page, then applied per page by [`extract_blocks`]. Heading levels+/// are ranked against the document's body font size, and running+/// headers/footers are the lines that repeat across pages — neither is+/// knowable from one page alone. A caller that classifies pages in batches+/// must compute this over the whole document first, or heading levels drift+/// between batches.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+pub struct DocumentSignals {+ /// Dominant body font size in points; `0.0` when no page has sized lines.+ pub body_size: f32,+ /// `(font_size, heading_level)` pairs, largest size first, levels 1..=6.+ pub heading_map: Vec<(f32, u8)>,+ /// Normalized texts of running headers/footers to suppress. Empty when+ /// `keep_headers_footers` was set or the document is too short to tell.+ pub header_footer: HashSet<String>,+}++/// Compute [`DocumentSignals`] over every page of the document.+pub fn document_signals(pages: &[ParsedPage], keep_headers_footers: bool) -> DocumentSignals {+ if pages.is_empty() {+ return DocumentSignals::default();+ }+ let body_size = markdown_layout::compute_body_size(pages);+ let heading_map = markdown_layout::build_heading_map(pages, body_size);+ let header_footer = if keep_headers_footers {+ HashSet::new()+ } else {+ markdown_layout::compute_header_footer_set(pages)+ };+ DocumentSignals {+ body_size,+ heading_map,+ header_footer,+ }+}++/// Per-document inputs to [`extract_blocks`] other than the signals.+#[derive(Clone, Copy)]+pub struct BlockOptions<'a> {+ /// Document outline; entries for the page act as a high-priority heading+ /// source on untagged PDFs.+ pub outline: &'a [OutlineTarget],+ /// Whether and how figures appear in the block list.+ pub image_mode: ImageMode,+ /// Disable running-chrome suppression (both the cross-page set in the+ /// signals and the single-page detector).+ pub keep_headers_footers: bool,+}++/// Classify one projected page into its final block sequence: headings,+/// paragraphs, lists, code, tables, rules, figures — after chrome+/// suppression, rule dedup and soft-hyphen splicing. This is exactly the+/// decomposition [`render_page_markdown`] renders and+/// `ParsedPage::blocks` reports.+///+/// Returns `None` for a page with no projected lines (blank, or fully OCR+/// without font metadata): there is no structure to report, and rendering+/// falls back to the projection text in a fence.+pub fn extract_blocks(+ page: &ParsedPage,+ signals: &DocumentSignals,+ options: &BlockOptions<'_>,+) -> Option<Vec<PositionedBlock>> {+ if page.projected_lines.is_empty() {+ return None;+ }++ // Filter outline entries to this page so the classifier's y/title match+ // is a O(entries_on_page) scan per line, not O(whole doc).+ let target_index = (page.page_number as i32).saturating_sub(1);+ let page_outline: Vec<OutlineTarget> = options+ .outline+ .iter()+ .filter(|e| e.page_index == target_index)+ .cloned()+ .collect();+ let chrome_indices = if options.keep_headers_footers {+ HashSet::new()+ } else {+ markdown_layout::detect_single_page_chrome(page, signals.body_size)+ };+ let mut blocks = markdown_layout::classify_page_with_filters(+ page,+ &signals.heading_map,+ &signals.header_footer,+ &page_outline,+ options.image_mode,+ &chrome_indices,+ );+ // A page whose only surviving blocks are horizontal rules (all its text+ // was stripped as chrome) should render empty, not as a stack of bare+ // `---` separators.+ let has_content = blocks+ .iter()+ .any(|b| !matches!(b.block, Block::HorizontalRule | Block::Figure { .. }));+ if !has_content {+ blocks.retain(|b| !matches!(b.block, Block::HorizontalRule));+ }+ crate::output::markdown::dedupe_rules(&mut blocks);+ Some(markdown_layout::splice_soft_hyphens(blocks))+}++/// Render one page's markdown from its blocks. A page that classified to+/// `None` renders its projection text inside a ```` ```text ```` fence so+/// nothing is silently dropped.+pub fn render_page_markdown(page: &ParsedPage, blocks: Option<&[PositionedBlock]>) -> String {+ match blocks {+ Some(blocks) => render_blocks(blocks),+ None => {+ let mut out = String::from("```text\n");+ out.push_str(&page.text);+ if !page.text.ends_with('\n') {+ out.push('\n');+ }+ out.push_str("```");+ out+ }+ }+}++/// The public, flat view of a page's blocks (`ParsedPage::blocks`), with+/// block boxes mapped back from the projected frame into viewport space.+pub fn layout_blocks(page: &ParsedPage, blocks: &[PositionedBlock]) -> Vec<LayoutBlock> {+ crate::layout::blocks_for_page(page, blocks)+}++/// Point duplicate-image figure references at the canonical image's file+/// name, in every page's markdown and in `full_text`. Only the canonical+/// image is written to disk, so a `` for a duplicate+/// placement would otherwise reference a file that does not exist.+pub fn canonicalize_image_refs(+ pages: &mut [ParsedPage],+ full_text: &mut String,+ images: &[ExtractedImage],+) {+ crate::parser::rewrite_duplicate_image_refs(pages, full_text, images)+}++// ── Screenshots ────────────────────────────────────────────────────────++/// How [`screenshots`] renders pages.+#[derive(Debug, Clone, Copy)]+pub struct ScreenshotOptions {+ pub dpi: f32,+ /// Detect solid rectangles/lines in the raster (`ScreenshotResult::rects`).+ pub detect_rects: bool,+ /// Paint form-field appearances (runs document actions).+ pub render_form_fields: bool,+ /// Skip pages that fail to render instead of failing the call.+ pub continue_on_page_error: bool,+}++/// Render pages to PNG. `page_numbers` is 1-based; `None` renders every+/// page.+pub fn screenshots(+ document: &Document,+ page_numbers: Option<&[u32]>,+ options: &ScreenshotOptions,+) -> Result<Vec<ScreenshotResult>, LiteParseError> {+ Ok(render::render_document_pages(+ document,+ page_numbers,+ options.dpi,+ options.detect_rects,+ options.render_form_fields,+ options.continue_on_page_error,+ )?+ .into_iter()+ .map(|page| ScreenshotResult {+ page_num: page.page_num,+ width: page.width,+ height: page.height,+ image_bytes: page.png_bytes,+ is_solid_fill: page.is_solid_fill,+ rects: page.rects,+ })+ .collect())+}diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rsindex 713ff70..abe88d0 100644--- a/crates/liteparse/src/types.rs+++ b/crates/liteparse/src/types.rs@@ -1,4 +1,4 @@-use serde::Serialize;+use serde::{Deserialize, Serialize};use std::collections::{BTreeMap, HashMap};#[doc(hidden)]@@ -13,7 +13,7 @@ pub enum PdfInput {/// Document-level provenance metadata extracted from PDFium plus a bounded/// streaming scan of the source PDF. Fields stay optional so malformed/// metadata never prevents the document itself from being parsed.-#[derive(Debug, Clone, Default, Serialize)]+#[derive(Debug, Clone, Default, Serialize, Deserialize)]pub struct DocumentMetadata {#[serde(skip_serializing_if = "Option::is_none")]pub creation_date: Option<String>,@@ -58,7 +58,8 @@ pub struct DocumentMetadata {/// Represents a single text item extracted from a PDF page,/// including its content, position, size, rotation, and font metadata.-#[derive(Debug, Clone, Default, Serialize)]+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(default)]pub struct TextItem {pub text: String,/// Viewport-space coordinates (top-left origin, 72 DPI).@@ -127,9 +128,10 @@ pub struct TextItem {/// breaks at line/column boundaries), so this exposes the finer word-level/// geometry needed for bbox attribution. Empty for items that produced no/// word split (e.g. OCR-sourced or single-token items). Internal/attribution- /// use only — `#[serde(skip)]` keeps it out of the JSON output but it is- /// marshalled across the napi boundary.- #[serde(skip)]+ /// use only — the public JSON output (`output/json.rs`) builds its own+ /// item view without it, but it is marshalled across the napi boundary+ /// and must survive a stage boundary (see `stages`), so it serializes.+ #[serde(skip_serializing_if = "Vec::is_empty")]pub words: Vec<WordBox>,}@@ -195,7 +197,7 @@ impl TextItem {/// One word's bounding box within a `TextItem`, in the same viewport space/// (top-left origin, 72 DPI) as the parent item. `text` is the word's content/// with inter-word spaces excluded.-#[derive(Debug, Clone, Default, Serialize)]+#[derive(Debug, Clone, Default, Serialize, Deserialize)]pub struct WordBox {pub text: String,pub x: f32,@@ -204,8 +206,13 @@ pub struct WordBox {pub height: f32,}+/// Serialization is lossless: every field the later stages read (graphics,+/// structure nodes, image refs, per-item word boxes) round-trips through+/// `Serialize`/`Deserialize`, so a caller sequencing the `stages` module can+/// put a process or activity boundary between any two stages. The public JSON+/// output does not use this shape (see `output/json.rs`).#[doc(hidden)]-#[derive(Debug, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Page {pub page_number: usize,/// The document's `/PageLabels` entry for this page ("iv", "A-1"), when@@ -222,8 +229,9 @@ pub struct Page {pub content_bounds: Option<Rect>,pub text_items: Vec<TextItem>,/// Vector graphics on the page, distilled from PDFium path objects.- /// Not emitted in JSON/text outputs — consumed by the markdown layout pass.- #[serde(skip)]+ /// Not part of the public JSON/text outputs — consumed by the markdown+ /// layout pass.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub graphics: Vec<GraphicPrimitive>,/// Lossless-enough, PDFium-compatible path output requested by the caller./// Kept separate from the lossy internal `graphics` layout primitives.@@ -232,26 +240,26 @@ pub struct Page {/// Structure-tree nodes for this page when the PDF is tagged. Each node/// carries its role, marked-content ids, and the union bbox of its tagged/// content. Empty for untagged PDFs.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub struct_nodes: Vec<StructNode>,/// Raster image objects detected on the page. Empty when the page has no/// images. Threaded through to `ParsedPage.image_refs`.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub image_refs: Vec<ImageRef>,/// Public annotation data when explicitly requested. `None` distinguishes/// disabled extraction from an enabled page with no annotations.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub annotations: Option<Vec<DocumentAnnotation>>,/// AcroForm widgets when explicitly requested.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub form_fields: Option<Vec<FormField>>,/// Tagged-PDF logical structure when explicitly requested.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Option::is_none")]pub structure_tree: Option<StructureTree>,}/// A page that could not be extracted while tolerant page errors were enabled.-#[derive(Debug, Clone, Serialize, PartialEq, Eq)]+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]pub struct PageError {/// Source page number (1-indexed). Serialized as `page` to match the/// sibling per-page fields in the JSON output (`pages[]`, `images[]`).@@ -263,7 +271,7 @@ pub struct PageError {/// One PDF page annotation. Coordinates use the same top-left, 72-DPI/// viewport space as [`TextItem`].-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct DocumentAnnotation {pub subtype: String,#[serde(skip_serializing_if = "Option::is_none")]@@ -276,14 +284,14 @@ pub struct DocumentAnnotation {pub title: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]pub rect: Option<Rect>,- #[serde(skip_serializing_if = "Vec::is_empty")]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub quadpoint_rects: Vec<Rect>,#[serde(skip_serializing_if = "Option::is_none")]pub uri: Option<String>,}/// Scalar value from a tagged-PDF structure element's `/A` dictionary.-#[derive(Debug, Clone, Serialize, PartialEq)]+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]#[serde(untagged)]pub enum StructureAttributeValue {Boolean(bool),@@ -292,14 +300,14 @@ pub enum StructureAttributeValue {}/// A complete page-scoped tagged-PDF logical structure tree.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct StructureTree {pub roots: Vec<StructureTreeElement>,}/// One tagged-PDF structure element. Field names follow the/// repository's snake_case JSON convention rather than PDFium's C spellings.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct StructureTreeElement {#[serde(rename = "type")]pub element_type: String,@@ -311,7 +319,7 @@ pub struct StructureTreeElement {pub alt_text: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]pub title: Option<String>,- #[serde(skip_serializing_if = "BTreeMap::is_empty")]+ #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]pub attributes: BTreeMap<String, StructureAttributeValue>,pub marked_content_ids: Vec<i32>,pub children: Vec<StructureTreeElement>,@@ -319,7 +327,7 @@ pub struct StructureTreeElement {}/// One AcroForm widget and its resolved field metadata.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct FormField {pub id: String,#[serde(rename = "type")]@@ -346,15 +354,15 @@ pub struct FormField {pub checked: Option<bool>,#[serde(skip_serializing_if = "Option::is_none")]pub rect: Option<Rect>,- #[serde(skip_serializing_if = "Vec::is_empty")]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub options: Vec<String>,- #[serde(skip_serializing_if = "Vec::is_empty")]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub selected_options: Vec<String>,}/// One raw packet from an XFA form document's `/XFA` array. Surfaced on/// `ParseResult.xfa_packets` when `extract_xfa_packets` is enabled.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct XfaPacket {/// Zero-based index in the XFA array.pub index: u32,@@ -373,7 +381,7 @@ pub struct XfaPacket {/// Coordinates are in the same top-left, 72-DPI viewport space as text/// items. Detection runs on the raster, so it also covers scanned/flattened/// pages that carry no vector paths.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ScreenshotRect {pub x: f32,pub y: f32,@@ -390,7 +398,7 @@ pub struct ScreenshotRect {/// user space (origin bottom-left) — convert to viewport with/// `page_height - y` once you know the page.#[doc(hidden)]-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct OutlineTarget {/// Hierarchy depth, 1-based.pub level: u8,@@ -405,7 +413,7 @@ pub struct OutlineTarget {/// One node from the structure tree of a page. Pre-flattened in pre-order/// (parent before children).#[doc(hidden)]-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct StructNode {pub role: String,pub mcids: Vec<i32>,@@ -416,7 +424,11 @@ pub struct StructNode {}/// Represents a fully parsed page with projected text layout.-#[derive(Debug, Serialize)]+///+/// Like [`Page`], serialization is lossless so the markdown stages can run on+/// the other side of a process boundary from projection. The public JSON+/// output builds its own view of this type (`output/json.rs`).+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ParsedPage {pub page_number: usize,/// The document's `/PageLabels` entry for this page ("iv", "A-1"), when@@ -432,21 +444,21 @@ pub struct ParsedPage {#[serde(skip_serializing_if = "Option::is_none")]pub content_bounds: Option<Rect>,pub text: String,- #[serde(skip_serializing_if = "String::is_empty")]+ #[serde(default, skip_serializing_if = "String::is_empty")]pub markdown: String,pub text_items: Vec<TextItem>,/// Per-line structural metadata used by the markdown emitter. Not part of- /// the JSON/text outputs (consumed internally) so it is `#[serde(skip)]`.- #[serde(skip)]+ /// the public JSON/text outputs.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub projected_lines: Vec<ProjectedLine>,/// Root of the XY-cut region tree for this page. Leaves correspond to the/// `region_path` on each `ProjectedLine`. Internal-only.- #[serde(skip)]+ #[serde(default)]pub regions: Region,/// Vector graphics on the page (decomposed paths) used by the markdown/// emitter for ruled-table / HR / figure-cluster detection. Not part of- /// the JSON/text output.- #[serde(skip)]+ /// the public JSON/text output.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub graphics: Vec<GraphicPrimitive>,/// Public vector path extraction. Absent unless explicitly enabled.#[serde(skip_serializing_if = "Option::is_none")]@@ -454,7 +466,7 @@ pub struct ParsedPage {/// Figure-region bounding rectangles derived from `graphics`. Pre-computed/// in `to_parsed_pages` so the XY-cut layout pass can treat them as/// obstacles, and reused downstream for figure classification.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub figures: Vec<Rect>,/// `(projected, original)` geometry of every text item, populated only/// when the two differ for at least one item on the page. Rotation@@ -465,18 +477,18 @@ pub struct ParsedPage {/// through this table before they are reported, so they land in the/// same viewport space as `text_items`. Empty for pages without rotated/// text, which is the overwhelming majority.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub projected_item_frames: Vec<(Rect, Rect)>,/// Structure-tree nodes for this page (tagged PDFs only). Pre-flattened in/// pre-order. Consumed by the markdown classifier for highest-priority/// heading / figure / table detection.- #[serde(skip)]+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub struct_nodes: Vec<StructNode>,/// Raster image objects on the page. Bbox in viewport coords. Populated/// during extraction; consumed by the markdown emitter to interleave/// `Block::Figure` references at the right y position. Empty when the- /// page has no embedded images. Not part of JSON/text output.- #[serde(skip)]+ /// page has no embedded images. Not part of the public JSON/text output.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]pub image_refs: Vec<ImageRef>,/// Per-page complexity signals (the same the `is_complex` API returns)./// Populated only when `LiteParseConfig::include_complexity` is set;@@ -507,7 +519,7 @@ pub struct ParsedPage {/// the image's position among image page-objects, so a later embed pass can/// re-open the document and pull pixel bytes with `render_image_object`.#[doc(hidden)]-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ImageRef {pub id: String,pub bbox: Rect,@@ -516,7 +528,13 @@ pub struct ImageRef {pub pixel_width: u32,pub pixel_height: u32,pub rotation: f32,+ /// Pixel payloads are consumed during extraction (they become+ /// `ExtractedImage.bytes`) and nothing after extraction reads them, so+ /// they stay out of the serialized page: a stage boundary after extract+ /// carries the image bytes on `ExtractedImage`, not here.+ #[serde(skip)]pub jpeg_bytes: Option<Vec<u8>>,+ #[serde(skip)]pub raw_bytes: Option<Vec<u8>>,pub bits_per_pixel: u32,pub colorspace: i32,@@ -528,7 +546,7 @@ pub struct ImageRef {/// produced. JPEG streams are preserved without re-encoding when PDFium/// exposes a valid directly decoded DCT stream; other images are encoded as/// PNG from PDFium's rendered bitmap.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ExtractedImage {pub id: String,pub name: String,@@ -543,13 +561,15 @@ pub struct ExtractedImage {#[serde(skip_serializing_if = "Option::is_none")]pub duplicate_of: Option<String>,/// Encoded image bytes. Shared (`Arc`) so duplicate entries reference the- /// canonical image's buffer instead of copying it per occurrence.+ /// canonical image's buffer instead of copying it per occurrence. Not+ /// serialized: an image payload crossing a boundary goes as a file or a+ /// separate blob, keyed by `id`.#[serde(skip)]pub bytes: std::sync::Arc<Vec<u8>>,}#[doc(hidden)]-#[derive(Debug, Clone, Default, PartialEq, Serialize)]+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]pub struct Rect {pub x: f32,pub y: f32,@@ -588,14 +608,14 @@ impl Rect {/// Page-scoped vector path output. Coordinates use the same top-left,/// 72-DPI viewport space as text items.-#[derive(Debug, Clone, Default, Serialize)]+#[derive(Debug, Clone, Default, Serialize, Deserialize)]pub struct VectorGraphics {pub shapes: Vec<VectorShape>,pub lines: Vec<VectorLine>,}/// One PDF path object's paint state and viewport bounding box.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct VectorShape {pub bbox: Rect,pub stroke: bool,@@ -609,7 +629,7 @@ pub struct VectorShape {/// A strict horizontal or vertical path segment after adjacent compatible/// segments have been merged, matching LlamaParse PDFium path semantics.-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct VectorLine {pub x1: f32,pub y1: f32,@@ -630,7 +650,7 @@ pub struct VectorLine {/// clusters) are kept — bezier curves and complex paths are decomposed into/// straight strokes, or dropped.#[doc(hidden)]-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub enum GraphicPrimitive {/// A single straight line segment in viewport coords. Used for HR/table/// border detection.@@ -673,7 +693,7 @@ impl GraphicPrimitive {/// Per-line structural metadata derived during grid projection. Used by the/// markdown emitter; not surfaced in JSON/text output.#[doc(hidden)]-#[derive(Debug, Clone, Serialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ProjectedLine {pub text: String,pub bbox: Rect,@@ -721,14 +741,14 @@ pub struct ProjectedLine {/// XY-cut region tree node. A page's root region recursively splits along H or/// V axes until each leaf holds a coherent block of items.#[doc(hidden)]-#[derive(Debug, Clone, Default)]+#[derive(Debug, Clone, Default, Serialize, Deserialize)]pub struct Region {pub bbox: Rect,pub kind: RegionKind,}#[doc(hidden)]-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub enum RegionKind {Leaf {item_indices: Vec<usize>,@@ -748,7 +768,7 @@ impl Default for RegionKind {}#[doc(hidden)]-#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]pub enum CutAxis {Horizontal,Vertical,@@ -763,7 +783,7 @@ pub enum Snap {}#[doc(hidden)]-#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub enum Anchor {Left,Right,diff --git a/crates/liteparse/tests/stages_compose.rs b/crates/liteparse/tests/stages_compose.rsnew file mode 100644index 0000000..ff49a27--- /dev/null+++ b/crates/liteparse/tests/stages_compose.rs@@ -0,0 +1,604 @@+//! Gate for the `stages` module: `parse()` must equal the stages composed by+//! hand.+//!+//! `LiteParse::parse` is written only in terms of `liteparse::stages`. This+//! test re-implements that sequence here, from the public stage functions,+//! and asserts the two produce identical output over the fixture corpus and+//! several configurations (markdown with every extraction option, OCR in+//! bounded rounds through a mock engine, orientation corrections, content+//! filters, AcroForm repair). If a step is added to `parse()` without a+//! public stage, this test is what breaks.+//!+//! Along the way the composed pipeline serializes and deserializes the page+//! set at two stage boundaries (after extract, after projection), so the+//! same test proves the boundary types round-trip losslessly.++use std::sync::Arc;++use liteparse::config::{CropBox, ImageMode, OutputFormat, PageOrientationCorrection};+use liteparse::ocr::{OcrEngine, OcrOptions, OcrResult};+use liteparse::stages::{self, Library};+use liteparse::types::{Page, ParsedPage, PdfInput};+use liteparse::{LiteParse, LiteParseConfig, ParseResult};+use serial_test::serial;++const FIXTURES: &str = "../../integration_tests_data";++fn skip_integration() -> bool {+ std::env::var("SKIP_INTEGRATION_TESTS").as_deref() == Ok("yes")+}++fn fixture(name: &str) -> String {+ format!("{FIXTURES}/{name}")+}++/// Every PDF in the fixture corpus, sorted for stable failure output.+fn corpus_pdfs() -> Vec<String> {+ let mut paths: Vec<String> = std::fs::read_dir(FIXTURES)+ .expect("fixture directory")+ .filter_map(|entry| {+ let path = entry.ok()?.path();+ (path.extension()? == "pdf").then(|| path.to_string_lossy().into_owned())+ })+ .collect();+ paths.sort();+ assert!(!paths.is_empty(), "no fixture PDFs found under {FIXTURES}");+ paths+}++/// Deterministic mock OCR: one box per raster whose text encodes the raster+/// size, so a raster routed to the wrong page is visible in the output.+struct MockOcr;++impl OcrEngine for MockOcr {+ fn name(&self) -> &str {+ "mock"+ }+ fn recognize<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ _image_data: &'c [u8],+ width: u32,+ height: u32,+ _options: &'b OcrOptions,+ ) -> std::pin::Pin<+ Box<+ dyn Future<Output = Result<Vec<OcrResult>, Box<dyn std::error::Error + Send + Sync>>>+ + Send+ + '_,+ >,+ > {+ Box::pin(async move {+ Ok(vec![+ OcrResult {+ text: format!("MOCK {width}x{height}"),+ bbox: [10.0, 10.0, 260.0, 40.0],+ confidence: 0.99,+ polygon: None,+ },+ OcrResult {+ text: "rotated".into(),+ bbox: [300.0, 100.0, 330.0, 400.0],+ confidence: 0.9,+ polygon: Some([+ [330.0, 100.0],+ [330.0, 400.0],+ [300.0, 400.0],+ [300.0, 100.0],+ ]),+ },+ ])+ })+ }+}++/// Everything `ParseResult` reports, in a comparable form. Pages go through+/// their (lossless) serde form so every field is compared, not just the+/// rendered strings.+#[derive(Debug, PartialEq)]+struct Snapshot {+ total_pages: u32,+ pages: Vec<serde_json::Value>,+ page_errors: Vec<(u32, String)>,+ text: String,+ outline: String,+ images: Vec<serde_json::Value>,+ screenshots: Vec<(u32, u32, u32, bool, usize, Vec<u8>)>,+ image_error_count: u32,+ form_type: Option<i32>,+ creator: Option<String>,+ producer: Option<String>,+ doc_meta: Option<serde_json::Value>,+ xfa_packets: Option<serde_json::Value>,+}++fn snapshot(result: &ParseResult) -> Snapshot {+ Snapshot {+ total_pages: result.total_pages,+ pages: result+ .pages+ .iter()+ .map(|p| serde_json::to_value(p).unwrap())+ .collect(),+ page_errors: result+ .page_errors+ .iter()+ .map(|e| (e.page_number, e.message.clone()))+ .collect(),+ text: result.text.clone(),+ outline: format!("{:?}", result.outline),+ images: result+ .images+ .iter()+ .map(|i| serde_json::to_value(i).unwrap())+ .collect(),+ screenshots: result+ .screenshots+ .iter()+ .map(|s| {+ (+ s.page_num,+ s.width,+ s.height,+ s.is_solid_fill,+ s.rects.len(),+ s.image_bytes.clone(),+ )+ })+ .collect(),+ image_error_count: result.image_error_count,+ form_type: result.form_type,+ creator: result.creator.clone(),+ producer: result.producer.clone(),+ doc_meta: result+ .doc_meta+ .as_ref()+ .map(|m| serde_json::to_value(m).unwrap()),+ xfa_packets: result+ .xfa_packets+ .as_ref()+ .map(|x| serde_json::to_value(x).unwrap()),+ }+}++/// Serialize and deserialize `value`, asserting the round trip is lossless+/// (re-serializing the deserialized value yields the same JSON).+fn round_trip<T: serde::Serialize + serde::de::DeserializeOwned>(value: T, what: &str) -> T {+ let json = serde_json::to_string(&value).unwrap();+ let back: T = serde_json::from_str(&json).unwrap_or_else(|e| panic!("{what}: {e}"));+ assert_eq!(+ serde_json::to_string(&back).unwrap(),+ json,+ "{what}: serde round trip changed the value"+ );+ back+}++/// `parse()` written out by hand from the public stages. Mirrors+/// `LiteParse::parse_resolved` step for step; the point is that nothing in+/// that function is unreachable from here.+async fn compose(parser: &LiteParse, input: PdfInput) -> ParseResult {+ let config = parser.config();+ let password = config.password.as_deref();+ let markdown = config.output_format == OutputFormat::Markdown;+ let target_pages = config+ .target_pages+ .as_ref()+ .map(|s| liteparse::config::parse_target_pages(s).unwrap());++ // Non-PDF inputs are converted before the pipeline starts; a PDF passes+ // through untouched.+ let (input, _guard) = liteparse::conversion::resolve_pdf_input(input, password, false)+ .await+ .unwrap();+ let want_doc_meta = config.extract_document_metadata && !_guard.is_converted();++ let engine = parser.ocr_engine().unwrap();+ let grayscale = engine.as_ref().is_some_and(|e| e.prefers_grayscale());++ // ── pdfium-bound: open, document facts, extract, complexity, screenshots+ let lib = Library::init();+ let repaired = config+ .extract_form_fields+ .then(|| stages::repair_acroform(&lib, &input, password))+ .flatten();+ let document_input = repaired.as_ref().unwrap_or(&input);+ let document = stages::open(+ &lib,+ document_input,+ password,+ &config.page_orientation_corrections,+ )+ .unwrap();+ let total_pages = document.page_count().max(0) as u32;+ let form_type = config.extract_form_fields.then(|| document.form_type());+ let creator = document.meta_text("Creator").filter(|v| !v.is_empty());+ let producer = document.meta_text("Producer").filter(|v| !v.is_empty());+ let doc_meta = want_doc_meta.then(|| {+ if repaired.is_some() {+ let source = stages::open(&lib, &input, password, &[]).unwrap();+ return stages::document_metadata(&input, &source);+ }+ stages::document_metadata(&input, &document)+ });+ let xfa_packets = config+ .extract_xfa_packets+ .then(|| stages::xfa_packets(&document));+ let outline = stages::outline(&document);+ let extracted = stages::extract(+ &document,+ &parser.extract_request(target_pages.as_deref(), config.max_pages),+ )+ .unwrap();+ let stages::ExtractedPages {+ pages,+ page_errors,+ mut images,+ image_error_count,+ flattened_form_widgets,+ flattened_page_numbers,+ } = extracted;+ let complexity: Vec<stages::PageComplexityStats> = if config.include_complexity {+ pages+ .iter()+ .map(|page| stages::page_complexity(&document, page).unwrap())+ .collect()+ } else {+ Vec::new()+ };+ let screenshots = if config.extract_screenshots {+ let page_numbers: Vec<u32> = pages.iter().map(|p| p.page_number as u32).collect();+ stages::screenshots(+ &document,+ Some(&page_numbers),+ &stages::ScreenshotOptions {+ dpi: config.dpi,+ detect_rects: config.detect_screenshot_rects,+ render_form_fields: config.render_form_fields,+ continue_on_page_error: config.continue_on_page_error,+ },+ )+ .unwrap()+ } else {+ Vec::new()+ };+ drop(document);+ drop(lib);++ // Boundary 1: the extracted page set crosses a (simulated) process+ // boundary before OCR.+ let mut pages: Vec<Page> = round_trip(pages, "Vec<Page> after extract");++ // ── OCR: render rounds (pdfium) → recognize (async) → merge (pure)+ if let Some(engine) = engine {+ let options =+ parser.ocr_render_options(grayscale, flattened_form_widgets, &flattened_page_numbers);+ let ocr_input = repaired.as_ref().unwrap_or(&input);+ let mut start = 0;+ while start < pages.len() {+ let (rasters, next) = {+ let lib = Library::init();+ let document = stages::open(+ &lib,+ ocr_input,+ password,+ &config.page_orientation_corrections,+ )+ .unwrap();+ stages::render_for_ocr(&document, &pages, start, &options).unwrap()+ };+ start = next;+ if rasters.is_empty() {+ continue;+ }+ let rasters = round_trip(rasters, "Vec<OcrRaster>");+ let outcomes = stages::recognize(+ rasters,+ engine.clone(),+ &config.ocr_language,+ config.num_workers,+ )+ .await;+ let outcomes = round_trip(outcomes, "Vec<PageOcrOutcome>");+ stages::merge_ocr(&mut pages, outcomes, config.ocr_failure_fatal).unwrap();+ }+ }++ // ── pure: filters, projection, complexity layout half+ stages::apply_content_filters(&mut pages, &parser.content_filters());+ let parsed = stages::project(pages);++ // Boundary 2: projected pages cross before the markdown stages.+ let mut parsed: Vec<ParsedPage> = round_trip(parsed, "Vec<ParsedPage> after project");++ let mut complexity = complexity.into_iter().peekable();+ for page in parsed.iter_mut() {+ if let Some(mut stats) = complexity.next_if(|s| s.page_number == page.page_number) {+ stats.layout = Some(stages::layout_complexity(page));+ page.complexity = Some(stats);+ }+ }++ // ── markdown: document signals once, then per page+ if markdown || config.extract_blocks {+ let signals = stages::document_signals(&parsed, config.keep_headers_footers);+ // `header_footer` is a set, so its JSON order is not stable across+ // serializations; compare the parts rather than the bytes.+ let back: stages::DocumentSignals =+ serde_json::from_str(&serde_json::to_string(&signals).unwrap()).unwrap();+ assert_eq!(back.body_size, signals.body_size);+ assert_eq!(back.heading_map, signals.heading_map);+ assert_eq!(back.header_footer, signals.header_footer);+ let signals = back;+ let options = stages::BlockOptions {+ outline: &outline,+ image_mode: config.image_mode,+ keep_headers_footers: config.keep_headers_footers,+ };+ for page in parsed.iter_mut() {+ let blocks = stages::extract_blocks(page, &signals, &options);+ let blocks = blocks.map(|b| round_trip(b, "Vec<PositionedBlock>"));+ if config.extract_blocks {+ page.blocks = Some(stages::layout_blocks(+ page,+ blocks.as_deref().unwrap_or_default(),+ ));+ }+ if markdown {+ page.markdown = stages::render_page_markdown(page, blocks.as_deref());+ }+ }+ }+ let mut text = if markdown {+ parsed+ .iter()+ .map(|p| p.markdown.as_str())+ .collect::<Vec<_>>()+ .join("\n\n-----\n\n")+ } else {+ parsed+ .iter()+ .map(|p| p.text.as_str())+ .collect::<Vec<_>>()+ .join("\n\n")+ };+ if markdown {+ stages::canonicalize_image_refs(&mut parsed, &mut text, &images);+ }+ // `parse()` writes image files only with `image_output_dir`; none of the+ // configurations here set it, so `path` stays unset on both sides.+ for image in images.iter_mut() {+ image.path = None;+ }++ ParseResult {+ total_pages,+ pages: parsed,+ page_errors,+ text,+ outline,+ images,+ screenshots,+ image_error_count,+ form_type,+ creator,+ producer,+ doc_meta,+ xfa_packets,+ }+}++/// Parse `path` both ways, assert equality, and hand back the result so a+/// caller can check the configuration actually exercised what it meant to.+async fn assert_parse_equals_composition(+ config: LiteParseConfig,+ path: &str,+ with_ocr: bool,+) -> ParseResult {+ let mut parser = LiteParse::new(config);+ if with_ocr {+ parser = parser.with_ocr_engine(Arc::new(MockOcr));+ }+ let input = PdfInput::Path(path.to_string());+ let via_parse = parser+ .parse_input(input.clone())+ .await+ .unwrap_or_else(|e| panic!("parse() failed on {path}: {e}"));+ let via_stages = compose(&parser, input).await;+ assert_eq!(+ snapshot(&via_parse),+ snapshot(&via_stages),+ "parse() and the composed stages diverged on {path}"+ );+ via_parse+}++/// Every extraction option on, no OCR: exercises extract, complexity,+/// screenshots, projection, blocks and markdown, and the duplicate-image+/// rewrite.+fn everything_config() -> LiteParseConfig {+ LiteParseConfig {+ ocr_enabled: false,+ quiet: true,+ output_format: OutputFormat::Markdown,+ image_mode: ImageMode::Embed,+ extract_images: true,+ extract_blocks: true,+ include_complexity: true,+ extract_content_bounds: true,+ emit_word_boxes: true,+ extract_text_metadata: true,+ extract_vector_graphics: true,+ extract_annotations: true,+ extract_structure_tree: true,+ extract_document_metadata: true,+ extract_xfa_packets: true,+ extract_screenshots: true,+ detect_screenshot_rects: true,+ extract_links: true,+ ..LiteParseConfig::default()+ }+}++#[tokio::test]+#[serial]+async fn parse_equals_composed_stages_markdown_with_every_option() {+ let (mut with_images, mut with_blocks, mut with_screenshots) = (0, 0, 0);+ for path in corpus_pdfs() {+ let result = assert_parse_equals_composition(everything_config(), &path, false).await;+ with_images += usize::from(!result.images.is_empty());+ with_blocks += usize::from(+ result+ .pages+ .iter()+ .any(|p| p.blocks.as_ref().is_some_and(|b| !b.is_empty())),+ );+ with_screenshots += usize::from(!result.screenshots.is_empty());+ }+ // The equality is only meaningful if the branches ran. None of the+ // fixture PDFs embeds a raster image; the demo 10-K does on three pages,+ // so those pages (plus one without) cover image extraction, the figure+ // block and the duplicate-image rewrite.+ assert!(with_blocks > 0, "no fixture produced blocks");+ assert!(with_screenshots > 0, "no fixture produced screenshots");+ let config = LiteParseConfig {+ target_pages: Some("1,2,108,119".into()),+ ..everything_config()+ };+ let result =+ assert_parse_equals_composition(config, "../../demo/docs/apple-10k-2024.pdf", false).await;+ with_images += usize::from(!result.images.is_empty());+ assert!(with_images > 0, "no fixture produced images");+}++/// Text output with blocks: the non-markdown branch of the layout stage.+#[tokio::test]+#[serial]+async fn parse_equals_composed_stages_text_output() {+ let config = LiteParseConfig {+ ocr_enabled: false,+ quiet: true,+ output_format: OutputFormat::Text,+ extract_blocks: true,+ ..LiteParseConfig::default()+ };+ for path in corpus_pdfs() {+ assert_parse_equals_composition(config.clone(), &path, false).await;+ }+}++/// OCR through the stage split, one page per round (`num_workers: 1`) so the+/// render → recognize → merge loop runs several times, plus the+/// selection-by-complexity gate and the native-text artifact filters.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+#[serial]+async fn parse_equals_composed_stages_with_ocr_rounds() {+ let config = LiteParseConfig {+ ocr_enabled: true,+ num_workers: 1,+ quiet: true,+ output_format: OutputFormat::Markdown,+ include_complexity: true,+ ..LiteParseConfig::default()+ };+ let mut ocr_pages = 0;+ for path in corpus_pdfs() {+ let result = assert_parse_equals_composition(config.clone(), &path, true).await;+ ocr_pages += result+ .pages+ .iter()+ .filter(|p| p.text_items.iter().any(|i| i.text.starts_with("MOCK ")))+ .count();+ }+ assert!(ocr_pages > 0, "no fixture page had mock OCR text merged in");+}++/// A converted image input is a scanned page: OCR is its only text.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+#[serial]+async fn parse_equals_composed_stages_on_converted_image() {+ if skip_integration() {+ return;+ }+ let config = LiteParseConfig {+ ocr_enabled: true,+ quiet: true,+ output_format: OutputFormat::Markdown,+ ..LiteParseConfig::default()+ };+ let result = assert_parse_equals_composition(config, &fixture("receipt.png"), true).await;+ assert!(+ result.text.contains("MOCK "),+ "a converted image should be OCR-only: {}",+ result.text+ );+}++/// Orientation corrections applied at open, a page selection, and the+/// content filters.+#[tokio::test]+#[serial]+async fn parse_equals_composed_stages_with_orientation_selection_and_filters() {+ let config = LiteParseConfig {+ ocr_enabled: false,+ quiet: true,+ output_format: OutputFormat::Markdown,+ page_orientation_corrections: vec![PageOrientationCorrection { page: 1, angle: 90 }],+ target_pages: Some("1".into()),+ max_pages: 1,+ crop_box: Some(CropBox {+ top: 0.05,+ bottom: 0.05,+ left: 0.0,+ right: 0.0,+ }),+ skip_diagonal_text: true,+ ..LiteParseConfig::default()+ };+ for name in ["sample_rotated_90cw.pdf", "diagonal_text.pdf"] {+ assert_parse_equals_composition(config.clone(), &fixture(name), false).await;+ }+}++/// Form-field extraction: AcroForm repair before open, widget flattening+/// during extract, and the re-flatten on the reopened OCR document.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+#[serial]+async fn parse_equals_composed_stages_with_form_fields() {+ let config = LiteParseConfig {+ ocr_enabled: true,+ quiet: true,+ output_format: OutputFormat::Markdown,+ extract_form_fields: true,+ extract_document_metadata: true,+ ..LiteParseConfig::default()+ };+ assert_parse_equals_composition(config, &fixture("filled_acroform.pdf"), true).await;+}++/// Sanity check on the round-trip helper itself: a page with every optional+/// field populated survives serde unchanged, including the fields the public+/// JSON output omits (word boxes, graphics, structure nodes, image refs).+#[tokio::test]+#[serial]+async fn extracted_pages_round_trip_keeps_internal_fields() {+ let parser = LiteParse::new(everything_config());+ let lib = Library::init();+ let document = stages::open(&lib, &PdfInput::Path(fixture("sample.pdf")), None, &[]).unwrap();+ let pages = stages::extract(&document, &parser.extract_request(None, usize::MAX))+ .unwrap()+ .pages;+ let has_words = pages+ .iter()+ .any(|p| p.text_items.iter().any(|i| !i.words.is_empty()));+ assert!(has_words, "fixture should produce word boxes");+ let back = round_trip(pages.clone(), "Vec<Page>");+ for (a, b) in pages.iter().zip(&back) {+ assert_eq!(a.text_items.len(), b.text_items.len());+ assert_eq!(a.graphics.len(), b.graphics.len());+ assert_eq!(a.struct_nodes.len(), b.struct_nodes.len());+ assert_eq!(a.image_refs.len(), b.image_refs.len());+ for (x, y) in a.text_items.iter().zip(&b.text_items) {+ assert_eq!(x.words.len(), y.words.len());+ }+ }+}