Skip to content

A fast, helpful, and open-source document parser

Expose the parse pipeline as public stage functions

Logan Markewich3bc9bcac85f2
diff --git a/crates/liteparse/src/layout.rs b/crates/liteparse/src/layout.rs
index 790e456..13b1d04 100644
--- a/crates/liteparse/src/layout.rs
+++ b/crates/liteparse/src/layout.rs
@@ -12,7 +12,7 @@
//! during serialization, so a heading serializes as `{kind, text, level, bbox}`
//! rather than a wall of nulls.
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use crate::markdown_layout::{Block, Cell, PositionedBlock, SpanCell};
use crate::types::{ParsedPage, Rect};
@@ -22,7 +22,7 @@ use crate::types::{ParsedPage, Rect};
/// `bbox` is `None` for cells with no ink behind them — padding inserted to
/// square off a ragged grid, or halves of a merged run split at an estimated
/// position rather than an observed boundary.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LayoutCell {
pub text: String,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -63,11 +63,11 @@ impl From<&SpanCell> for LayoutCell {
/// `kind` discriminates the block; see each field for which kinds populate it.
/// Blocks appear in reading order, matching the order the markdown renderer
/// emits them, so the Nth block here is the Nth block of that page's markdown.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LayoutBlock {
/// One of `heading`, `paragraph`, `list_item`, `code`, `table`,
/// `merged_table`, `grid_fallback`, `rule`, `figure`.
- pub kind: &'static str,
+ pub kind: String,
/// Rendered text for the text-bearing kinds (`heading`, `paragraph`,
/// `list_item`). Table text lives in `header`/`rows`; code and grid text in
/// `lines`.
@@ -78,9 +78,9 @@ pub struct LayoutBlock {
pub level: Option<u8>,
/// Whether the block's text is uniformly bold / italic. `paragraph` and
/// `list_item` only; omitted when false.
- #[serde(skip_serializing_if = "std::ops::Not::not")]
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub bold: bool,
- #[serde(skip_serializing_if = "std::ops::Not::not")]
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub italic: bool,
/// `list_item`: whether the list is ordered, and the original marker as it
/// appeared on the page (`138.`, `iii)`, `•`).
@@ -123,7 +123,7 @@ impl LayoutBlock {
/// Base value with every variant-specific field cleared.
fn of(kind: &'static str, bbox: Option<Rect>) -> Self {
LayoutBlock {
- kind,
+ kind: kind.to_string(),
text: None,
level: None,
bold: false,
@@ -236,7 +236,7 @@ pub(crate) fn blocks_for_page(page: &ParsedPage, blocks: &[PositionedBlock]) ->
/// map it through).
fn remap_to_page_frame(blocks: &mut [LayoutBlock], frames: &[(Rect, Rect)]) {
for block in blocks.iter_mut() {
- if matches!(block.kind, "figure" | "rule") {
+ if matches!(block.kind.as_str(), "figure" | "rule") {
continue;
}
remap_opt(&mut block.bbox, frames);
diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs
index 106f713..2a4f499 100644
--- a/crates/liteparse/src/lib.rs
+++ b/crates/liteparse/src/lib.rs
@@ -3,6 +3,10 @@
//! This crate is the core Rust library. Language bindings for Node.js, Python,
//! and WebAssembly re-export the same types with language-idiomatic wrappers.
//!
+//! [`LiteParse::parse`] runs a fixed pipeline; the [`stages`] module exposes
+//! each step of it as a public function for callers that want to sequence
+//! the pipeline themselves.
+//!
// ── Public API re-exports ──────────────────────────────────────────────
pub use config::{DEFAULT_PAGE_BATCH_SIZE, LiteParseConfig, OutputFormat};
@@ -23,6 +27,7 @@ pub mod layout;
pub mod parser;
pub mod raw_text;
pub mod search;
+pub mod stages;
pub mod types;
// ── Internal modules (available for binding crates, hidden from docs) ──
diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rs
index 59c483f..89ab502 100644
--- a/crates/liteparse/src/markdown_layout/blocks.rs
+++ b/crates/liteparse/src/markdown_layout/blocks.rs
@@ -2,13 +2,14 @@ use super::inline::escape_inline;
use super::paragraphs::{ParaAccum, is_soft_hyphen_break};
use super::tables::escape_table_cell;
use crate::types::Rect;
+use serde::{Deserialize, Serialize};
/// One table cell: rendered text plus, when the cell came from real page
/// content, the region it occupied. `bbox` is `None` for cells that exist only
/// to square off a ragged grid (padding inserted when rows disagree on column
/// count) — those occupy no ink on the page, and reporting a rect for them
/// would invent geometry the classifier never saw.
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Cell {
pub text: String,
pub bbox: Option<Rect>,
@@ -84,7 +85,7 @@ impl PartialEq<String> for Cell {
/// merge structure a producer *declared* — but both carry the region the cell
/// occupies when the producer knows it, so merged tables ground exactly like
/// plain ones.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpanCell {
pub text: String,
pub colspan: u16,
@@ -142,7 +143,7 @@ impl Eq for SpanCell {}
/// Coarse block representation: the output of page classification, consumed by
/// `render_blocks` to produce the final markdown string.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Block {
Heading {
level: u8,
@@ -250,7 +251,7 @@ fn wrap_emphasis(text: &str, bold: bool, italic: bool) -> String {
/// a wrapped heading or a multi-line paragraph reports the whole band it
/// occupies. It is `None` only for blocks with no page geometry behind them —
/// synthesized content, or sources that never carried coordinates.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PositionedBlock {
pub block: Block,
pub bbox: Option<Rect>,
diff --git a/crates/liteparse/src/ocr/mod.rs b/crates/liteparse/src/ocr/mod.rs
index a65685b..dcfb2e5 100644
--- a/crates/liteparse/src/ocr/mod.rs
+++ b/crates/liteparse/src/ocr/mod.rs
@@ -1,5 +1,7 @@
use std::pin::Pin;
+use serde::{Deserialize, Serialize};
+
#[cfg(not(target_arch = "wasm32"))]
pub mod http_simple;
#[cfg(all(feature = "oar-ocr", not(target_arch = "wasm32")))]
@@ -8,7 +10,7 @@ pub mod oar;
pub mod tesseract;
/// A single word-level OCR result with bounding box and confidence.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OcrResult {
pub text: String,
/// Bounding box in pixel coordinates: [x1, y1, x2, y2] (left, top, right, bottom).
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index 6ea2cbd..14aa440 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -3,9 +3,9 @@ use std::sync::Arc;
use crate::error::LiteParseError;
use crate::ocr::{OcrEngine, OcrOptions, OcrResult};
-use crate::types::{CutAxis, Page, ParsedPage, Region, RegionKind, TextItem};
+use crate::types::{CutAxis, Page, ParsedPage, Rect, Region, RegionKind, TextItem};
use pdfium::{Document, ImageBounds};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
/// Minimum dark filled-path area (pt², ~72 DPI page space) not covered by
/// native text before a page is sent to OCR. 400 pt² is roughly one word at
@@ -37,14 +37,24 @@ const MIN_COLUMN_ITEM_FRACTION: f32 = 0.15;
/// `DenseGraphics`.
const DENSE_GRAPHICS_MIN_COVERAGE: f32 = 0.2;
-/// Owned page bitmap prepared for OCR. Indices refer to positions in the `pages` slice.
-pub(crate) struct RenderedPage {
- pub idx: usize,
+/// Owned page bitmap prepared for OCR, plus the page facts the merge needs
+/// to judge the engine's output. Produced by [`render_pages_for_ocr`],
+/// consumed by [`recognize_rasters`]; the non-pixel facts are carried into
+/// the [`PageOcrOutcome`] so [`merge_ocr_results`] never needs the raster or
+/// the document again.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrRaster {
+ /// Position in the `pages` slice the raster was rendered from.
+ pub page_idx: usize,
+ /// 1-based source page number, for logs and for callers that key by
+ /// page rather than by slice position.
+ pub page_number: usize,
/// Tightly-packed pixels: 1 byte/px (grayscale) or 3 (RGB).
pub pixels: Vec<u8>,
pub width: u32,
pub height: u32,
- /// DPI this page was actually rendered at.
+ /// DPI this page was actually rendered at (the long-edge cap can lower it
+ /// below the requested DPI). Pixel → point scale is `72 / dpi`.
pub dpi: f32,
/// The page has real native text (it was not sent to OCR as scanned,
/// blank, or garbled). OCR is then an enrichment pass over native text,
@@ -53,13 +63,38 @@ pub(crate) struct RenderedPage {
/// Embedded raster figure rects (viewport pt, excluding full-page
/// backgrounds), so the merge can judge each figure's OCR output as a
/// group — a chart's axis/legend labels are dropped together.
- pub image_rects: Vec<ImageBounds>,
+ pub image_rects: Vec<Rect>,
+}
+
+/// How [`render_pages_for_ocr`] picks and rasterizes pages.
+#[derive(Debug, Clone, Default)]
+pub struct OcrRenderOptions {
+ /// Stop after this many rasters (`0` = no limit), so a long document can
+ /// be processed in bounded rounds; the function returns where to resume.
+ pub max_rasters: usize,
+ /// Requested render resolution; clamped per page by the long-edge cap.
+ pub dpi: f32,
+ /// Emit 1 byte/px luma instead of RGB (see `OcrEngine::prefers_grayscale`).
+ pub grayscale: bool,
+ /// Paint form-field appearances into the raster (runs document actions).
+ pub render_form_fields: bool,
+ /// Keep going when one page fails to render (it keeps its native text).
+ pub continue_on_page_error: bool,
+ /// Pages extraction flattened widget annotations on, which must be
+ /// flattened again on this (reopened) document so the raster shows what
+ /// extraction saw. See `ExtractedPages::flattened_page_numbers`.
+ pub reflatten_pages: HashSet<u32>,
+ /// Explicit 1-based page selection. `None` lets the page-complexity
+ /// scoring decide which pages need OCR (the default parse behaviour);
+ /// `Some` renders exactly these pages, whatever the scoring says, so a
+ /// caller with its own routing can override it.
+ pub selection: Option<HashSet<u32>>,
}
/// Why a page was flagged as needing more than the cheap text-only path.
/// Multiple reasons can apply to one page (e.g. a sparse page whose little
/// text is also garbled). Empty exactly when `needs_ocr` is false.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ComplexityReason {
/// A single raster covers essentially the whole page and there is little or
@@ -104,7 +139,7 @@ impl ComplexityReason {
}
}
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageComplexityStats {
pub page_number: usize,
pub text_length: usize,
@@ -438,7 +473,7 @@ pub fn calculate_native_page_complexity(
/// that the text-only path may mangle reading order or structure, so a caller
/// can route the page to a higher-accuracy pipeline. Same leniency contract
/// as `ComplexityReason`: new variants will be added, treat unknowns kindly.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum LayoutComplexityReason {
/// The XY-cut layout tree splits the page into two or more side-by-side
@@ -464,7 +499,7 @@ impl LayoutComplexityReason {
}
}
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LayoutComplexityStats {
/// Side-by-side text columns found by the XY-cut layout pass (1 = single
/// column). Only subtrees holding a substantial share of the page's text
@@ -587,31 +622,39 @@ fn count_columns(region: &Region, total_items: usize) -> usize {
}
}
-/// Render pages that need OCR from an already-open document.
-///
-/// The pdfium `Document` holds raw pointers that are not `Send`, so callers must
-/// drop it before awaiting the OCR engine.
-///
/// Cap on the rendered long edge (in pixels) for OCR page rasters
/// to avoid excessive memory usage.
pub(crate) const MAX_OCR_RENDER_LONG_EDGE_PX: f32 = 4096.0;
-/// Render the pages in `pages[start..]` that need OCR, stopping once
-/// `max_rasters` of them have been rendered (`0` means no limit). Returns the
-/// rasters plus the index to resume scanning from, so a caller can process a
-/// long document in bounded rounds.
-#[allow(clippy::too_many_arguments)]
-pub(crate) fn render_pages_for_ocr(
+/// Render the pages in `pages[start..]` that need OCR from an already-open
+/// document, stopping once `options.max_rasters` of them have been rendered.
+/// Returns the rasters plus the index to resume scanning from, so a caller
+/// can process a long document in bounded rounds.
+///
+/// The pdfium `Document` holds raw pointers that are not `Send`, so callers
+/// must drop it before awaiting the OCR engine.
+pub fn render_pages_for_ocr(
document: &Document,
pages: &[Page],
start: usize,
- max_rasters: usize,
- dpi: f32,
- grayscale: bool,
- render_form_fields: bool,
- continue_on_page_error: bool,
- flatten_page_numbers: &std::collections::HashSet<u32>,
-) -> Result<(Vec<RenderedPage>, usize), LiteParseError> {
+ options: &OcrRenderOptions,
+) -> Result<(Vec<OcrRaster>, usize), LiteParseError> {
+ let OcrRenderOptions {
+ max_rasters,
+ dpi,
+ grayscale,
+ render_form_fields,
+ continue_on_page_error,
+ reflatten_pages: flatten_page_numbers,
+ selection,
+ } = options;
+ let (max_rasters, dpi, grayscale, render_form_fields, continue_on_page_error) = (
+ *max_rasters,
+ *dpi,
+ *grayscale,
+ *render_form_fields,
+ *continue_on_page_error,
+ );
let mut rendered = Vec::new();
// With `render_form_fields`, draw form-field appearances into the OCR
// raster so filled-in form values are visible to the OCR engine (matches
@@ -625,7 +668,12 @@ pub(crate) fn render_pages_for_ocr(
}
let mut next_start = pages.len();
for (idx, page) in pages.iter().enumerate().skip(start) {
- let page_render = (|| -> Result<Option<RenderedPage>, LiteParseError> {
+ if let Some(selection) = selection
+ && !selection.contains(&(page.page_number as u32))
+ {
+ continue;
+ }
+ let page_render = (|| -> Result<Option<OcrRaster>, LiteParseError> {
let page_index = (page.page_number - 1) as i32;
// Text extraction may have flattened THIS page's widget
// annotations into page content. When the caller hands us a
@@ -644,7 +692,10 @@ pub(crate) fn render_pages_for_ocr(
};
let page_complexity = calculate_page_complexity(page, &page_obj)?;
- if !page_complexity.needs_ocr {
+ // An explicit selection overrides the scoring's OCR gate; the
+ // scoring still decides `has_native_text`, which drives the
+ // merge's artifact filters.
+ if selection.is_none() && !page_complexity.needs_ocr {
return Ok(None);
}
@@ -687,13 +738,20 @@ pub(crate) fn render_pages_for_ocr(
!(b.width > pw * MAX_IMAGE_PAGE_COVERAGE
&& b.height > ph * MAX_IMAGE_PAGE_COVERAGE)
})
+ .map(|b| Rect {
+ x: b.x,
+ y: b.y,
+ width: b.width,
+ height: b.height,
+ })
.collect()
} else {
Vec::new()
};
- Ok(Some(RenderedPage {
- idx,
+ Ok(Some(OcrRaster {
+ page_idx: idx,
+ page_number: page.page_number,
pixels,
width,
height,
@@ -723,27 +781,49 @@ pub(crate) fn render_pages_for_ocr(
Ok((rendered, next_start))
}
-/// Run OCR on pre-rendered page bitmaps and merge results into `pages`.
-pub(crate) async fn ocr_and_merge_rendered(
- pages: &mut [Page],
- rendered: Vec<RenderedPage>,
+/// One page's recognition outcome: what the engine returned for one
+/// [`OcrRaster`], plus the raster's page facts so the merge is a pure
+/// function of `(pages, outcomes)`. The pixels are gone — this is the small
+/// payload that crosses a stage boundary, not the bitmap.
+///
+/// `result` is `Err(message)` when the engine failed on this page; the merge
+/// counts failures to tell a systemic engine problem from a flaky page.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PageOcrOutcome {
+ /// Position in the `pages` slice the raster came from.
+ pub page_idx: usize,
+ /// 1-based source page number.
+ pub page_number: usize,
+ /// DPI the raster was rendered at; pixel → point scale is `72 / dpi`.
+ pub dpi: f32,
+ pub has_native_text: bool,
+ pub image_rects: Vec<Rect>,
+ /// Engine output in raster pixel coordinates.
+ pub result: Result<Vec<OcrResult>, String>,
+}
+
+/// Run `ocr_engine` over pre-rendered page bitmaps, at most `num_workers`
+/// pages at a time, returning one outcome per raster in the order given.
+/// Pure recognition: nothing is merged, and a failed page is reported as an
+/// `Err` outcome rather than aborting the batch.
+pub async fn recognize_rasters(
+ rendered: Vec<OcrRaster>,
ocr_engine: Arc<dyn OcrEngine>,
ocr_language: &str,
num_workers: usize,
- ocr_failure_fatal: bool,
-) -> Result<(), LiteParseError> {
+) -> Vec<PageOcrOutcome> {
type OcrTaskResult = Result<Vec<OcrResult>, Box<dyn std::error::Error + Send + Sync>>;
// Browser WASM uses the JavaScript event loop. It has no Tokio runtime or
// blocking thread pool. Run each JavaScript OCR callback directly so the
// returned Promise can make progress on the browser event loop.
#[cfg(target_arch = "wasm32")]
- let task_results: Vec<(usize, usize, f32, (bool, Vec<ImageBounds>), OcrTaskResult)> = {
+ let task_results: Vec<(usize, usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {
let _ = num_workers;
let mut results = Vec::with_capacity(rendered.len());
for r in rendered {
- let idx = r.idx;
- let page_number = pages[idx].page_number;
+ let idx = r.page_idx;
+ let page_number = r.page_number;
let page_dpi = r.dpi;
let native = (r.has_native_text, r.image_rects.clone());
let options = OcrOptions {
@@ -772,7 +852,7 @@ pub(crate) async fn ocr_and_merge_rendered(
// pass deadlocks. Acquiring the permit asynchronously parks the lightweight
// task instead, so only `num_workers` blocking threads are ever consumed.
#[cfg(not(target_arch = "wasm32"))]
- let task_results: Vec<(usize, usize, f32, (bool, Vec<ImageBounds>), OcrTaskResult)> = {
+ let task_results: Vec<(usize, usize, f32, (bool, Vec<Rect>), OcrTaskResult)> = {
let num_workers = num_workers.max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(num_workers));
let mut handles = Vec::with_capacity(rendered.len());
@@ -783,11 +863,11 @@ pub(crate) async fn ocr_and_merge_rendered(
let engine = ocr_engine.clone();
let sem = semaphore.clone();
let language = ocr_language.to_string();
- let page_number = pages[r.idx].page_number;
+ let page_number = r.page_number;
let rt_handle = handle.clone();
handles.push((
- r.idx,
+ r.page_idx,
page_number,
r.dpi,
(r.has_native_text, r.image_rects.clone()),
@@ -830,8 +910,34 @@ pub(crate) async fn ocr_and_merge_rendered(
results
};
- // Phase 3: collect results and merge into pages.
+ task_results
+ .into_iter()
+ .map(
+ |(page_idx, page_number, dpi, (has_native_text, image_rects), result)| PageOcrOutcome {
+ page_idx,
+ page_number,
+ dpi,
+ has_native_text,
+ image_rects,
+ result: result.map_err(|e| e.to_string()),
+ },
+ )
+ .collect()
+}
+/// Merge recognition outcomes into `pages`, in place. Pure: needs neither the
+/// document nor the rasters, only the outcomes [`recognize_rasters`] produced
+/// (or an equivalent a caller built from its own OCR engine's output).
+///
+/// Per page, unusable native text (garbled fonts, unmappable Type3) is dropped
+/// first so OCR can replace it; then each result is filtered against rule
+/// artifacts, chart figures and overlap with surviving native text before
+/// being appended as an `OCR`-font [`TextItem`] in viewport points.
+pub fn merge_ocr_results(
+ pages: &mut [Page],
+ outcomes: Vec<PageOcrOutcome>,
+ ocr_failure_fatal: bool,
+) -> Result<(), LiteParseError> {
// Track OCR task outcomes so we can distinguish a systemic failure (e.g.
// missing Tesseract language data, which fails identically on every page)
// from incidental per-page failures. Without this, every page logs the same
@@ -844,19 +950,26 @@ pub(crate) async fn ocr_and_merge_rendered(
// enrichment but already has all its text. We must only fail loud when OCR
// failure destroyed a sparse page's likely primary text source — otherwise
// a broken OCR setup would abort perfectly good native-text documents.
- let total_tasks = task_results.len();
+ let total_tasks = outcomes.len();
let mut failed_tasks = 0usize;
let mut failed_sparse_text_page = false;
let mut first_error: Option<String> = None;
- for (idx, page_number, page_dpi, (has_native_text, image_rects), result) in task_results {
+ for outcome in outcomes {
+ let PageOcrOutcome {
+ page_idx: idx,
+ page_number,
+ dpi: page_dpi,
+ has_native_text,
+ image_rects,
+ result,
+ } = outcome;
let ocr_results: Vec<OcrResult> = match result {
Ok(results) => results,
- Err(e) => {
+ Err(msg) => {
failed_tasks += 1;
failed_sparse_text_page |= page_has_sparse_native_text(&pages[idx]);
if first_error.is_none() {
- let msg = e.to_string();
eprintln!("[ocr] failed for page {}: {}", page_number, msg);
first_error = Some(msg);
}
@@ -1334,7 +1447,7 @@ const CHART_MIN_RESULTS: usize = 4;
/// tokens for the figure to count as a chart.
const CHART_LABEL_FRACTION: f32 = 0.5;
-fn box_center_in(rect: &ImageBounds, x: f32, y: f32, w: f32, h: f32) -> bool {
+fn box_center_in(rect: &Rect, x: f32, y: f32, w: f32, h: f32) -> bool {
let cx = x + w / 2.0;
let cy = y + h / 2.0;
cx >= rect.x && cx <= rect.x + rect.width && cy >= rect.y && cy <= rect.y + rect.height
@@ -1358,7 +1471,7 @@ fn is_chart_label(text: &str) -> bool {
/// For each figure rect, whether its OCR results look like a chart's
/// axes/ticks/legend rather than text worth keeping.
-fn chart_like_images(rects: &[ImageBounds], results: &[OcrResult], scale_factor: f32) -> Vec<bool> {
+fn chart_like_images(rects: &[Rect], results: &[OcrResult], scale_factor: f32) -> Vec<bool> {
rects
.iter()
.map(|rect| {
@@ -1469,7 +1582,7 @@ mod tests {
assert!(!is_chart_label("Year"));
assert!(!is_chart_label("Fruit Production in British Columbia"));
- let rect = ImageBounds {
+ let rect = Rect {
x: 50.0,
y: 100.0,
width: 200.0,
@@ -1996,11 +2109,12 @@ mod tests {
assert!(matches!(garbled_scope(&page), GarbledScope::WholePage));
}
- fn make_rendered(idx: usize) -> RenderedPage {
- RenderedPage {
+ fn make_rendered(idx: usize) -> OcrRaster {
+ OcrRaster {
has_native_text: false,
image_rects: Vec::new(),
- idx,
+ page_idx: idx,
+ page_number: idx + 1,
// 1x1 grayscale pixel; the engine never inspects it.
pixels: vec![0u8],
width: 1,
@@ -2009,6 +2123,20 @@ mod tests {
}
}
+ /// The pre-split entry point, kept for these tests: recognition followed
+ /// by the merge, exactly as `parse()` composes the two stages.
+ async fn ocr_and_merge_rendered(
+ pages: &mut [Page],
+ rendered: Vec<OcrRaster>,
+ engine: Arc<dyn OcrEngine>,
+ language: &str,
+ num_workers: usize,
+ ocr_failure_fatal: bool,
+ ) -> Result<(), LiteParseError> {
+ let outcomes = recognize_rasters(rendered, engine, language, num_workers).await;
+ merge_ocr_results(pages, outcomes, ocr_failure_fatal)
+ }
+
// A page that already has substantial native text coverage, as would be the
// case for a native-text PDF page that was only rendered for OCR because it
// also contains an image.
diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rs
index c31d229..2224bef 100644
--- a/crates/liteparse/src/output/markdown.rs
+++ b/crates/liteparse/src/output/markdown.rs
@@ -1,8 +1,6 @@
use crate::config::ImageMode;
-use crate::markdown_layout::{
- PositionedBlock, build_heading_map, classify_page_with_filters, compute_body_size,
- compute_header_footer_set, detect_single_page_chrome, render_blocks, splice_soft_hyphens,
-};
+use crate::markdown_layout::PositionedBlock;
+use crate::stages;
use crate::types::{OutlineTarget, ParsedPage};
/// Format parsed pages as markdown.
@@ -67,59 +65,15 @@ pub fn classify_document(
return Vec::new();
}
- let body_size = compute_body_size(pages);
- let heading_map = build_heading_map(pages, body_size);
- let header_footer = if keep_headers_footers {
- Default::default()
- } else {
- compute_header_footer_set(pages)
+ let signals = stages::document_signals(pages, keep_headers_footers);
+ let options = stages::BlockOptions {
+ outline,
+ image_mode,
+ keep_headers_footers,
};
-
pages
.iter()
- .map(|page| {
- if page.projected_lines.is_empty() {
- return None;
- }
-
- // Filter outline entries to this page so the classifier's y/title
- // match is a O(entries_on_page) scan per line, not O(whole doc).
- let target_index = (page.page_number as i32).saturating_sub(1);
- let page_outline: Vec<OutlineTarget> = outline
- .iter()
- .filter(|e| e.page_index == target_index)
- .cloned()
- .collect();
- let chrome_indices = if keep_headers_footers {
- Default::default()
- } else {
- detect_single_page_chrome(page, body_size)
- };
- let mut blocks = classify_page_with_filters(
- page,
- &heading_map,
- &header_footer,
- &page_outline,
- image_mode,
- &chrome_indices,
- );
- // A page whose only surviving blocks are horizontal rules (all its
- // text was stripped as chrome) should render empty, not as a stack
- // of bare `---` separators.
- let has_content = blocks.iter().any(|b| {
- !matches!(
- b.block,
- crate::markdown_layout::Block::HorizontalRule
- | crate::markdown_layout::Block::Figure { .. }
- )
- });
- if !has_content {
- blocks
- .retain(|b| !matches!(b.block, crate::markdown_layout::Block::HorizontalRule));
- }
- dedupe_rules(&mut blocks);
- Some(splice_soft_hyphens(blocks))
- })
+ .map(|page| stages::extract_blocks(page, &signals, &options))
.collect()
}
@@ -133,18 +87,7 @@ pub fn render_classified(
pages
.iter()
.zip(classified)
- .map(|(page, blocks)| match blocks {
- Some(blocks) => render_blocks(blocks),
- None => {
- let mut out = String::from("```text\n");
- out.push_str(&page.text);
- if !page.text.ends_with('\n') {
- out.push('\n');
- }
- out.push_str("```");
- out
- }
- })
+ .map(|(page, blocks)| stages::render_page_markdown(page, blocks.as_deref()))
.collect()
}
@@ -154,7 +97,7 @@ pub fn render_classified(
/// two sources — vector-graphics detection and decorative divider text — and
/// doubling up reads as sloppy output to a human, while carrying no extra
/// structure for an LLM.
-fn dedupe_rules(blocks: &mut Vec<crate::markdown_layout::PositionedBlock>) {
+pub(crate) fn dedupe_rules(blocks: &mut Vec<crate::markdown_layout::PositionedBlock>) {
use crate::markdown_layout::Block::HorizontalRule;
while matches!(blocks.first().map(|b| &b.block), Some(HorizontalRule)) {
blocks.remove(0);
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index ed364ce..6d0b14c 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -1,17 +1,23 @@
+//! The `LiteParse` orchestrator: input resolution, configuration, and the
+//! fixed parse sequence.
+//!
+//! Invariant: every step of `parse()` goes through [`crate::stages`]. This
+//! file owns the *sequence* and the config-to-stage plumbing; it must not
+//! reach into `extract`, `ocr_merge`, `projection` or `markdown_layout`
+//! directly, or a caller sequencing the stages themselves could no longer
+//! reproduce `parse()`. `tests/stages_compose.rs` asserts that equivalence.
+
use crate::config::{LiteParseConfig, parse_target_pages};
#[cfg(not(target_arch = "wasm32"))]
use crate::conversion;
use crate::error::LiteParseError;
-use crate::extract;
use crate::ocr::OcrEngine;
#[cfg(not(target_arch = "wasm32"))]
use crate::ocr::http_simple::HttpOcrEngine;
#[cfg(feature = "tesseract")]
use crate::ocr::tesseract::TesseractOcrEngine;
-use crate::ocr_merge;
-use crate::output::markdown;
-use crate::projection;
-use crate::render;
+
+use crate::stages;
use crate::types::{
DocumentMetadata, ExtractedImage, OutlineTarget, Page, PageError, ParsedPage, PdfInput,
ScreenshotRect, XfaPacket,
@@ -120,8 +126,9 @@ fn write_extracted_images(
/// by its own placement id (`![](img_p2_1.jpg)`), but only the canonical
/// file is written to disk (see `write_extracted_images`), so duplicate
/// placements must reference the canonical name, matching the resolution the
-/// LlamaParse worker applies via `resolveOutputImageName`.
-fn rewrite_duplicate_image_refs(
+/// LlamaParse worker applies via `resolveOutputImageName`. Public as
+/// [`stages::canonicalize_image_refs`].
+pub(crate) fn rewrite_duplicate_image_refs(
pages: &mut [ParsedPage],
full_text: &mut String,
images: &[ExtractedImage],
@@ -247,27 +254,31 @@ fn apply_layout(
if !wants_markdown && !config.extract_blocks {
return None;
}
- let classified = markdown::classify_document(
- parsed_pages,
+ let signals = stages::document_signals(parsed_pages, config.keep_headers_footers);
+ let block_options = stages::BlockOptions {
outline,
- config.image_mode,
- config.keep_headers_footers,
- );
- if config.extract_blocks {
- for (page, blocks) in parsed_pages.iter_mut().zip(&classified) {
+ image_mode: config.image_mode,
+ keep_headers_footers: config.keep_headers_footers,
+ };
+ let mut page_md = Vec::with_capacity(parsed_pages.len());
+ for page in parsed_pages.iter_mut() {
+ let blocks = stages::extract_blocks(page, &signals, &block_options);
+ if config.extract_blocks {
// A page with no structural decomposition reports an empty list,
// not `None` — extraction *was* enabled, there was just nothing to
// decompose.
- page.blocks = Some(crate::layout::blocks_for_page(
+ page.blocks = Some(stages::layout_blocks(
page,
blocks.as_deref().unwrap_or_default(),
));
}
+ if wants_markdown {
+ page_md.push(stages::render_page_markdown(page, blocks.as_deref()));
+ }
}
if !wants_markdown {
return None;
}
- let page_md = markdown::render_classified(parsed_pages, &classified);
let md = page_md.join("\n\n-----\n\n");
for (page, page_md) in parsed_pages.iter_mut().zip(page_md) {
page.markdown = page_md;
@@ -324,12 +335,12 @@ impl LiteParse {
input: &PdfInput,
password: Option<&str>,
) -> Result<pdfium::Document<'lib>, LiteParseError> {
- let document = extract::load_document_from_input(lib, input, password)?;
- extract::apply_page_orientation_corrections(
- &document,
+ stages::open(
+ lib,
+ input,
+ password,
&self.config.page_orientation_corrections,
- )?;
- Ok(document)
+ )
}
fn validate_output_config(&self) -> Result<(), LiteParseError> {
@@ -342,6 +353,118 @@ impl LiteParse {
Ok(())
}
+ /// The extract stage's request for a full parse, derived from config.
+ /// One place, so a caller composing the stages can build the same
+ /// request from the same config (`tests/stages_compose.rs` does exactly that).
+ pub fn extract_request<'a>(
+ &'a self,
+ target_pages: Option<&'a [u32]>,
+ max_pages: usize,
+ ) -> stages::ExtractRequest<'a> {
+ let markdown = self.config.output_format == crate::config::OutputFormat::Markdown;
+ stages::ExtractRequest {
+ target_pages,
+ max_pages,
+ extract_links: self.config.extract_links && markdown,
+ glyph_resolver: self.glyph_resolver.as_deref(),
+ output: stages::ExtractionOutputOptions {
+ continue_on_page_error: self.config.continue_on_page_error,
+ extract_content_bounds: self.config.extract_content_bounds,
+ extract_images: self.config.effective_extract_images(),
+ // The markdown table detector splits PDFium's merged
+ // multi-cell runs on real word geometry, so it needs word
+ // boxes even when the caller didn't ask for them.
+ emit_word_boxes: self.config.emit_word_boxes || markdown,
+ extract_text_metadata: self.config.extract_text_metadata,
+ extract_vector_graphics: self.config.extract_vector_graphics,
+ extract_annotations: self.config.extract_annotations,
+ extract_form_fields: self.config.extract_form_fields,
+ extract_structure_tree: self.config.extract_structure_tree,
+ },
+ }
+ }
+
+ /// The OCR render stage's options for a full parse, derived from config.
+ ///
+ /// Extraction may have flattened SOME pages' form widgets into page
+ /// content in its (since dropped) document instance; those pages are
+ /// re-flattened on the reopened OCR document so the rasters match what
+ /// extraction saw. With `render_form_fields` the form environment paints
+ /// the widgets instead, so no re-flatten is needed.
+ pub fn ocr_render_options(
+ &self,
+ grayscale: bool,
+ flattened_form_widgets: bool,
+ flattened_page_numbers: &[u32],
+ ) -> stages::OcrRenderOptions {
+ let reflatten_pages = if flattened_form_widgets && !self.config.render_form_fields {
+ flattened_page_numbers.iter().copied().collect()
+ } else {
+ std::collections::HashSet::new()
+ };
+ stages::OcrRenderOptions {
+ // One round per `num_workers` pages bounds raster memory.
+ max_rasters: self.config.num_workers.max(1),
+ dpi: self.config.dpi,
+ grayscale,
+ render_form_fields: self.config.render_form_fields,
+ continue_on_page_error: self.config.continue_on_page_error,
+ reflatten_pages,
+ selection: None,
+ }
+ }
+
+ /// The content-filter stage's options, derived from config.
+ pub fn content_filters(&self) -> stages::ContentFilters<'_> {
+ stages::ContentFilters {
+ crop_box: self.config.crop_box.as_ref(),
+ skip_diagonal_text: self.config.skip_diagonal_text,
+ }
+ }
+
+ /// The OCR engine a parse would use: the override when set, else the
+ /// HTTP engine for `ocr_server_url`, else built-in Tesseract. `None` when
+ /// OCR is disabled; an error when it is enabled but nothing can serve it.
+ pub fn ocr_engine(&self) -> Result<Option<std::sync::Arc<dyn OcrEngine>>, LiteParseError> {
+ if !self.config.ocr_enabled {
+ return Ok(None);
+ }
+ if let Some(engine) = self.ocr_engine_override.clone() {
+ return Ok(Some(engine));
+ }
+ #[cfg(not(target_arch = "wasm32"))]
+ {
+ if let Some(ref url) = self.config.ocr_server_url {
+ return Ok(Some(std::sync::Arc::new(
+ HttpOcrEngine::with_headers(
+ url.clone(),
+ self.config.ocr_server_headers.clone(),
+ )
+ .with_retry(crate::ocr::http_simple::OcrRetryConfig {
+ hedge_delays_ms: self.config.ocr_hedge_delays_ms.clone(),
+ ..Default::default()
+ }),
+ )));
+ }
+ #[cfg(feature = "tesseract")]
+ {
+ Ok(Some(std::sync::Arc::new(TesseractOcrEngine::new(
+ self.config.tessdata_path.clone(),
+ ))))
+ }
+ #[cfg(not(feature = "tesseract"))]
+ {
+ Err("OCR enabled but no --ocr-server-url provided and tesseract feature is disabled".into())
+ }
+ }
+ #[cfg(target_arch = "wasm32")]
+ {
+ Err(
+ "OCR enabled but no `ocrEngine` callback was provided (WASM builds have no built-in OCR engine)".into(),
+ )
+ }
+ }
+
/// Determine the complexity of each page in a document, returning a vector
/// of `PageComplexityStats` for each page. This is useful for deciding
/// whether to enable OCR on a per-page basis, or for other heuristics.
@@ -353,7 +476,7 @@ impl LiteParse {
pub async fn is_complex(
&self,
input: PdfInput,
- ) -> Result<Vec<ocr_merge::PageComplexityStats>, LiteParseError> {
+ ) -> Result<Vec<stages::PageComplexityStats>, LiteParseError> {
let log = |msg: &str| {
if !self.config.quiet {
eprintln!("{}", msg);
@@ -388,15 +511,17 @@ impl LiteParse {
// instead of routing the page to OCR to recover text we already
// have. `AnnotationText` still fires for the non-widget appearance
// text it was introduced for.
- let pages = extract::extract_pages_and_images(
+ let pages = stages::extract(
&document,
- target_pages.as_deref(),
- self.config.max_pages,
- false, // extract_links: irrelevant for complexity stats
- self.glyph_resolver.as_deref(),
- extract::ExtractionOutputOptions {
- continue_on_page_error: self.config.continue_on_page_error,
- ..Default::default()
+ &stages::ExtractRequest {
+ target_pages: target_pages.as_deref(),
+ max_pages: self.config.max_pages,
+ extract_links: false, // irrelevant for complexity stats
+ glyph_resolver: self.glyph_resolver.as_deref(),
+ output: stages::ExtractionOutputOptions {
+ continue_on_page_error: self.config.continue_on_page_error,
+ ..Default::default()
+ },
},
)?
.pages;
@@ -413,11 +538,7 @@ impl LiteParse {
let mut kept_pages = Vec::with_capacity(pages.len());
let mut page_complexities = Vec::with_capacity(pages.len());
for page in pages {
- let stats = document
- .page((page.page_number - 1) as i32)
- .map_err(LiteParseError::from)
- .and_then(|page_obj| ocr_merge::calculate_page_complexity(&page, &page_obj));
- match stats {
+ match stages::page_complexity(&document, &page) {
Ok(stats) => {
page_complexities.push(stats);
kept_pages.push(page);
@@ -445,9 +566,9 @@ impl LiteParse {
// Layout signals come from the real projection pass so they match
// what a full parse will decide.
let t_layout = web_time::Instant::now();
- let parsed_pages = projection::project_pages_to_grid(pages);
+ let parsed_pages = stages::project(pages);
for (stats, page) in page_complexities.iter_mut().zip(&parsed_pages) {
- stats.layout = Some(ocr_merge::calculate_layout_complexity(page));
+ stats.layout = Some(stages::layout_complexity(page));
}
log(&format!(
"[liteparse] layout: {:.1}ms",
@@ -547,48 +668,7 @@ impl LiteParse {
let password = self.config.password.as_deref();
// Build the OCR engine up front so the renderer knows whether to emit a
// grayscale buffer (cheaper, for engines that binarize internally) or RGB.
- let ocr_engine: Option<std::sync::Arc<dyn OcrEngine>> = if self.config.ocr_enabled {
- Some(if let Some(e) = self.ocr_engine_override.clone() {
- e
- } else {
- #[cfg(not(target_arch = "wasm32"))]
- {
- if let Some(ref url) = self.config.ocr_server_url {
- std::sync::Arc::new(
- HttpOcrEngine::with_headers(
- url.clone(),
- self.config.ocr_server_headers.clone(),
- )
- .with_retry(
- crate::ocr::http_simple::OcrRetryConfig {
- hedge_delays_ms: self.config.ocr_hedge_delays_ms.clone(),
- ..Default::default()
- },
- ),
- )
- } else {
- #[cfg(feature = "tesseract")]
- {
- std::sync::Arc::new(TesseractOcrEngine::new(
- self.config.tessdata_path.clone(),
- ))
- }
- #[cfg(not(feature = "tesseract"))]
- {
- return Err("OCR enabled but no --ocr-server-url provided and tesseract feature is disabled".into());
- }
- }
- }
- #[cfg(target_arch = "wasm32")]
- {
- return Err(
- "OCR enabled but no `ocrEngine` callback was provided (WASM builds have no built-in OCR engine)".into(),
- );
- }
- })
- } else {
- None
- };
+ let ocr_engine = self.ocr_engine()?;
let ocr_grayscale = ocr_engine.as_ref().is_some_and(|e| e.prefers_grayscale());
#[allow(unused_mut)] // mutated only by the native image-output writer
@@ -615,9 +695,7 @@ impl LiteParse {
let repaired_input = self
.config
.extract_form_fields
- .then(|| {
- crate::acroform_repair::repair_orphaned_widgets(&lib, validated_input, password)
- })
+ .then(|| stages::repair_acroform(&lib, validated_input, password))
.flatten();
#[cfg(not(target_arch = "wasm32"))]
let document_input = repaired_input.as_ref().unwrap_or(validated_input);
@@ -642,55 +720,20 @@ impl LiteParse {
// from the original document; fall back if it no longer loads.
#[cfg(not(target_arch = "wasm32"))]
if repaired_input.is_some()
- && let Ok(source) =
- extract::load_document_from_input(&lib, validated_input, password)
+ && let Ok(source) = self.open_document(&lib, validated_input, password)
{
- return crate::document_metadata::extract(validated_input, &source);
+ return stages::document_metadata(validated_input, &source);
}
- crate::document_metadata::extract(validated_input, &document)
- });
- let xfa_packets = self.config.extract_xfa_packets.then(|| {
- document
- .xfa_packets()
- .into_iter()
- .map(|packet| XfaPacket {
- index: packet.index.max(0) as u32,
- name: packet.name,
- content_length: packet
- .content
- .as_ref()
- .map_or(0, |content| content.len() as u32),
- content: packet
- .content
- .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()),
- })
- .collect::<Vec<_>>()
+ stages::document_metadata(validated_input, &document)
});
- let outline = outline.unwrap_or_else(|| extract::extract_outline(&document));
- let extracted = extract::extract_pages_and_images(
- &document,
- target_pages,
- max_pages,
- self.config.extract_links
- && self.config.output_format == crate::config::OutputFormat::Markdown,
- self.glyph_resolver.as_deref(),
- extract::ExtractionOutputOptions {
- continue_on_page_error: self.config.continue_on_page_error,
- extract_content_bounds: self.config.extract_content_bounds,
- extract_images: self.config.effective_extract_images(),
- // The markdown table detector splits PDFium's merged
- // multi-cell runs on real word geometry, so it needs word
- // boxes even when the caller didn't ask for them.
- emit_word_boxes: self.config.emit_word_boxes
- || self.config.output_format == crate::config::OutputFormat::Markdown,
- extract_text_metadata: self.config.extract_text_metadata,
- extract_vector_graphics: self.config.extract_vector_graphics,
- extract_annotations: self.config.extract_annotations,
- extract_form_fields: self.config.extract_form_fields,
- extract_structure_tree: self.config.extract_structure_tree,
- },
- )?;
- let extract::ExtractedPages {
+ let xfa_packets = self
+ .config
+ .extract_xfa_packets
+ .then(|| stages::xfa_packets(&document));
+ let outline = outline.unwrap_or_else(|| stages::outline(&document));
+ let extracted =
+ stages::extract(&document, &self.extract_request(target_pages, max_pages))?;
+ let stages::ExtractedPages {
pages,
page_errors,
images,
@@ -726,11 +769,7 @@ impl LiteParse {
let complexity = if self.config.include_complexity {
let mut complexity = Vec::with_capacity(pages.len());
for page in &pages {
- let stats = analysis_document
- .page((page.page_number - 1) as i32)
- .map_err(LiteParseError::from)
- .and_then(|page_obj| ocr_merge::calculate_page_complexity(page, &page_obj));
- match stats {
+ match stages::page_complexity(analysis_document, page) {
Ok(stats) => complexity.push(stats),
// The page's text is already extracted; a tolerant
// parse keeps it and just leaves `complexity` unset
@@ -751,24 +790,16 @@ impl LiteParse {
.iter()
.map(|page| page.page_number as u32)
.collect::<Vec<_>>();
- render::render_document_pages(
+ stages::screenshots(
analysis_document,
Some(&page_numbers),
- self.config.dpi,
- self.config.detect_screenshot_rects,
- self.config.render_form_fields,
- self.config.continue_on_page_error,
+ &stages::ScreenshotOptions {
+ dpi: self.config.dpi,
+ detect_rects: self.config.detect_screenshot_rects,
+ render_form_fields: self.config.render_form_fields,
+ continue_on_page_error: self.config.continue_on_page_error,
+ },
)?
- .into_iter()
- .map(|page| ScreenshotResult {
- page_num: page.page_num,
- width: page.width,
- height: page.height,
- image_bytes: page.png_bytes,
- is_solid_fill: page.is_solid_fill,
- rects: page.rects,
- })
- .collect()
} else {
Vec::new()
};
@@ -798,35 +829,18 @@ impl LiteParse {
let t1 = web_time::Instant::now();
if let Some(engine) = ocr_engine {
- let round_rasters = self.config.num_workers.max(1);
- // Extraction may have flattened SOME pages' form widgets into
- // page content in its (now dropped) document instance; re-apply
- // per round on exactly those pages so the rasters match what
- // extraction saw. With `render_form_fields` the form environment
- // paints the widgets instead, so no re-flatten is needed.
- let reflatten_pages: std::collections::HashSet<u32> =
- if flattened_form_widgets && !self.config.render_form_fields {
- flattened_page_numbers.iter().copied().collect()
- } else {
- std::collections::HashSet::new()
- };
+ let render_options = self.ocr_render_options(
+ ocr_grayscale,
+ flattened_form_widgets,
+ &flattened_page_numbers,
+ );
let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);
let mut round_start = 0usize;
while round_start < pages.len() {
let (rendered, next_start) = {
let lib = Library::init();
let document = self.open_document(&lib, ocr_input, password)?;
- ocr_merge::render_pages_for_ocr(
- &document,
- &pages,
- round_start,
- round_rasters,
- self.config.dpi,
- ocr_grayscale,
- self.config.render_form_fields,
- self.config.continue_on_page_error,
- &reflatten_pages,
- )?
+ stages::render_for_ocr(&document, &pages, round_start, &render_options)?
// `lib` drops here, releasing the PDFium lock before the
// engine's async recognition below.
};
@@ -836,17 +850,16 @@ impl LiteParse {
// that needs OCR.
continue;
}
- // `RenderedPage::idx` is absolute, so the whole slice is
+ // `OcrRaster::page_idx` is absolute, so the whole slice is
// passed regardless of where this round started.
- ocr_merge::ocr_and_merge_rendered(
- &mut pages,
+ let outcomes = stages::recognize(
rendered,
engine.clone(),
&self.config.ocr_language,
self.config.num_workers,
- self.config.ocr_failure_fatal,
)
- .await?;
+ .await;
+ stages::merge_ocr(&mut pages, outcomes, self.config.ocr_failure_fatal)?;
}
}
let t_ocr = web_time::Instant::now();
@@ -858,14 +871,10 @@ impl LiteParse {
// Caller-requested content filters (page-region crop, diagonal-text
// removal). Runs after OCR merge so it also drops OCR text outside the
// crop region, and before projection so filtered items never surface.
- extract::apply_content_filters(
- &mut pages,
- self.config.crop_box.as_ref(),
- self.config.skip_diagonal_text,
- );
+ stages::apply_content_filters(&mut pages, &self.content_filters());
// Grid projection
- let mut parsed_pages = projection::project_pages_to_grid(pages);
+ let mut parsed_pages = stages::project(pages);
// Attach per-page complexity signals, including the layout signals
// that need the projected page (same as `is_complex()` reports).
@@ -877,7 +886,7 @@ impl LiteParse {
else {
continue;
};
- stats.layout = Some(ocr_merge::calculate_layout_complexity(page));
+ stats.layout = Some(stages::layout_complexity(page));
page.complexity = Some(stats);
}
let t2 = web_time::Instant::now();
@@ -902,7 +911,7 @@ impl LiteParse {
.join("\n\n")
};
if self.config.output_format == crate::config::OutputFormat::Markdown {
- rewrite_duplicate_image_refs(&mut parsed_pages, &mut full_text, &images);
+ stages::canonicalize_image_refs(&mut parsed_pages, &mut full_text, &images);
}
let total = web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;
@@ -942,7 +951,7 @@ impl LiteParse {
/// own font-recovery pipeline) owns text extraction.
pub fn parse_from_pages(&self, pages: Vec<Page>, outline: Vec<OutlineTarget>) -> ParseResult {
let total_pages = pages.len().min(u32::MAX as usize) as u32;
- let mut parsed_pages = projection::project_pages_to_grid(pages);
+ let mut parsed_pages = stages::project(pages);
let full_text = if let Some(md) = apply_layout(&self.config, &mut parsed_pages, &outline) {
md
@@ -978,18 +987,18 @@ impl LiteParse {
fn select_block_pages(
&self,
pages: Vec<Page>,
- page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,
- complexity: Vec<Option<crate::ocr_merge::PageComplexityStats>>,
+ page_blocks: Vec<Vec<stages::PositionedBlock>>,
+ complexity: Vec<Option<stages::PageComplexityStats>>,
) -> Result<
(
Vec<Page>,
- Vec<Vec<crate::markdown_layout::PositionedBlock>>,
- Vec<Option<crate::ocr_merge::PageComplexityStats>>,
+ Vec<Vec<stages::PositionedBlock>>,
+ Vec<Option<stages::PageComplexityStats>>,
bool,
),
LiteParseError,
> {
- let mut selected: Vec<((Page, Vec<crate::markdown_layout::PositionedBlock>), _)> =
+ let mut selected: Vec<((Page, Vec<stages::PositionedBlock>), _)> =
pages.into_iter().zip(page_blocks).zip(complexity).collect();
let mut page_filtered = false;
if let Some(targets) = self.resolve_target_pages()? {
@@ -1021,11 +1030,11 @@ impl LiteParse {
pub fn parse_from_blocks(
&self,
pages: Vec<Page>,
- page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,
- all_blocks: Option<Vec<crate::markdown_layout::PositionedBlock>>,
+ page_blocks: Vec<Vec<stages::PositionedBlock>>,
+ all_blocks: Option<Vec<stages::PositionedBlock>>,
outline: Vec<OutlineTarget>,
images: Vec<crate::types::ExtractedImage>,
- page_stats: Vec<Option<crate::ocr_merge::PageComplexityStats>>,
+ page_stats: Vec<Option<stages::PageComplexityStats>>,
) -> Result<ParseResult, LiteParseError> {
// Reported before page filtering, matching every other path.
let total_pages = pages.len().min(u32::MAX as usize) as u32;
@@ -1034,13 +1043,13 @@ impl LiteParse {
let all_blocks = if page_filtered { None } else { all_blocks };
let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;
- let mut parsed_pages = projection::project_pages_to_grid(pages);
+ let mut parsed_pages = stages::project(pages);
for ((page, blocks), stats) in parsed_pages.iter_mut().zip(&page_blocks).zip(page_stats) {
if markdown_out {
- page.markdown = crate::markdown_layout::render_blocks(blocks);
+ page.markdown = stages::render_blocks(blocks);
}
if self.config.extract_blocks {
- page.blocks = Some(crate::layout::blocks_for_page(page, blocks));
+ page.blocks = Some(stages::layout_blocks(page, blocks));
}
page.complexity = stats;
if !self.config.extract_content_bounds {
@@ -1050,7 +1059,7 @@ impl LiteParse {
let full_text = if markdown_out {
match all_blocks {
- Some(blocks) => crate::markdown_layout::render_blocks(&blocks),
+ Some(blocks) => stages::render_blocks(&blocks),
None => parsed_pages
.iter()
.map(|p| p.markdown.as_str())
@@ -1122,27 +1131,19 @@ impl LiteParse {
log("[liteparse] converted input to PDF for screenshot rendering");
}
- let rendered = render::render_pages_to_png(
- &validated_input,
+ let lib = Library::init();
+ let document =
+ self.open_document(&lib, &validated_input, self.config.password.as_deref())?;
+ stages::screenshots(
+ &document,
page_numbers.as_deref(),
- self.config.dpi,
- self.config.password.as_deref(),
- self.config.detect_screenshot_rects,
- self.config.render_form_fields,
- &self.config.page_orientation_corrections,
- )?;
-
- Ok(rendered
- .into_iter()
- .map(|page| ScreenshotResult {
- page_num: page.page_num,
- width: page.width,
- height: page.height,
- image_bytes: page.png_bytes,
- is_solid_fill: page.is_solid_fill,
- rects: page.rects,
- })
- .collect())
+ &stages::ScreenshotOptions {
+ dpi: self.config.dpi,
+ detect_rects: self.config.detect_screenshot_rects,
+ render_form_fields: self.config.render_form_fields,
+ continue_on_page_error: false,
+ },
+ )
}
pub fn config(&self) -> &LiteParseConfig {
@@ -1177,14 +1178,11 @@ impl LiteParse {
// document-level bookmark walk is paid once instead of per batch.
let (total_pages, outline) = {
let lib = Library::init();
- let document = extract::load_document_from_input(
- &lib,
- &input.input,
- self.config.password.as_deref(),
- )?;
+ let document =
+ self.open_document(&lib, &input.input, self.config.password.as_deref())?;
(
document.page_count().max(0) as u32,
- extract::extract_outline(&document),
+ stages::outline(&document),
)
};
diff --git a/crates/liteparse/src/stages.rs b/crates/liteparse/src/stages.rs
new file mode 100644
index 0000000..42a96c4
--- /dev/null
+++ b/crates/liteparse/src/stages.rs
@@ -0,0 +1,462 @@
+//! The parse pipeline as public stage functions.
+//!
+//! [`LiteParse::parse`](crate::LiteParse::parse) is a fixed sequence of
+//! stages: open → orientation → extract → complexity → OCR render →
+//! recognition → OCR merge → content filters → grid projection → block
+//! classification → markdown rendering. This module exposes each stage as a
+//! function over serializable types so a caller can run the same sequence
+//! itself and insert its own inputs between stages — its own OCR engine, a
+//! page selection, extra blocks — or put a process boundary between any two.
+//!
+//! Two invariants hold this together:
+//!
+//! * `parse()` is implemented *only* through these functions (see
+//! `parser.rs`), and an integration test composes them by hand and asserts
+//! the result equals `parse()`. A step cannot be added to `parse()` without
+//! becoming a public stage.
+//! * Every type crossing a stage boundary is `Clone + Serialize +
+//! Deserialize`, and the serialization is lossless for everything a later
+//! stage reads.
+//!
+//! Stages split into two kinds. **pdfium-bound** stages take an open
+//! [`Document`] and must run while the [`Library`] that opened it is alive
+//! (PDFium is single-threaded behind a process-global lock, so keep these
+//! sections short and never hold a `Document` across an `.await`).
+//! **Pure** stages take and return plain data and can run anywhere.
+//!
+//! | Stage | Function | Kind |
+//! | --- | --- | --- |
+//! | open + orientation | [`open`], [`apply_orientation`] | pdfium |
+//! | document facts | [`outline`], [`xfa_packets`], [`document_metadata`] | pdfium |
+//! | extract | [`extract`] | pdfium |
+//! | complexity | [`page_complexity`], [`layout_complexity`] | pdfium / pure |
+//! | OCR | [`render_for_ocr`] → [`recognize`] → [`merge_ocr`] | pdfium / any / pure |
+//! | content filters | [`apply_content_filters`] | pure |
+//! | projection | [`project`] | pure |
+//! | markdown | [`document_signals`] → [`extract_blocks`] → [`render_page_markdown`] | pure |
+//! | screenshots | [`screenshots`] | pdfium |
+//!
+//! Everything else in the crate that is not re-exported here is unstable.
+
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use serde::{Deserialize, Serialize};
+
+use crate::config::{CropBox, ImageMode, PageOrientationCorrection};
+use crate::error::LiteParseError;
+use crate::extract;
+use crate::markdown_layout;
+use crate::ocr::OcrEngine;
+use crate::ocr_merge;
+use crate::projection;
+use crate::render;
+use crate::types::{
+ DocumentMetadata, ExtractedImage, OutlineTarget, Page, ParsedPage, PdfInput, XfaPacket,
+};
+
+pub use crate::extract::{ExtractedPages, ExtractionOutputOptions};
+pub use crate::layout::{LayoutBlock, LayoutCell};
+pub use crate::markdown_layout::{Block, Cell, PositionedBlock, SpanCell, render_blocks};
+pub use crate::ocr::OcrResult;
+pub use crate::ocr_merge::{
+ ComplexityReason, LayoutComplexityReason, LayoutComplexityStats, OcrRaster, OcrRenderOptions,
+ PageComplexityStats, PageOcrOutcome,
+};
+pub use crate::parser::ScreenshotResult;
+/// The PDFium handle types the pdfium-bound stages take. Re-exported so a
+/// caller needs no direct dependency on the pdfium crate.
+pub use pdfium::{Document, Library};
+
+// ── Open ───────────────────────────────────────────────────────────────
+
+/// Open `input` and apply per-page orientation corrections.
+///
+/// `/Rotate` rewrites live in the open document only, so every reopen of the
+/// same input (extraction, OCR rounds, screenshots) must go through this
+/// function with the same `corrections` to see the same page geometry.
+pub fn open<'lib>(
+ lib: &'lib Library,
+ input: &PdfInput,
+ password: Option<&str>,
+ corrections: &[PageOrientationCorrection],
+) -> Result<Document<'lib>, LiteParseError> {
+ let document = extract::load_document_from_input(lib, input, password)?;
+ apply_orientation(&document, corrections)?;
+ Ok(document)
+}
+
+/// Counter-rotate the pages named in `corrections` so their content reads
+/// upright. [`open`] already does this; call it directly only on a document
+/// opened some other way.
+pub fn apply_orientation(
+ document: &Document,
+ corrections: &[PageOrientationCorrection],
+) -> Result<(), LiteParseError> {
+ extract::apply_page_orientation_corrections(document, corrections)
+}
+
+/// Rewrite an AcroForm whose widgets are orphaned from their pages so
+/// form-field extraction can see them. Returns the repaired input when a
+/// rewrite was needed, `None` when the document is fine as is. The result is
+/// what to [`open`] for extraction; provenance facts should still come from
+/// the original input.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn repair_acroform(
+ lib: &Library,
+ input: &PdfInput,
+ password: Option<&str>,
+) -> Option<PdfInput> {
+ crate::acroform_repair::repair_orphaned_widgets(lib, input, password)
+}
+
+// ── Document facts ─────────────────────────────────────────────────────
+
+/// Walk the document outline (bookmarks). Document-level, and several times
+/// more expensive than opening the document, so a caller parsing in page
+/// batches should do it once.
+pub fn outline(document: &Document) -> Vec<OutlineTarget> {
+ extract::extract_outline(document)
+}
+
+/// Raw XFA packets from the document's `/XFA` array. Empty for a non-XFA
+/// document.
+pub fn xfa_packets(document: &Document) -> Vec<XfaPacket> {
+ document
+ .xfa_packets()
+ .into_iter()
+ .map(|packet| XfaPacket {
+ index: packet.index.max(0) as u32,
+ name: packet.name,
+ content_length: packet
+ .content
+ .as_ref()
+ .map_or(0, |content| content.len() as u32),
+ content: packet
+ .content
+ .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()),
+ })
+ .collect()
+}
+
+/// Provenance metadata (dates, security, signatures, incremental-save
+/// markers, XMP). `input` must be the file the facts describe: pass the
+/// original input, not an AcroForm-repaired or format-converted copy.
+pub fn document_metadata(input: &PdfInput, document: &Document) -> DocumentMetadata {
+ crate::document_metadata::extract(input, document)
+}
+
+// ── Extract ────────────────────────────────────────────────────────────
+
+/// What [`extract`] pulls out of the document.
+#[derive(Clone, Copy)]
+pub struct ExtractRequest<'a> {
+ /// 1-based pages to extract; `None` means every page.
+ pub target_pages: Option<&'a [u32]>,
+ /// Upper bound on extracted pages, applied after `target_pages`.
+ pub max_pages: usize,
+ /// Resolve link annotations onto the text items they cover.
+ pub extract_links: bool,
+ /// Last-resort glyph recovery for buggy fonts (see [`crate::GlyphResolver`]).
+ pub glyph_resolver: Option<&'a dyn crate::GlyphResolver>,
+ /// Which optional outputs to produce (images, word boxes, form fields…).
+ pub output: ExtractionOutputOptions,
+}
+
+impl Default for ExtractRequest<'_> {
+ fn default() -> Self {
+ Self {
+ target_pages: None,
+ max_pages: usize::MAX,
+ extract_links: false,
+ glyph_resolver: None,
+ output: ExtractionOutputOptions::default(),
+ }
+ }
+}
+
+/// Extract text items, graphics, structure nodes and (optionally) images
+/// from an open document. This is the only stage that reads page content
+/// through PDFium; everything downstream works on the returned [`Page`]s.
+pub fn extract(
+ document: &Document,
+ request: &ExtractRequest<'_>,
+) -> Result<ExtractedPages, LiteParseError> {
+ extract::extract_pages_and_images(
+ document,
+ request.target_pages,
+ request.max_pages,
+ request.extract_links,
+ request.glyph_resolver,
+ request.output,
+ )
+}
+
+// ── Complexity ─────────────────────────────────────────────────────────
+
+/// Score one extracted page: does it need OCR, and why. Reads page objects
+/// (images, filled paths) through PDFium, so it runs against the document
+/// the page was extracted from. `layout` is left unset; see
+/// [`layout_complexity`] for the post-projection half.
+pub fn page_complexity(
+ document: &Document,
+ page: &Page,
+) -> Result<PageComplexityStats, LiteParseError> {
+ let page_obj = document.page((page.page_number - 1) as i32)?;
+ ocr_merge::calculate_page_complexity(page, &page_obj)
+}
+
+/// Layout-difficulty signals (columns, tables, dense graphics) for a
+/// projected page. Pure; fills `PageComplexityStats::layout`.
+pub fn layout_complexity(page: &ParsedPage) -> LayoutComplexityStats {
+ ocr_merge::calculate_layout_complexity(page)
+}
+
+// ── OCR ────────────────────────────────────────────────────────────────
+
+/// Rasterize the pages in `pages[start..]` that need OCR (or that
+/// `options.selection` names), stopping after `options.max_rasters`. Returns
+/// the rasters and the index to resume from, so a long document runs in
+/// bounded rounds. Drop the [`Document`] before awaiting recognition.
+pub fn render_for_ocr(
+ document: &Document,
+ pages: &[Page],
+ start: usize,
+ options: &OcrRenderOptions,
+) -> Result<(Vec<OcrRaster>, usize), LiteParseError> {
+ ocr_merge::render_pages_for_ocr(document, pages, start, options)
+}
+
+/// Run an [`OcrEngine`] over rasters, at most `num_workers` at a time. Pure
+/// recognition; failures are per-outcome, not errors. A caller with its own
+/// OCR service can skip this and build [`PageOcrOutcome`]s directly from
+/// the [`OcrRaster`] facts and its engine's word boxes (raster pixel space).
+pub async fn recognize(
+ rasters: Vec<OcrRaster>,
+ engine: Arc<dyn OcrEngine>,
+ language: &str,
+ num_workers: usize,
+) -> Vec<PageOcrOutcome> {
+ ocr_merge::recognize_rasters(rasters, engine, language, num_workers).await
+}
+
+/// Merge recognition outcomes into `pages` in place: drop unusable native
+/// text, filter engine artifacts, append the surviving results as `OCR`
+/// text items in viewport points. Errors only when every outcome failed and
+/// one of the failed pages had no usable native text (`ocr_failure_fatal`).
+pub fn merge_ocr(
+ pages: &mut [Page],
+ outcomes: Vec<PageOcrOutcome>,
+ ocr_failure_fatal: bool,
+) -> Result<(), LiteParseError> {
+ ocr_merge::merge_ocr_results(pages, outcomes, ocr_failure_fatal)
+}
+
+// ── Content filters ────────────────────────────────────────────────────
+
+/// Caller-requested item filters applied between OCR merge and projection.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct ContentFilters<'a> {
+ /// Keep only items lying entirely inside the surviving page region.
+ pub crop_box: Option<&'a CropBox>,
+ /// Drop skewed text (watermarks, rotated stamps).
+ pub skip_diagonal_text: bool,
+}
+
+/// Apply [`ContentFilters`] to `pages` in place. Runs after [`merge_ocr`] so
+/// OCR text is filtered too, and before [`project`] so removed items never
+/// reach the output. No-op when no filter is requested.
+pub fn apply_content_filters(pages: &mut [Page], filters: &ContentFilters<'_>) {
+ extract::apply_content_filters(pages, filters.crop_box, filters.skip_diagonal_text);
+}
+
+// ── Projection ─────────────────────────────────────────────────────────
+
+/// Grid projection: turn each page's items into plain text with spatial
+/// layout, plus the per-line structure the markdown stages read. Pure and
+/// per page, so it can run over any page subset.
+pub fn project(pages: Vec<Page>) -> Vec<ParsedPage> {
+ projection::project_pages_to_grid(pages)
+}
+
+// ── Markdown ───────────────────────────────────────────────────────────
+
+/// Whole-document signals the block classifier needs: computed once over
+/// every page, then applied per page by [`extract_blocks`]. Heading levels
+/// are ranked against the document's body font size, and running
+/// headers/footers are the lines that repeat across pages — neither is
+/// knowable from one page alone. A caller that classifies pages in batches
+/// must compute this over the whole document first, or heading levels drift
+/// between batches.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct DocumentSignals {
+ /// Dominant body font size in points; `0.0` when no page has sized lines.
+ pub body_size: f32,
+ /// `(font_size, heading_level)` pairs, largest size first, levels 1..=6.
+ pub heading_map: Vec<(f32, u8)>,
+ /// Normalized texts of running headers/footers to suppress. Empty when
+ /// `keep_headers_footers` was set or the document is too short to tell.
+ pub header_footer: HashSet<String>,
+}
+
+/// Compute [`DocumentSignals`] over every page of the document.
+pub fn document_signals(pages: &[ParsedPage], keep_headers_footers: bool) -> DocumentSignals {
+ if pages.is_empty() {
+ return DocumentSignals::default();
+ }
+ let body_size = markdown_layout::compute_body_size(pages);
+ let heading_map = markdown_layout::build_heading_map(pages, body_size);
+ let header_footer = if keep_headers_footers {
+ HashSet::new()
+ } else {
+ markdown_layout::compute_header_footer_set(pages)
+ };
+ DocumentSignals {
+ body_size,
+ heading_map,
+ header_footer,
+ }
+}
+
+/// Per-document inputs to [`extract_blocks`] other than the signals.
+#[derive(Clone, Copy)]
+pub struct BlockOptions<'a> {
+ /// Document outline; entries for the page act as a high-priority heading
+ /// source on untagged PDFs.
+ pub outline: &'a [OutlineTarget],
+ /// Whether and how figures appear in the block list.
+ pub image_mode: ImageMode,
+ /// Disable running-chrome suppression (both the cross-page set in the
+ /// signals and the single-page detector).
+ pub keep_headers_footers: bool,
+}
+
+/// Classify one projected page into its final block sequence: headings,
+/// paragraphs, lists, code, tables, rules, figures — after chrome
+/// suppression, rule dedup and soft-hyphen splicing. This is exactly the
+/// decomposition [`render_page_markdown`] renders and
+/// `ParsedPage::blocks` reports.
+///
+/// Returns `None` for a page with no projected lines (blank, or fully OCR
+/// without font metadata): there is no structure to report, and rendering
+/// falls back to the projection text in a fence.
+pub fn extract_blocks(
+ page: &ParsedPage,
+ signals: &DocumentSignals,
+ options: &BlockOptions<'_>,
+) -> Option<Vec<PositionedBlock>> {
+ if page.projected_lines.is_empty() {
+ return None;
+ }
+
+ // Filter outline entries to this page so the classifier's y/title match
+ // is a O(entries_on_page) scan per line, not O(whole doc).
+ let target_index = (page.page_number as i32).saturating_sub(1);
+ let page_outline: Vec<OutlineTarget> = options
+ .outline
+ .iter()
+ .filter(|e| e.page_index == target_index)
+ .cloned()
+ .collect();
+ let chrome_indices = if options.keep_headers_footers {
+ HashSet::new()
+ } else {
+ markdown_layout::detect_single_page_chrome(page, signals.body_size)
+ };
+ let mut blocks = markdown_layout::classify_page_with_filters(
+ page,
+ &signals.heading_map,
+ &signals.header_footer,
+ &page_outline,
+ options.image_mode,
+ &chrome_indices,
+ );
+ // A page whose only surviving blocks are horizontal rules (all its text
+ // was stripped as chrome) should render empty, not as a stack of bare
+ // `---` separators.
+ let has_content = blocks
+ .iter()
+ .any(|b| !matches!(b.block, Block::HorizontalRule | Block::Figure { .. }));
+ if !has_content {
+ blocks.retain(|b| !matches!(b.block, Block::HorizontalRule));
+ }
+ crate::output::markdown::dedupe_rules(&mut blocks);
+ Some(markdown_layout::splice_soft_hyphens(blocks))
+}
+
+/// Render one page's markdown from its blocks. A page that classified to
+/// `None` renders its projection text inside a ```` ```text ```` fence so
+/// nothing is silently dropped.
+pub fn render_page_markdown(page: &ParsedPage, blocks: Option<&[PositionedBlock]>) -> String {
+ match blocks {
+ Some(blocks) => render_blocks(blocks),
+ None => {
+ let mut out = String::from("```text\n");
+ out.push_str(&page.text);
+ if !page.text.ends_with('\n') {
+ out.push('\n');
+ }
+ out.push_str("```");
+ out
+ }
+ }
+}
+
+/// The public, flat view of a page's blocks (`ParsedPage::blocks`), with
+/// block boxes mapped back from the projected frame into viewport space.
+pub fn layout_blocks(page: &ParsedPage, blocks: &[PositionedBlock]) -> Vec<LayoutBlock> {
+ crate::layout::blocks_for_page(page, blocks)
+}
+
+/// Point duplicate-image figure references at the canonical image's file
+/// name, in every page's markdown and in `full_text`. Only the canonical
+/// image is written to disk, so a `![](img_p2_1.jpg)` for a duplicate
+/// placement would otherwise reference a file that does not exist.
+pub fn canonicalize_image_refs(
+ pages: &mut [ParsedPage],
+ full_text: &mut String,
+ images: &[ExtractedImage],
+) {
+ crate::parser::rewrite_duplicate_image_refs(pages, full_text, images)
+}
+
+// ── Screenshots ────────────────────────────────────────────────────────
+
+/// How [`screenshots`] renders pages.
+#[derive(Debug, Clone, Copy)]
+pub struct ScreenshotOptions {
+ pub dpi: f32,
+ /// Detect solid rectangles/lines in the raster (`ScreenshotResult::rects`).
+ pub detect_rects: bool,
+ /// Paint form-field appearances (runs document actions).
+ pub render_form_fields: bool,
+ /// Skip pages that fail to render instead of failing the call.
+ pub continue_on_page_error: bool,
+}
+
+/// Render pages to PNG. `page_numbers` is 1-based; `None` renders every
+/// page.
+pub fn screenshots(
+ document: &Document,
+ page_numbers: Option<&[u32]>,
+ options: &ScreenshotOptions,
+) -> Result<Vec<ScreenshotResult>, LiteParseError> {
+ Ok(render::render_document_pages(
+ document,
+ page_numbers,
+ options.dpi,
+ options.detect_rects,
+ options.render_form_fields,
+ options.continue_on_page_error,
+ )?
+ .into_iter()
+ .map(|page| ScreenshotResult {
+ page_num: page.page_num,
+ width: page.width,
+ height: page.height,
+ image_bytes: page.png_bytes,
+ is_solid_fill: page.is_solid_fill,
+ rects: page.rects,
+ })
+ .collect())
+}
diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs
index 713ff70..abe88d0 100644
--- a/crates/liteparse/src/types.rs
+++ b/crates/liteparse/src/types.rs
@@ -1,4 +1,4 @@
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
#[doc(hidden)]
@@ -13,7 +13,7 @@ pub enum PdfInput {
/// Document-level provenance metadata extracted from PDFium plus a bounded
/// streaming scan of the source PDF. Fields stay optional so malformed
/// metadata never prevents the document itself from being parsed.
-#[derive(Debug, Clone, Default, Serialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DocumentMetadata {
#[serde(skip_serializing_if = "Option::is_none")]
pub creation_date: Option<String>,
@@ -58,7 +58,8 @@ pub struct DocumentMetadata {
/// Represents a single text item extracted from a PDF page,
/// including its content, position, size, rotation, and font metadata.
-#[derive(Debug, Clone, Default, Serialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(default)]
pub struct TextItem {
pub text: String,
/// Viewport-space coordinates (top-left origin, 72 DPI).
@@ -127,9 +128,10 @@ pub struct TextItem {
/// breaks at line/column boundaries), so this exposes the finer word-level
/// geometry needed for bbox attribution. Empty for items that produced no
/// word split (e.g. OCR-sourced or single-token items). Internal/attribution
- /// use only — `#[serde(skip)]` keeps it out of the JSON output but it is
- /// marshalled across the napi boundary.
- #[serde(skip)]
+ /// use only — the public JSON output (`output/json.rs`) builds its own
+ /// item view without it, but it is marshalled across the napi boundary
+ /// and must survive a stage boundary (see `stages`), so it serializes.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
pub words: Vec<WordBox>,
}
@@ -195,7 +197,7 @@ impl TextItem {
/// One word's bounding box within a `TextItem`, in the same viewport space
/// (top-left origin, 72 DPI) as the parent item. `text` is the word's content
/// with inter-word spaces excluded.
-#[derive(Debug, Clone, Default, Serialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct WordBox {
pub text: String,
pub x: f32,
@@ -204,8 +206,13 @@ pub struct WordBox {
pub height: f32,
}
+/// Serialization is lossless: every field the later stages read (graphics,
+/// structure nodes, image refs, per-item word boxes) round-trips through
+/// `Serialize`/`Deserialize`, so a caller sequencing the `stages` module can
+/// put a process or activity boundary between any two stages. The public JSON
+/// output does not use this shape (see `output/json.rs`).
#[doc(hidden)]
-#[derive(Debug, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Page {
pub page_number: usize,
/// The document's `/PageLabels` entry for this page ("iv", "A-1"), when
@@ -222,8 +229,9 @@ pub struct Page {
pub content_bounds: Option<Rect>,
pub text_items: Vec<TextItem>,
/// Vector graphics on the page, distilled from PDFium path objects.
- /// Not emitted in JSON/text outputs — consumed by the markdown layout pass.
- #[serde(skip)]
+ /// Not part of the public JSON/text outputs — consumed by the markdown
+ /// layout pass.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub graphics: Vec<GraphicPrimitive>,
/// Lossless-enough, PDFium-compatible path output requested by the caller.
/// Kept separate from the lossy internal `graphics` layout primitives.
@@ -232,26 +240,26 @@ pub struct Page {
/// Structure-tree nodes for this page when the PDF is tagged. Each node
/// carries its role, marked-content ids, and the union bbox of its tagged
/// content. Empty for untagged PDFs.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub struct_nodes: Vec<StructNode>,
/// Raster image objects detected on the page. Empty when the page has no
/// images. Threaded through to `ParsedPage.image_refs`.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub image_refs: Vec<ImageRef>,
/// Public annotation data when explicitly requested. `None` distinguishes
/// disabled extraction from an enabled page with no annotations.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub annotations: Option<Vec<DocumentAnnotation>>,
/// AcroForm widgets when explicitly requested.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub form_fields: Option<Vec<FormField>>,
/// Tagged-PDF logical structure when explicitly requested.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub structure_tree: Option<StructureTree>,
}
/// A page that could not be extracted while tolerant page errors were enabled.
-#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PageError {
/// Source page number (1-indexed). Serialized as `page` to match the
/// sibling per-page fields in the JSON output (`pages[]`, `images[]`).
@@ -263,7 +271,7 @@ pub struct PageError {
/// One PDF page annotation. Coordinates use the same top-left, 72-DPI
/// viewport space as [`TextItem`].
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DocumentAnnotation {
pub subtype: String,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -276,14 +284,14 @@ pub struct DocumentAnnotation {
pub title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rect: Option<Rect>,
- #[serde(skip_serializing_if = "Vec::is_empty")]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub quadpoint_rects: Vec<Rect>,
#[serde(skip_serializing_if = "Option::is_none")]
pub uri: Option<String>,
}
/// Scalar value from a tagged-PDF structure element's `/A` dictionary.
-#[derive(Debug, Clone, Serialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(untagged)]
pub enum StructureAttributeValue {
Boolean(bool),
@@ -292,14 +300,14 @@ pub enum StructureAttributeValue {
}
/// A complete page-scoped tagged-PDF logical structure tree.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StructureTree {
pub roots: Vec<StructureTreeElement>,
}
/// One tagged-PDF structure element. Field names follow the
/// repository's snake_case JSON convention rather than PDFium's C spellings.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StructureTreeElement {
#[serde(rename = "type")]
pub element_type: String,
@@ -311,7 +319,7 @@ pub struct StructureTreeElement {
pub alt_text: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
- #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+ #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub attributes: BTreeMap<String, StructureAttributeValue>,
pub marked_content_ids: Vec<i32>,
pub children: Vec<StructureTreeElement>,
@@ -319,7 +327,7 @@ pub struct StructureTreeElement {
}
/// One AcroForm widget and its resolved field metadata.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FormField {
pub id: String,
#[serde(rename = "type")]
@@ -346,15 +354,15 @@ pub struct FormField {
pub checked: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rect: Option<Rect>,
- #[serde(skip_serializing_if = "Vec::is_empty")]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub options: Vec<String>,
- #[serde(skip_serializing_if = "Vec::is_empty")]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub selected_options: Vec<String>,
}
/// One raw packet from an XFA form document's `/XFA` array. Surfaced on
/// `ParseResult.xfa_packets` when `extract_xfa_packets` is enabled.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XfaPacket {
/// Zero-based index in the XFA array.
pub index: u32,
@@ -373,7 +381,7 @@ pub struct XfaPacket {
/// Coordinates are in the same top-left, 72-DPI viewport space as text
/// items. Detection runs on the raster, so it also covers scanned/flattened
/// pages that carry no vector paths.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScreenshotRect {
pub x: f32,
pub y: f32,
@@ -390,7 +398,7 @@ pub struct ScreenshotRect {
/// user space (origin bottom-left) — convert to viewport with
/// `page_height - y` once you know the page.
#[doc(hidden)]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OutlineTarget {
/// Hierarchy depth, 1-based.
pub level: u8,
@@ -405,7 +413,7 @@ pub struct OutlineTarget {
/// One node from the structure tree of a page. Pre-flattened in pre-order
/// (parent before children).
#[doc(hidden)]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StructNode {
pub role: String,
pub mcids: Vec<i32>,
@@ -416,7 +424,11 @@ pub struct StructNode {
}
/// Represents a fully parsed page with projected text layout.
-#[derive(Debug, Serialize)]
+///
+/// Like [`Page`], serialization is lossless so the markdown stages can run on
+/// the other side of a process boundary from projection. The public JSON
+/// output builds its own view of this type (`output/json.rs`).
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParsedPage {
pub page_number: usize,
/// The document's `/PageLabels` entry for this page ("iv", "A-1"), when
@@ -432,21 +444,21 @@ pub struct ParsedPage {
#[serde(skip_serializing_if = "Option::is_none")]
pub content_bounds: Option<Rect>,
pub text: String,
- #[serde(skip_serializing_if = "String::is_empty")]
+ #[serde(default, skip_serializing_if = "String::is_empty")]
pub markdown: String,
pub text_items: Vec<TextItem>,
/// Per-line structural metadata used by the markdown emitter. Not part of
- /// the JSON/text outputs (consumed internally) so it is `#[serde(skip)]`.
- #[serde(skip)]
+ /// the public JSON/text outputs.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub projected_lines: Vec<ProjectedLine>,
/// Root of the XY-cut region tree for this page. Leaves correspond to the
/// `region_path` on each `ProjectedLine`. Internal-only.
- #[serde(skip)]
+ #[serde(default)]
pub regions: Region,
/// Vector graphics on the page (decomposed paths) used by the markdown
/// emitter for ruled-table / HR / figure-cluster detection. Not part of
- /// the JSON/text output.
- #[serde(skip)]
+ /// the public JSON/text output.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub graphics: Vec<GraphicPrimitive>,
/// Public vector path extraction. Absent unless explicitly enabled.
#[serde(skip_serializing_if = "Option::is_none")]
@@ -454,7 +466,7 @@ pub struct ParsedPage {
/// Figure-region bounding rectangles derived from `graphics`. Pre-computed
/// in `to_parsed_pages` so the XY-cut layout pass can treat them as
/// obstacles, and reused downstream for figure classification.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub figures: Vec<Rect>,
/// `(projected, original)` geometry of every text item, populated only
/// when the two differ for at least one item on the page. Rotation
@@ -465,18 +477,18 @@ pub struct ParsedPage {
/// through this table before they are reported, so they land in the
/// same viewport space as `text_items`. Empty for pages without rotated
/// text, which is the overwhelming majority.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub projected_item_frames: Vec<(Rect, Rect)>,
/// Structure-tree nodes for this page (tagged PDFs only). Pre-flattened in
/// pre-order. Consumed by the markdown classifier for highest-priority
/// heading / figure / table detection.
- #[serde(skip)]
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub struct_nodes: Vec<StructNode>,
/// Raster image objects on the page. Bbox in viewport coords. Populated
/// during extraction; consumed by the markdown emitter to interleave
/// `Block::Figure` references at the right y position. Empty when the
- /// page has no embedded images. Not part of JSON/text output.
- #[serde(skip)]
+ /// page has no embedded images. Not part of the public JSON/text output.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub image_refs: Vec<ImageRef>,
/// Per-page complexity signals (the same the `is_complex` API returns).
/// Populated only when `LiteParseConfig::include_complexity` is set;
@@ -507,7 +519,7 @@ pub struct ParsedPage {
/// the image's position among image page-objects, so a later embed pass can
/// re-open the document and pull pixel bytes with `render_image_object`.
#[doc(hidden)]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImageRef {
pub id: String,
pub bbox: Rect,
@@ -516,7 +528,13 @@ pub struct ImageRef {
pub pixel_width: u32,
pub pixel_height: u32,
pub rotation: f32,
+ /// Pixel payloads are consumed during extraction (they become
+ /// `ExtractedImage.bytes`) and nothing after extraction reads them, so
+ /// they stay out of the serialized page: a stage boundary after extract
+ /// carries the image bytes on `ExtractedImage`, not here.
+ #[serde(skip)]
pub jpeg_bytes: Option<Vec<u8>>,
+ #[serde(skip)]
pub raw_bytes: Option<Vec<u8>>,
pub bits_per_pixel: u32,
pub colorspace: i32,
@@ -528,7 +546,7 @@ pub struct ImageRef {
/// produced. JPEG streams are preserved without re-encoding when PDFium
/// exposes a valid directly decoded DCT stream; other images are encoded as
/// PNG from PDFium's rendered bitmap.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExtractedImage {
pub id: String,
pub name: String,
@@ -543,13 +561,15 @@ pub struct ExtractedImage {
#[serde(skip_serializing_if = "Option::is_none")]
pub duplicate_of: Option<String>,
/// Encoded image bytes. Shared (`Arc`) so duplicate entries reference the
- /// canonical image's buffer instead of copying it per occurrence.
+ /// canonical image's buffer instead of copying it per occurrence. Not
+ /// serialized: an image payload crossing a boundary goes as a file or a
+ /// separate blob, keyed by `id`.
#[serde(skip)]
pub bytes: std::sync::Arc<Vec<u8>>,
}
#[doc(hidden)]
-#[derive(Debug, Clone, Default, PartialEq, Serialize)]
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Rect {
pub x: f32,
pub y: f32,
@@ -588,14 +608,14 @@ impl Rect {
/// Page-scoped vector path output. Coordinates use the same top-left,
/// 72-DPI viewport space as text items.
-#[derive(Debug, Clone, Default, Serialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct VectorGraphics {
pub shapes: Vec<VectorShape>,
pub lines: Vec<VectorLine>,
}
/// One PDF path object's paint state and viewport bounding box.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VectorShape {
pub bbox: Rect,
pub stroke: bool,
@@ -609,7 +629,7 @@ pub struct VectorShape {
/// A strict horizontal or vertical path segment after adjacent compatible
/// segments have been merged, matching LlamaParse PDFium path semantics.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VectorLine {
pub x1: f32,
pub y1: f32,
@@ -630,7 +650,7 @@ pub struct VectorLine {
/// clusters) are kept — bezier curves and complex paths are decomposed into
/// straight strokes, or dropped.
#[doc(hidden)]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum GraphicPrimitive {
/// A single straight line segment in viewport coords. Used for HR/table
/// border detection.
@@ -673,7 +693,7 @@ impl GraphicPrimitive {
/// Per-line structural metadata derived during grid projection. Used by the
/// markdown emitter; not surfaced in JSON/text output.
#[doc(hidden)]
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProjectedLine {
pub text: String,
pub bbox: Rect,
@@ -721,14 +741,14 @@ pub struct ProjectedLine {
/// XY-cut region tree node. A page's root region recursively splits along H or
/// V axes until each leaf holds a coherent block of items.
#[doc(hidden)]
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Region {
pub bbox: Rect,
pub kind: RegionKind,
}
#[doc(hidden)]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RegionKind {
Leaf {
item_indices: Vec<usize>,
@@ -748,7 +768,7 @@ impl Default for RegionKind {
}
#[doc(hidden)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum CutAxis {
Horizontal,
Vertical,
@@ -763,7 +783,7 @@ pub enum Snap {
}
#[doc(hidden)]
-#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Anchor {
Left,
Right,
diff --git a/crates/liteparse/tests/stages_compose.rs b/crates/liteparse/tests/stages_compose.rs
new file mode 100644
index 0000000..ff49a27
--- /dev/null
+++ b/crates/liteparse/tests/stages_compose.rs
@@ -0,0 +1,604 @@
+//! Gate for the `stages` module: `parse()` must equal the stages composed by
+//! hand.
+//!
+//! `LiteParse::parse` is written only in terms of `liteparse::stages`. This
+//! test re-implements that sequence here, from the public stage functions,
+//! and asserts the two produce identical output over the fixture corpus and
+//! several configurations (markdown with every extraction option, OCR in
+//! bounded rounds through a mock engine, orientation corrections, content
+//! filters, AcroForm repair). If a step is added to `parse()` without a
+//! public stage, this test is what breaks.
+//!
+//! Along the way the composed pipeline serializes and deserializes the page
+//! set at two stage boundaries (after extract, after projection), so the
+//! same test proves the boundary types round-trip losslessly.
+
+use std::sync::Arc;
+
+use liteparse::config::{CropBox, ImageMode, OutputFormat, PageOrientationCorrection};
+use liteparse::ocr::{OcrEngine, OcrOptions, OcrResult};
+use liteparse::stages::{self, Library};
+use liteparse::types::{Page, ParsedPage, PdfInput};
+use liteparse::{LiteParse, LiteParseConfig, ParseResult};
+use serial_test::serial;
+
+const FIXTURES: &str = "../../integration_tests_data";
+
+fn skip_integration() -> bool {
+ std::env::var("SKIP_INTEGRATION_TESTS").as_deref() == Ok("yes")
+}
+
+fn fixture(name: &str) -> String {
+ format!("{FIXTURES}/{name}")
+}
+
+/// Every PDF in the fixture corpus, sorted for stable failure output.
+fn corpus_pdfs() -> Vec<String> {
+ let mut paths: Vec<String> = std::fs::read_dir(FIXTURES)
+ .expect("fixture directory")
+ .filter_map(|entry| {
+ let path = entry.ok()?.path();
+ (path.extension()? == "pdf").then(|| path.to_string_lossy().into_owned())
+ })
+ .collect();
+ paths.sort();
+ assert!(!paths.is_empty(), "no fixture PDFs found under {FIXTURES}");
+ paths
+}
+
+/// Deterministic mock OCR: one box per raster whose text encodes the raster
+/// size, so a raster routed to the wrong page is visible in the output.
+struct MockOcr;
+
+impl OcrEngine for MockOcr {
+ fn name(&self) -> &str {
+ "mock"
+ }
+ fn recognize<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ _image_data: &'c [u8],
+ width: u32,
+ height: u32,
+ _options: &'b OcrOptions,
+ ) -> std::pin::Pin<
+ Box<
+ dyn Future<Output = Result<Vec<OcrResult>, Box<dyn std::error::Error + Send + Sync>>>
+ + Send
+ + '_,
+ >,
+ > {
+ Box::pin(async move {
+ Ok(vec![
+ OcrResult {
+ text: format!("MOCK {width}x{height}"),
+ bbox: [10.0, 10.0, 260.0, 40.0],
+ confidence: 0.99,
+ polygon: None,
+ },
+ OcrResult {
+ text: "rotated".into(),
+ bbox: [300.0, 100.0, 330.0, 400.0],
+ confidence: 0.9,
+ polygon: Some([
+ [330.0, 100.0],
+ [330.0, 400.0],
+ [300.0, 400.0],
+ [300.0, 100.0],
+ ]),
+ },
+ ])
+ })
+ }
+}
+
+/// Everything `ParseResult` reports, in a comparable form. Pages go through
+/// their (lossless) serde form so every field is compared, not just the
+/// rendered strings.
+#[derive(Debug, PartialEq)]
+struct Snapshot {
+ total_pages: u32,
+ pages: Vec<serde_json::Value>,
+ page_errors: Vec<(u32, String)>,
+ text: String,
+ outline: String,
+ images: Vec<serde_json::Value>,
+ screenshots: Vec<(u32, u32, u32, bool, usize, Vec<u8>)>,
+ image_error_count: u32,
+ form_type: Option<i32>,
+ creator: Option<String>,
+ producer: Option<String>,
+ doc_meta: Option<serde_json::Value>,
+ xfa_packets: Option<serde_json::Value>,
+}
+
+fn snapshot(result: &ParseResult) -> Snapshot {
+ Snapshot {
+ total_pages: result.total_pages,
+ pages: result
+ .pages
+ .iter()
+ .map(|p| serde_json::to_value(p).unwrap())
+ .collect(),
+ page_errors: result
+ .page_errors
+ .iter()
+ .map(|e| (e.page_number, e.message.clone()))
+ .collect(),
+ text: result.text.clone(),
+ outline: format!("{:?}", result.outline),
+ images: result
+ .images
+ .iter()
+ .map(|i| serde_json::to_value(i).unwrap())
+ .collect(),
+ screenshots: result
+ .screenshots
+ .iter()
+ .map(|s| {
+ (
+ s.page_num,
+ s.width,
+ s.height,
+ s.is_solid_fill,
+ s.rects.len(),
+ s.image_bytes.clone(),
+ )
+ })
+ .collect(),
+ image_error_count: result.image_error_count,
+ form_type: result.form_type,
+ creator: result.creator.clone(),
+ producer: result.producer.clone(),
+ doc_meta: result
+ .doc_meta
+ .as_ref()
+ .map(|m| serde_json::to_value(m).unwrap()),
+ xfa_packets: result
+ .xfa_packets
+ .as_ref()
+ .map(|x| serde_json::to_value(x).unwrap()),
+ }
+}
+
+/// Serialize and deserialize `value`, asserting the round trip is lossless
+/// (re-serializing the deserialized value yields the same JSON).
+fn round_trip<T: serde::Serialize + serde::de::DeserializeOwned>(value: T, what: &str) -> T {
+ let json = serde_json::to_string(&value).unwrap();
+ let back: T = serde_json::from_str(&json).unwrap_or_else(|e| panic!("{what}: {e}"));
+ assert_eq!(
+ serde_json::to_string(&back).unwrap(),
+ json,
+ "{what}: serde round trip changed the value"
+ );
+ back
+}
+
+/// `parse()` written out by hand from the public stages. Mirrors
+/// `LiteParse::parse_resolved` step for step; the point is that nothing in
+/// that function is unreachable from here.
+async fn compose(parser: &LiteParse, input: PdfInput) -> ParseResult {
+ let config = parser.config();
+ let password = config.password.as_deref();
+ let markdown = config.output_format == OutputFormat::Markdown;
+ let target_pages = config
+ .target_pages
+ .as_ref()
+ .map(|s| liteparse::config::parse_target_pages(s).unwrap());
+
+ // Non-PDF inputs are converted before the pipeline starts; a PDF passes
+ // through untouched.
+ let (input, _guard) = liteparse::conversion::resolve_pdf_input(input, password, false)
+ .await
+ .unwrap();
+ let want_doc_meta = config.extract_document_metadata && !_guard.is_converted();
+
+ let engine = parser.ocr_engine().unwrap();
+ let grayscale = engine.as_ref().is_some_and(|e| e.prefers_grayscale());
+
+ // ── pdfium-bound: open, document facts, extract, complexity, screenshots
+ let lib = Library::init();
+ let repaired = config
+ .extract_form_fields
+ .then(|| stages::repair_acroform(&lib, &input, password))
+ .flatten();
+ let document_input = repaired.as_ref().unwrap_or(&input);
+ let document = stages::open(
+ &lib,
+ document_input,
+ password,
+ &config.page_orientation_corrections,
+ )
+ .unwrap();
+ let total_pages = document.page_count().max(0) as u32;
+ let form_type = config.extract_form_fields.then(|| document.form_type());
+ let creator = document.meta_text("Creator").filter(|v| !v.is_empty());
+ let producer = document.meta_text("Producer").filter(|v| !v.is_empty());
+ let doc_meta = want_doc_meta.then(|| {
+ if repaired.is_some() {
+ let source = stages::open(&lib, &input, password, &[]).unwrap();
+ return stages::document_metadata(&input, &source);
+ }
+ stages::document_metadata(&input, &document)
+ });
+ let xfa_packets = config
+ .extract_xfa_packets
+ .then(|| stages::xfa_packets(&document));
+ let outline = stages::outline(&document);
+ let extracted = stages::extract(
+ &document,
+ &parser.extract_request(target_pages.as_deref(), config.max_pages),
+ )
+ .unwrap();
+ let stages::ExtractedPages {
+ pages,
+ page_errors,
+ mut images,
+ image_error_count,
+ flattened_form_widgets,
+ flattened_page_numbers,
+ } = extracted;
+ let complexity: Vec<stages::PageComplexityStats> = if config.include_complexity {
+ pages
+ .iter()
+ .map(|page| stages::page_complexity(&document, page).unwrap())
+ .collect()
+ } else {
+ Vec::new()
+ };
+ let screenshots = if config.extract_screenshots {
+ let page_numbers: Vec<u32> = pages.iter().map(|p| p.page_number as u32).collect();
+ stages::screenshots(
+ &document,
+ Some(&page_numbers),
+ &stages::ScreenshotOptions {
+ dpi: config.dpi,
+ detect_rects: config.detect_screenshot_rects,
+ render_form_fields: config.render_form_fields,
+ continue_on_page_error: config.continue_on_page_error,
+ },
+ )
+ .unwrap()
+ } else {
+ Vec::new()
+ };
+ drop(document);
+ drop(lib);
+
+ // Boundary 1: the extracted page set crosses a (simulated) process
+ // boundary before OCR.
+ let mut pages: Vec<Page> = round_trip(pages, "Vec<Page> after extract");
+
+ // ── OCR: render rounds (pdfium) → recognize (async) → merge (pure)
+ if let Some(engine) = engine {
+ let options =
+ parser.ocr_render_options(grayscale, flattened_form_widgets, &flattened_page_numbers);
+ let ocr_input = repaired.as_ref().unwrap_or(&input);
+ let mut start = 0;
+ while start < pages.len() {
+ let (rasters, next) = {
+ let lib = Library::init();
+ let document = stages::open(
+ &lib,
+ ocr_input,
+ password,
+ &config.page_orientation_corrections,
+ )
+ .unwrap();
+ stages::render_for_ocr(&document, &pages, start, &options).unwrap()
+ };
+ start = next;
+ if rasters.is_empty() {
+ continue;
+ }
+ let rasters = round_trip(rasters, "Vec<OcrRaster>");
+ let outcomes = stages::recognize(
+ rasters,
+ engine.clone(),
+ &config.ocr_language,
+ config.num_workers,
+ )
+ .await;
+ let outcomes = round_trip(outcomes, "Vec<PageOcrOutcome>");
+ stages::merge_ocr(&mut pages, outcomes, config.ocr_failure_fatal).unwrap();
+ }
+ }
+
+ // ── pure: filters, projection, complexity layout half
+ stages::apply_content_filters(&mut pages, &parser.content_filters());
+ let parsed = stages::project(pages);
+
+ // Boundary 2: projected pages cross before the markdown stages.
+ let mut parsed: Vec<ParsedPage> = round_trip(parsed, "Vec<ParsedPage> after project");
+
+ let mut complexity = complexity.into_iter().peekable();
+ for page in parsed.iter_mut() {
+ if let Some(mut stats) = complexity.next_if(|s| s.page_number == page.page_number) {
+ stats.layout = Some(stages::layout_complexity(page));
+ page.complexity = Some(stats);
+ }
+ }
+
+ // ── markdown: document signals once, then per page
+ if markdown || config.extract_blocks {
+ let signals = stages::document_signals(&parsed, config.keep_headers_footers);
+ // `header_footer` is a set, so its JSON order is not stable across
+ // serializations; compare the parts rather than the bytes.
+ let back: stages::DocumentSignals =
+ serde_json::from_str(&serde_json::to_string(&signals).unwrap()).unwrap();
+ assert_eq!(back.body_size, signals.body_size);
+ assert_eq!(back.heading_map, signals.heading_map);
+ assert_eq!(back.header_footer, signals.header_footer);
+ let signals = back;
+ let options = stages::BlockOptions {
+ outline: &outline,
+ image_mode: config.image_mode,
+ keep_headers_footers: config.keep_headers_footers,
+ };
+ for page in parsed.iter_mut() {
+ let blocks = stages::extract_blocks(page, &signals, &options);
+ let blocks = blocks.map(|b| round_trip(b, "Vec<PositionedBlock>"));
+ if config.extract_blocks {
+ page.blocks = Some(stages::layout_blocks(
+ page,
+ blocks.as_deref().unwrap_or_default(),
+ ));
+ }
+ if markdown {
+ page.markdown = stages::render_page_markdown(page, blocks.as_deref());
+ }
+ }
+ }
+ let mut text = if markdown {
+ parsed
+ .iter()
+ .map(|p| p.markdown.as_str())
+ .collect::<Vec<_>>()
+ .join("\n\n-----\n\n")
+ } else {
+ parsed
+ .iter()
+ .map(|p| p.text.as_str())
+ .collect::<Vec<_>>()
+ .join("\n\n")
+ };
+ if markdown {
+ stages::canonicalize_image_refs(&mut parsed, &mut text, &images);
+ }
+ // `parse()` writes image files only with `image_output_dir`; none of the
+ // configurations here set it, so `path` stays unset on both sides.
+ for image in images.iter_mut() {
+ image.path = None;
+ }
+
+ ParseResult {
+ total_pages,
+ pages: parsed,
+ page_errors,
+ text,
+ outline,
+ images,
+ screenshots,
+ image_error_count,
+ form_type,
+ creator,
+ producer,
+ doc_meta,
+ xfa_packets,
+ }
+}
+
+/// Parse `path` both ways, assert equality, and hand back the result so a
+/// caller can check the configuration actually exercised what it meant to.
+async fn assert_parse_equals_composition(
+ config: LiteParseConfig,
+ path: &str,
+ with_ocr: bool,
+) -> ParseResult {
+ let mut parser = LiteParse::new(config);
+ if with_ocr {
+ parser = parser.with_ocr_engine(Arc::new(MockOcr));
+ }
+ let input = PdfInput::Path(path.to_string());
+ let via_parse = parser
+ .parse_input(input.clone())
+ .await
+ .unwrap_or_else(|e| panic!("parse() failed on {path}: {e}"));
+ let via_stages = compose(&parser, input).await;
+ assert_eq!(
+ snapshot(&via_parse),
+ snapshot(&via_stages),
+ "parse() and the composed stages diverged on {path}"
+ );
+ via_parse
+}
+
+/// Every extraction option on, no OCR: exercises extract, complexity,
+/// screenshots, projection, blocks and markdown, and the duplicate-image
+/// rewrite.
+fn everything_config() -> LiteParseConfig {
+ LiteParseConfig {
+ ocr_enabled: false,
+ quiet: true,
+ output_format: OutputFormat::Markdown,
+ image_mode: ImageMode::Embed,
+ extract_images: true,
+ extract_blocks: true,
+ include_complexity: true,
+ extract_content_bounds: true,
+ emit_word_boxes: true,
+ extract_text_metadata: true,
+ extract_vector_graphics: true,
+ extract_annotations: true,
+ extract_structure_tree: true,
+ extract_document_metadata: true,
+ extract_xfa_packets: true,
+ extract_screenshots: true,
+ detect_screenshot_rects: true,
+ extract_links: true,
+ ..LiteParseConfig::default()
+ }
+}
+
+#[tokio::test]
+#[serial]
+async fn parse_equals_composed_stages_markdown_with_every_option() {
+ let (mut with_images, mut with_blocks, mut with_screenshots) = (0, 0, 0);
+ for path in corpus_pdfs() {
+ let result = assert_parse_equals_composition(everything_config(), &path, false).await;
+ with_images += usize::from(!result.images.is_empty());
+ with_blocks += usize::from(
+ result
+ .pages
+ .iter()
+ .any(|p| p.blocks.as_ref().is_some_and(|b| !b.is_empty())),
+ );
+ with_screenshots += usize::from(!result.screenshots.is_empty());
+ }
+ // The equality is only meaningful if the branches ran. None of the
+ // fixture PDFs embeds a raster image; the demo 10-K does on three pages,
+ // so those pages (plus one without) cover image extraction, the figure
+ // block and the duplicate-image rewrite.
+ assert!(with_blocks > 0, "no fixture produced blocks");
+ assert!(with_screenshots > 0, "no fixture produced screenshots");
+ let config = LiteParseConfig {
+ target_pages: Some("1,2,108,119".into()),
+ ..everything_config()
+ };
+ let result =
+ assert_parse_equals_composition(config, "../../demo/docs/apple-10k-2024.pdf", false).await;
+ with_images += usize::from(!result.images.is_empty());
+ assert!(with_images > 0, "no fixture produced images");
+}
+
+/// Text output with blocks: the non-markdown branch of the layout stage.
+#[tokio::test]
+#[serial]
+async fn parse_equals_composed_stages_text_output() {
+ let config = LiteParseConfig {
+ ocr_enabled: false,
+ quiet: true,
+ output_format: OutputFormat::Text,
+ extract_blocks: true,
+ ..LiteParseConfig::default()
+ };
+ for path in corpus_pdfs() {
+ assert_parse_equals_composition(config.clone(), &path, false).await;
+ }
+}
+
+/// OCR through the stage split, one page per round (`num_workers: 1`) so the
+/// render → recognize → merge loop runs several times, plus the
+/// selection-by-complexity gate and the native-text artifact filters.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn parse_equals_composed_stages_with_ocr_rounds() {
+ let config = LiteParseConfig {
+ ocr_enabled: true,
+ num_workers: 1,
+ quiet: true,
+ output_format: OutputFormat::Markdown,
+ include_complexity: true,
+ ..LiteParseConfig::default()
+ };
+ let mut ocr_pages = 0;
+ for path in corpus_pdfs() {
+ let result = assert_parse_equals_composition(config.clone(), &path, true).await;
+ ocr_pages += result
+ .pages
+ .iter()
+ .filter(|p| p.text_items.iter().any(|i| i.text.starts_with("MOCK ")))
+ .count();
+ }
+ assert!(ocr_pages > 0, "no fixture page had mock OCR text merged in");
+}
+
+/// A converted image input is a scanned page: OCR is its only text.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn parse_equals_composed_stages_on_converted_image() {
+ if skip_integration() {
+ return;
+ }
+ let config = LiteParseConfig {
+ ocr_enabled: true,
+ quiet: true,
+ output_format: OutputFormat::Markdown,
+ ..LiteParseConfig::default()
+ };
+ let result = assert_parse_equals_composition(config, &fixture("receipt.png"), true).await;
+ assert!(
+ result.text.contains("MOCK "),
+ "a converted image should be OCR-only: {}",
+ result.text
+ );
+}
+
+/// Orientation corrections applied at open, a page selection, and the
+/// content filters.
+#[tokio::test]
+#[serial]
+async fn parse_equals_composed_stages_with_orientation_selection_and_filters() {
+ let config = LiteParseConfig {
+ ocr_enabled: false,
+ quiet: true,
+ output_format: OutputFormat::Markdown,
+ page_orientation_corrections: vec![PageOrientationCorrection { page: 1, angle: 90 }],
+ target_pages: Some("1".into()),
+ max_pages: 1,
+ crop_box: Some(CropBox {
+ top: 0.05,
+ bottom: 0.05,
+ left: 0.0,
+ right: 0.0,
+ }),
+ skip_diagonal_text: true,
+ ..LiteParseConfig::default()
+ };
+ for name in ["sample_rotated_90cw.pdf", "diagonal_text.pdf"] {
+ assert_parse_equals_composition(config.clone(), &fixture(name), false).await;
+ }
+}
+
+/// Form-field extraction: AcroForm repair before open, widget flattening
+/// during extract, and the re-flatten on the reopened OCR document.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn parse_equals_composed_stages_with_form_fields() {
+ let config = LiteParseConfig {
+ ocr_enabled: true,
+ quiet: true,
+ output_format: OutputFormat::Markdown,
+ extract_form_fields: true,
+ extract_document_metadata: true,
+ ..LiteParseConfig::default()
+ };
+ assert_parse_equals_composition(config, &fixture("filled_acroform.pdf"), true).await;
+}
+
+/// Sanity check on the round-trip helper itself: a page with every optional
+/// field populated survives serde unchanged, including the fields the public
+/// JSON output omits (word boxes, graphics, structure nodes, image refs).
+#[tokio::test]
+#[serial]
+async fn extracted_pages_round_trip_keeps_internal_fields() {
+ let parser = LiteParse::new(everything_config());
+ let lib = Library::init();
+ let document = stages::open(&lib, &PdfInput::Path(fixture("sample.pdf")), None, &[]).unwrap();
+ let pages = stages::extract(&document, &parser.extract_request(None, usize::MAX))
+ .unwrap()
+ .pages;
+ let has_words = pages
+ .iter()
+ .any(|p| p.text_items.iter().any(|i| !i.words.is_empty()));
+ assert!(has_words, "fixture should produce word boxes");
+ let back = round_trip(pages.clone(), "Vec<Page>");
+ for (a, b) in pages.iter().zip(&back) {
+ assert_eq!(a.text_items.len(), b.text_items.len());
+ assert_eq!(a.graphics.len(), b.graphics.len());
+ assert_eq!(a.struct_nodes.len(), b.struct_nodes.len());
+ assert_eq!(a.image_refs.len(), b.image_refs.len());
+ for (x, y) in a.text_items.iter().zip(&b.text_items) {
+ assert_eq!(x.words.len(), y.words.len());
+ }
+ }
+}