Skip to content

A fast, helpful, and open-source document parser

Merge branch 'main' into pierre/tolerant-page-errors

Logan Markewichbeca73a8ee8f
diff --cc crates/liteparse-napi/src/types.rs
index 0eb479c,1504abc..34c2ee6
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@@ -888,8 -881,9 +888,10 @@@ impl JsParsedPage
#[napi(object)]
#[derive(Clone)]
pub struct JsParseResult {
+ /// Total source-document pages before target/max-page filtering.
+ pub total_pages: u32,
pub pages: Vec<JsParsedPage>,
+ pub page_errors: Vec<JsPageError>,
pub text: String,
pub images: Vec<JsExtractedImage>,
pub image_error_count: u32,
@@@ -905,13 -899,17 +907,24 @@@
pub xfa_packets: Option<Vec<JsXfaPacket>>,
}
+ /// One batch of pages from a `ParseSession`.
+ #[napi(object)]
+ pub struct JsParseBatch {
+ /// First source page in this batch, 1-indexed.
+ pub start_page: u32,
+ /// Last source page in this batch, 1-indexed and inclusive.
+ pub end_page: u32,
+ /// The pages in `startPage..=endPage`, as an ordinary parse result.
+ pub result: JsParseResult,
+ }
+
+#[napi(object)]
+#[derive(Clone)]
+pub struct JsPageError {
+ pub page_num: u32,
+ pub message: String,
+}
+
#[napi(object)]
#[derive(Clone)]
pub struct JsDocumentMetadata {
diff --cc crates/liteparse-python/src/lib.rs
index 3bc1f4b,c9c16b2..912aae4
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@@ -1583,7 -1685,8 +1712,9 @@@ fn _liteparse(m: &Bound<'_, PyModule>)
m.add_class::<LiteParse>()?;
m.add_class::<PyLiteParseConfig>()?;
m.add_class::<PyParseResult>()?;
+ m.add_class::<PyPageError>()?;
+ m.add_class::<PyParseBatch>()?;
+ m.add_class::<PyParseSession>()?;
m.add_class::<PyDocumentMetadata>()?;
m.add_class::<PyExtractedImage>()?;
m.add_class::<PyImageRect>()?;
diff --cc crates/liteparse-wasm/src/lib.rs
index 07dc79e,57176cc..7543559
--- a/crates/liteparse-wasm/src/lib.rs
+++ b/crates/liteparse-wasm/src/lib.rs
@@@ -925,175 -911,181 +927,189 @@@ impl LiteParse
.await
.map_err(|e| JsError::new(&format!("parse failed: {}", e)))?;
- let extract_text_metadata = self.inner.config().extract_text_metadata;
- let pages: Vec<ParsedPage> = result
- .pages
- .iter()
- .map(|p| ParsedPage {
- page_num: p.page_number,
- width: p.page_width,
- height: p.page_height,
- content_bounds: p.content_bounds.as_ref().map(|b| VectorRect {
- x: b.x,
- y: b.y,
- width: b.width,
- height: b.height,
- }),
- text: p.text.clone(),
- markdown: p.markdown.clone(),
- text_items: p
- .text_items
- .iter()
- .map(|i| {
- // Core-gated metadata view; `TextMetadata` defines
- // which fields `extractTextMetadata` covers.
- let meta = i.text_metadata(extract_text_metadata);
- TextItem {
- text: i.text.clone(),
- x: i.x,
- y: i.y,
- width: i.width,
- height: i.height,
- font_name: i.font_name.clone(),
- font_size: i.font_size,
- confidence: i.confidence,
- rotation: i.rotation,
- font_height: meta.font_height,
- font_ascent: meta.font_ascent,
- font_descent: meta.font_descent,
- font_weight: meta.font_weight,
- text_width: meta.text_width,
- font_is_buggy: meta.font_is_buggy,
- mcid: meta.mcid,
- fill_color: meta.fill_color.map(str::to_owned),
- stroke_color: meta.stroke_color.map(str::to_owned),
- char_codes: meta
- .char_codes
- .filter(|codes| !codes.is_empty())
- .map(<[u32]>::to_vec),
- trailing_space_generated: meta.trailing_space_generated,
- words: if i.words.is_empty() {
- None
- } else {
- Some(
- i.words
- .iter()
- .map(|w| WordBox {
- text: w.text.clone(),
- x: w.x,
- y: w.y,
- width: w.width,
- height: w.height,
- })
- .collect(),
- )
- },
- }
- })
- .collect(),
- complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust),
- vector_graphics: p.vector_graphics.as_ref().map(|v| VectorGraphics {
- shapes: v
- .shapes
- .iter()
- .map(|s| VectorShape {
- bbox: VectorRect {
- x: s.bbox.x,
- y: s.bbox.y,
- width: s.bbox.width,
- height: s.bbox.height,
- },
- stroke: s.stroke,
- stroke_color: s.stroke_color.clone(),
- fill: s.fill,
- fill_color: s.fill_color.clone(),
- has_curve: s.has_curve,
- })
- .collect(),
- lines: v
- .lines
- .iter()
- .map(|l| VectorLine {
- x1: l.x1,
- y1: l.y1,
- x2: l.x2,
- y2: l.y2,
- stroke: l.stroke,
- stroke_width: l.stroke_width,
- stroke_color: l.stroke_color.clone(),
- fill: l.fill,
- fill_color: l.fill_color.clone(),
- })
- .collect(),
- }),
- annotations: p.annotations.as_ref().map(|annotations| {
- annotations
- .iter()
- .map(DocumentAnnotation::from_rust)
- .collect()
- }),
- form_fields: p
- .form_fields
- .as_ref()
- .map(|fields| fields.iter().map(FormField::from_rust).collect()),
- structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust),
- })
- .collect();
+ Ok(to_js_result(
+ &result,
+ self.inner.config().extract_text_metadata,
+ ))
+ }
+ }
- let images: Vec<ExtractedImage> = result
- .images
- .iter()
- .map(|img| ExtractedImage {
- id: img.id.clone(),
- name: img.name.clone(),
- path: img.path.clone(),
- page: img.page,
- bbox: ImageRect {
- x: img.bbox.x,
- y: img.bbox.y,
- width: img.bbox.width,
- height: img.bbox.height,
- },
- width: img.width,
- height: img.height,
- rotation: img.rotation,
- format: img.format.clone(),
- duplicate_of: img.duplicate_of.clone(),
- bytes: img.bytes.as_slice().to_vec(),
- })
- .collect();
-
- Ok(ParseResult {
- pages,
- text: result.text.clone(),
- images,
- image_error_count: result.image_error_count,
- page_errors: result
- .page_errors
+ /// Convert a core [`CoreParseResult`] into the wasm-bindgen view.
+ ///
+ /// Shared by `parse()` and `ParseSession::nextBatch()` so a batch is mapped
+ /// exactly the same way a whole-document result is.
+ fn to_js_result(result: &liteparse::ParseResult, extract_text_metadata: bool) -> ParseResult {
+ let pages: Vec<ParsedPage> = result
+ .pages
+ .iter()
+ .map(|p| ParsedPage {
+ page_num: p.page_number,
+ width: p.page_width,
+ height: p.page_height,
+ content_bounds: p.content_bounds.as_ref().map(|b| VectorRect {
+ x: b.x,
+ y: b.y,
+ width: b.width,
+ height: b.height,
+ }),
+ text: p.text.clone(),
+ markdown: p.markdown.clone(),
+ text_items: p
+ .text_items
.iter()
- .map(|error| PageError {
- page_num: error.page_number,
- message: error.message.clone(),
+ .map(|i| {
+ // Core-gated metadata view; `TextMetadata` defines
+ // which fields `extractTextMetadata` covers.
+ let meta = i.text_metadata(extract_text_metadata);
+ TextItem {
+ text: i.text.clone(),
+ x: i.x,
+ y: i.y,
+ width: i.width,
+ height: i.height,
+ font_name: i.font_name.clone(),
+ font_size: i.font_size,
+ confidence: i.confidence,
+ rotation: i.rotation,
+ font_height: meta.font_height,
+ font_ascent: meta.font_ascent,
+ font_descent: meta.font_descent,
+ font_weight: meta.font_weight,
+ text_width: meta.text_width,
+ font_is_buggy: meta.font_is_buggy,
+ mcid: meta.mcid,
+ fill_color: meta.fill_color.map(str::to_owned),
+ stroke_color: meta.stroke_color.map(str::to_owned),
+ char_codes: meta
+ .char_codes
+ .filter(|codes| !codes.is_empty())
+ .map(<[u32]>::to_vec),
+ trailing_space_generated: meta.trailing_space_generated,
+ words: if i.words.is_empty() {
+ None
+ } else {
+ Some(
+ i.words
+ .iter()
+ .map(|w| WordBox {
+ text: w.text.clone(),
+ x: w.x,
+ y: w.y,
+ width: w.width,
+ height: w.height,
+ })
+ .collect(),
+ )
+ },
+ }
})
.collect(),
- form_type: result.form_type,
- creator: result.creator.clone(),
- producer: result.producer.clone(),
- doc_meta: result.doc_meta.as_ref().map(DocumentMetadata::from),
- xfa_packets: result.xfa_packets.as_ref().map(|packets| {
- packets
+ complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust),
+ vector_graphics: p.vector_graphics.as_ref().map(|v| VectorGraphics {
+ shapes: v
+ .shapes
.iter()
- .map(|packet| XfaPacket {
- index: packet.index,
- name: packet.name.clone(),
- content_length: packet.content_length,
- content: packet.content.clone(),
+ .map(|s| VectorShape {
+ bbox: VectorRect {
+ x: s.bbox.x,
+ y: s.bbox.y,
+ width: s.bbox.width,
+ height: s.bbox.height,
+ },
+ stroke: s.stroke,
+ stroke_color: s.stroke_color.clone(),
+ fill: s.fill,
+ fill_color: s.fill_color.clone(),
+ has_curve: s.has_curve,
})
+ .collect(),
+ lines: v
+ .lines
+ .iter()
+ .map(|l| VectorLine {
+ x1: l.x1,
+ y1: l.y1,
+ x2: l.x2,
+ y2: l.y2,
+ stroke: l.stroke,
+ stroke_width: l.stroke_width,
+ stroke_color: l.stroke_color.clone(),
+ fill: l.fill,
+ fill_color: l.fill_color.clone(),
+ })
+ .collect(),
+ }),
+ annotations: p.annotations.as_ref().map(|annotations| {
+ annotations
+ .iter()
+ .map(DocumentAnnotation::from_rust)
.collect()
}),
+ form_fields: p
+ .form_fields
+ .as_ref()
+ .map(|fields| fields.iter().map(FormField::from_rust).collect()),
+ structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust),
+ })
+ .collect();
+
+ let images: Vec<ExtractedImage> = result
+ .images
+ .iter()
+ .map(|img| ExtractedImage {
+ id: img.id.clone(),
+ name: img.name.clone(),
+ path: img.path.clone(),
+ page: img.page,
+ bbox: ImageRect {
+ x: img.bbox.x,
+ y: img.bbox.y,
+ width: img.bbox.width,
+ height: img.bbox.height,
+ },
+ width: img.width,
+ height: img.height,
+ rotation: img.rotation,
+ format: img.format.clone(),
+ duplicate_of: img.duplicate_of.clone(),
+ bytes: img.bytes.as_slice().to_vec(),
})
+ .collect();
+
+ ParseResult {
+ total_pages: result.total_pages,
+ pages,
+ text: result.text.clone(),
+ images,
+ image_error_count: result.image_error_count,
++ page_errors: result
++ .page_errors
++ .iter()
++ .map(|error| PageError {
++ page_num: error.page_number,
++ message: error.message.clone(),
++ })
++ .collect(),
+ form_type: result.form_type,
+ creator: result.creator.clone(),
+ producer: result.producer.clone(),
+ doc_meta: result.doc_meta.as_ref().map(DocumentMetadata::from),
+ xfa_packets: result.xfa_packets.as_ref().map(|packets| {
+ packets
+ .iter()
+ .map(|packet| XfaPacket {
+ index: packet.index,
+ name: packet.name.clone(),
+ content_length: packet.content_length,
+ content: packet.content.clone(),
+ })
+ .collect()
+ }),
}
+ }
+ #[wasm_bindgen]
+ impl LiteParse {
/// Determine per-page complexity for the given PDF bytes. Returns
/// `Promise<PageComplexityStats[]>` — a cheap pre-OCR check with per-page
/// signals and a `needsOcr` verdict.
diff --cc crates/liteparse/src/output/json.rs
index f70c274,5dee0dd..a74098a
--- a/crates/liteparse/src/output/json.rs
+++ b/crates/liteparse/src/output/json.rs
@@@ -66,10 -66,9 +66,11 @@@ pub(crate) struct JsonPage
#[derive(Debug, Serialize)]
pub(crate) struct ParseResultJson {
+ pub total_pages: u32,
pub pages: Vec<JsonPage>,
#[serde(skip_serializing_if = "Vec::is_empty")]
+ pub page_errors: Vec<crate::types::PageError>,
+ #[serde(skip_serializing_if = "Vec::is_empty")]
pub images: Vec<JsonImage>,
#[serde(skip_serializing_if = "is_zero")]
pub image_error_count: u32,
@@@ -106,8 -105,8 +107,9 @@@ pub(crate) struct JsonImage
/// Build structured JSON output from parsed pages.
pub(crate) fn build_json(pages: &[ParsedPage], extract_text_metadata: bool) -> ParseResultJson {
ParseResultJson {
+ total_pages: pages.len().min(u32::MAX as usize) as u32,
images: Vec::new(),
+ page_errors: Vec::new(),
image_error_count: 0,
form_type: None,
xfa_packets: None,
@@@ -391,11 -390,8 +394,12 @@@ mod tests
bytes: std::sync::Arc::new(vec![1, 2, 3]),
};
let result = crate::parser::ParseResult {
+ total_pages: 3,
pages: vec![],
+ page_errors: vec![crate::types::PageError {
+ page_number: 3,
+ message: "page extraction failed".into(),
+ }],
text: String::new(),
outline: vec![],
images: vec![image],
diff --cc crates/liteparse/src/parser.rs
index 1f07f36,f7e3a39..fe1204c
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@@ -21,11 -21,11 +21,14 @@@ use pdfium::Library
/// Result of parsing a document.
pub struct ParseResult {
+ /// Total number of pages in the source document, before `target_pages` or
+ /// `max_pages` limits are applied.
+ pub total_pages: u32,
/// Parsed pages with projected text layout.
pub pages: Vec<ParsedPage>,
+ /// Page-level PDFium extraction failures collected when
+ /// `continue_on_page_error` is enabled.
+ pub page_errors: Vec<PageError>,
/// Full document text, concatenated from all pages.
pub text: String,
/// Document outline (bookmarks) when present. Used by the markdown
@@@ -444,7 -507,7 +510,8 @@@ impl LiteParse
#[allow(unused_mut)] // mutated only by the native image-output writer
let (
pages,
+ page_errors,
+ total_pages,
ocr_rendered,
outline,
mut images,
@@@ -597,7 -655,7 +661,8 @@@
// `lib` is dropped here, releasing the PDFium lock.
(
pages,
+ page_errors,
+ total_pages,
rendered,
outline,
images,
@@@ -694,8 -752,8 +759,9 @@@
}
Ok(ParseResult {
+ total_pages,
pages: parsed_pages,
+ page_errors,
text: full_text,
outline,
images,
@@@ -740,8 -799,8 +807,9 @@@
};
ParseResult {
+ total_pages,
pages: parsed_pages,
+ page_errors: Vec::new(),
text: full_text,
outline,
images: Vec::new(),
diff --cc packages/node/native.d.ts
index 244aa05,6689e54..e4bb544
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@@ -316,8 -325,9 +330,10 @@@ export interface JsFormField
selectedOptions: Array<string>
}
export interface JsParseResult {
+ /** Total source-document pages before target/max-page filtering. */
+ totalPages: number
pages: Array<JsParsedPage>
+ pageErrors: Array<JsPageError>
text: string
images: Array<JsExtractedImage>
imageErrorCount: number
@@@ -334,10 -344,15 +350,19 @@@
/** Raw XFA packets; present only when `extractXfaPackets` is enabled. */
xfaPackets?: Array<JsXfaPacket>
}
+export interface JsPageError {
+ pageNum: number
+ message: string
+}
+ /** One batch of pages from a `ParseSession`. */
+ export interface JsParseBatch {
+ /** First source page in this batch, 1-indexed. */
+ startPage: number
+ /** Last source page in this batch, 1-indexed and inclusive. */
+ endPage: number
+ /** The pages in `startPage..=endPage`, as an ordinary parse result. */
+ result: JsParseResult
+ }
export interface JsDocumentMetadata {
creationDate?: string
modDate?: string
diff --cc packages/node/src/lib.ts
index 4c7e7c8,a01dbf6..2860599
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@@ -349,9 -347,9 +349,11 @@@ export interface ExtractedImage
}
export interface ParseResult {
+ /** Total source-document pages before `targetPages` or `maxPages` filtering. */
+ totalPages: number;
pages: ParsedPage[];
+ /** Page-level PDFium extraction failures when tolerance is enabled. */
+ pageErrors: Array<{ pageNum: number; message: string }>;
text: string;
/** Populated only when `extractImages` is true. */
images: ExtractedImage[];
@@@ -643,8 -695,8 +701,9 @@@ export class LiteParse
}));
const result = this._native.parsePages(nativePages);
return {
+ totalPages: result.totalPages,
pages: result.pages.map(toPage),
+ pageErrors: result.pageErrors ?? [],
text: result.text,
images: (result.images ?? []).map(toImage),
imageErrorCount: result.imageErrorCount ?? 0,
@@@ -720,6 -772,21 +779,22 @@@ function toComplexity(s: NativePageComp
};
}
+ function toParseResult(result: NativeParseResult): ParseResult {
+ return {
+ totalPages: result.totalPages,
+ pages: result.pages.map(toPage),
++ pageErrors: result.pageErrors ?? [],
+ text: result.text,
+ images: (result.images ?? []).map(toImage),
+ imageErrorCount: result.imageErrorCount ?? 0,
+ formType: result.formType,
+ creator: result.creator,
+ producer: result.producer,
+ docMeta: result.docMeta,
+ xfaPackets: result.xfaPackets,
+ };
+ }
+
function toPage(p: NativeParsedPage): ParsedPage {
return {
pageNum: p.pageNum,
diff --cc packages/node/src/native.ts
index e3f929b,72f46af..0868b91
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@@ -235,8 -234,8 +235,9 @@@ export interface NativeExtractedImage
}
export interface NativeParseResult {
+ totalPages: number;
pages: NativeParsedPage[];
+ pageErrors: Array<{ pageNum: number; message: string }>;
text: string;
images: NativeExtractedImage[];
imageErrorCount: number;
diff --cc packages/python/liteparse/__init__.py
index 2c1fa6e,5ba62d1..483bde5
--- a/packages/python/liteparse/__init__.py
+++ b/packages/python/liteparse/__init__.py
@@@ -12,8 -12,8 +12,9 @@@ from .types import
LayoutComplexityStats,
LiteParseConfig,
PageComplexityStats,
+ PageError,
ParseResult,
+ ParseBatch,
DocumentMetadata,
XfaPacket,
ParsedPage,
@@@ -40,7 -40,7 +41,8 @@@ __all__ =
"StructureTreeElement",
"LiteParseConfig",
"ParseResult",
+ "PageError",
+ "ParseBatch",
"DocumentMetadata",
"XfaPacket",
"ParsedPage",
diff --cc packages/python/liteparse/parser.py
index 49d85fc,dc645f4..487260d
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@@ -18,8 -18,8 +18,9 @@@ from .types import
LiteParseConfig,
PageComplexityStats,
ParsedPage,
+ ParseBatch,
ParseError,
+ PageError,
ParseResult,
DocumentMetadata,
ScreenshotRect,
@@@ -340,12 -340,9 +341,13 @@@ def _convert_native_result(native_resul
return ParseResult(
pages=pages,
text=native_result.text,
+ total_pages=getattr(native_result, "total_pages", len(pages)),
images=images,
image_error_count=getattr(native_result, "image_error_count", 0),
+ page_errors=[
+ PageError(page_num=error.page_num, message=error.message)
+ for error in getattr(native_result, "page_errors", [])
+ ],
form_type=getattr(native_result, "form_type", None),
creator=getattr(native_result, "creator", None),
producer=getattr(native_result, "producer", None),
diff --cc packages/python/liteparse/types.py
index d7b67a1,fae8372..18b8db3
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@@ -255,9 -248,10 +255,11 @@@ class ParseResult
"""Result of parsing a document."""
pages: List[ParsedPage]
text: str
+ #: Total source-document pages before target/max-page filtering.
+ total_pages: int = 0
images: List[ExtractedImage] = field(default_factory=list)
image_error_count: int = 0
+ page_errors: List[PageError] = field(default_factory=list)
#: PDFium form type, present only when ``extract_form_fields=True``.
form_type: Optional[int] = None
#: The document's ``/Info`` ``Creator`` entry, when present.