A fast, helpful, and open-source document parser
Merge branch 'main' into pierre/tolerant-page-errors
Logan Markewichbeca73a8ee8f
diff --cc crates/liteparse-napi/src/types.rsindex 0eb479c,1504abc..34c2ee6--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@@ -888,8 -881,9 +888,10 @@@ impl JsParsedPage#[napi(object)]#[derive(Clone)]pub struct JsParseResult {+ /// Total source-document pages before target/max-page filtering.+ pub total_pages: u32,pub pages: Vec<JsParsedPage>,+ pub page_errors: Vec<JsPageError>,pub text: String,pub images: Vec<JsExtractedImage>,pub image_error_count: u32,@@@ -905,13 -899,17 +907,24 @@@pub xfa_packets: Option<Vec<JsXfaPacket>>,}+ /// One batch of pages from a `ParseSession`.+ #[napi(object)]+ pub struct JsParseBatch {+ /// First source page in this batch, 1-indexed.+ pub start_page: u32,+ /// Last source page in this batch, 1-indexed and inclusive.+ pub end_page: u32,+ /// The pages in `startPage..=endPage`, as an ordinary parse result.+ pub result: JsParseResult,+ }++#[napi(object)]+#[derive(Clone)]+pub struct JsPageError {+ pub page_num: u32,+ pub message: String,+}+#[napi(object)]#[derive(Clone)]pub struct JsDocumentMetadata {diff --cc crates/liteparse-python/src/lib.rsindex 3bc1f4b,c9c16b2..912aae4--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@@ -1583,7 -1685,8 +1712,9 @@@ fn _liteparse(m: &Bound<'_, PyModule>)m.add_class::<LiteParse>()?;m.add_class::<PyLiteParseConfig>()?;m.add_class::<PyParseResult>()?;+ m.add_class::<PyPageError>()?;+ m.add_class::<PyParseBatch>()?;+ m.add_class::<PyParseSession>()?;m.add_class::<PyDocumentMetadata>()?;m.add_class::<PyExtractedImage>()?;m.add_class::<PyImageRect>()?;diff --cc crates/liteparse-wasm/src/lib.rsindex 07dc79e,57176cc..7543559--- a/crates/liteparse-wasm/src/lib.rs+++ b/crates/liteparse-wasm/src/lib.rs@@@ -925,175 -911,181 +927,189 @@@ impl LiteParse.await.map_err(|e| JsError::new(&format!("parse failed: {}", e)))?;- let extract_text_metadata = self.inner.config().extract_text_metadata;- let pages: Vec<ParsedPage> = result- .pages- .iter()- .map(|p| ParsedPage {- page_num: p.page_number,- width: p.page_width,- height: p.page_height,- content_bounds: p.content_bounds.as_ref().map(|b| VectorRect {- x: b.x,- y: b.y,- width: b.width,- height: b.height,- }),- text: p.text.clone(),- markdown: p.markdown.clone(),- text_items: p- .text_items- .iter()- .map(|i| {- // Core-gated metadata view; `TextMetadata` defines- // which fields `extractTextMetadata` covers.- let meta = i.text_metadata(extract_text_metadata);- TextItem {- text: i.text.clone(),- x: i.x,- y: i.y,- width: i.width,- height: i.height,- font_name: i.font_name.clone(),- font_size: i.font_size,- confidence: i.confidence,- rotation: i.rotation,- font_height: meta.font_height,- font_ascent: meta.font_ascent,- font_descent: meta.font_descent,- font_weight: meta.font_weight,- text_width: meta.text_width,- font_is_buggy: meta.font_is_buggy,- mcid: meta.mcid,- fill_color: meta.fill_color.map(str::to_owned),- stroke_color: meta.stroke_color.map(str::to_owned),- char_codes: meta- .char_codes- .filter(|codes| !codes.is_empty())- .map(<[u32]>::to_vec),- trailing_space_generated: meta.trailing_space_generated,- words: if i.words.is_empty() {- None- } else {- Some(- i.words- .iter()- .map(|w| WordBox {- text: w.text.clone(),- x: w.x,- y: w.y,- width: w.width,- height: w.height,- })- .collect(),- )- },- }- })- .collect(),- complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust),- vector_graphics: p.vector_graphics.as_ref().map(|v| VectorGraphics {- shapes: v- .shapes- .iter()- .map(|s| VectorShape {- bbox: VectorRect {- x: s.bbox.x,- y: s.bbox.y,- width: s.bbox.width,- height: s.bbox.height,- },- stroke: s.stroke,- stroke_color: s.stroke_color.clone(),- fill: s.fill,- fill_color: s.fill_color.clone(),- has_curve: s.has_curve,- })- .collect(),- lines: v- .lines- .iter()- .map(|l| VectorLine {- x1: l.x1,- y1: l.y1,- x2: l.x2,- y2: l.y2,- stroke: l.stroke,- stroke_width: l.stroke_width,- stroke_color: l.stroke_color.clone(),- fill: l.fill,- fill_color: l.fill_color.clone(),- })- .collect(),- }),- annotations: p.annotations.as_ref().map(|annotations| {- annotations- .iter()- .map(DocumentAnnotation::from_rust)- .collect()- }),- form_fields: p- .form_fields- .as_ref()- .map(|fields| fields.iter().map(FormField::from_rust).collect()),- structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust),- })- .collect();+ Ok(to_js_result(+ &result,+ self.inner.config().extract_text_metadata,+ ))+ }+ }- let images: Vec<ExtractedImage> = result- .images- .iter()- .map(|img| ExtractedImage {- id: img.id.clone(),- name: img.name.clone(),- path: img.path.clone(),- page: img.page,- bbox: ImageRect {- x: img.bbox.x,- y: img.bbox.y,- width: img.bbox.width,- height: img.bbox.height,- },- width: img.width,- height: img.height,- rotation: img.rotation,- format: img.format.clone(),- duplicate_of: img.duplicate_of.clone(),- bytes: img.bytes.as_slice().to_vec(),- })- .collect();-- Ok(ParseResult {- pages,- text: result.text.clone(),- images,- image_error_count: result.image_error_count,- page_errors: result- .page_errors+ /// Convert a core [`CoreParseResult`] into the wasm-bindgen view.+ ///+ /// Shared by `parse()` and `ParseSession::nextBatch()` so a batch is mapped+ /// exactly the same way a whole-document result is.+ fn to_js_result(result: &liteparse::ParseResult, extract_text_metadata: bool) -> ParseResult {+ let pages: Vec<ParsedPage> = result+ .pages+ .iter()+ .map(|p| ParsedPage {+ page_num: p.page_number,+ width: p.page_width,+ height: p.page_height,+ content_bounds: p.content_bounds.as_ref().map(|b| VectorRect {+ x: b.x,+ y: b.y,+ width: b.width,+ height: b.height,+ }),+ text: p.text.clone(),+ markdown: p.markdown.clone(),+ text_items: p+ .text_items.iter()- .map(|error| PageError {- page_num: error.page_number,- message: error.message.clone(),+ .map(|i| {+ // Core-gated metadata view; `TextMetadata` defines+ // which fields `extractTextMetadata` covers.+ let meta = i.text_metadata(extract_text_metadata);+ TextItem {+ text: i.text.clone(),+ x: i.x,+ y: i.y,+ width: i.width,+ height: i.height,+ font_name: i.font_name.clone(),+ font_size: i.font_size,+ confidence: i.confidence,+ rotation: i.rotation,+ font_height: meta.font_height,+ font_ascent: meta.font_ascent,+ font_descent: meta.font_descent,+ font_weight: meta.font_weight,+ text_width: meta.text_width,+ font_is_buggy: meta.font_is_buggy,+ mcid: meta.mcid,+ fill_color: meta.fill_color.map(str::to_owned),+ stroke_color: meta.stroke_color.map(str::to_owned),+ char_codes: meta+ .char_codes+ .filter(|codes| !codes.is_empty())+ .map(<[u32]>::to_vec),+ trailing_space_generated: meta.trailing_space_generated,+ words: if i.words.is_empty() {+ None+ } else {+ Some(+ i.words+ .iter()+ .map(|w| WordBox {+ text: w.text.clone(),+ x: w.x,+ y: w.y,+ width: w.width,+ height: w.height,+ })+ .collect(),+ )+ },+ }}).collect(),- form_type: result.form_type,- creator: result.creator.clone(),- producer: result.producer.clone(),- doc_meta: result.doc_meta.as_ref().map(DocumentMetadata::from),- xfa_packets: result.xfa_packets.as_ref().map(|packets| {- packets+ complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust),+ vector_graphics: p.vector_graphics.as_ref().map(|v| VectorGraphics {+ shapes: v+ .shapes.iter()- .map(|packet| XfaPacket {- index: packet.index,- name: packet.name.clone(),- content_length: packet.content_length,- content: packet.content.clone(),+ .map(|s| VectorShape {+ bbox: VectorRect {+ x: s.bbox.x,+ y: s.bbox.y,+ width: s.bbox.width,+ height: s.bbox.height,+ },+ stroke: s.stroke,+ stroke_color: s.stroke_color.clone(),+ fill: s.fill,+ fill_color: s.fill_color.clone(),+ has_curve: s.has_curve,})+ .collect(),+ lines: v+ .lines+ .iter()+ .map(|l| VectorLine {+ x1: l.x1,+ y1: l.y1,+ x2: l.x2,+ y2: l.y2,+ stroke: l.stroke,+ stroke_width: l.stroke_width,+ stroke_color: l.stroke_color.clone(),+ fill: l.fill,+ fill_color: l.fill_color.clone(),+ })+ .collect(),+ }),+ annotations: p.annotations.as_ref().map(|annotations| {+ annotations+ .iter()+ .map(DocumentAnnotation::from_rust).collect()}),+ form_fields: p+ .form_fields+ .as_ref()+ .map(|fields| fields.iter().map(FormField::from_rust).collect()),+ structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust),+ })+ .collect();++ let images: Vec<ExtractedImage> = result+ .images+ .iter()+ .map(|img| ExtractedImage {+ id: img.id.clone(),+ name: img.name.clone(),+ path: img.path.clone(),+ page: img.page,+ bbox: ImageRect {+ x: img.bbox.x,+ y: img.bbox.y,+ width: img.bbox.width,+ height: img.bbox.height,+ },+ width: img.width,+ height: img.height,+ rotation: img.rotation,+ format: img.format.clone(),+ duplicate_of: img.duplicate_of.clone(),+ bytes: img.bytes.as_slice().to_vec(),})+ .collect();++ ParseResult {+ total_pages: result.total_pages,+ pages,+ text: result.text.clone(),+ images,+ image_error_count: result.image_error_count,++ page_errors: result++ .page_errors++ .iter()++ .map(|error| PageError {++ page_num: error.page_number,++ message: error.message.clone(),++ })++ .collect(),+ form_type: result.form_type,+ creator: result.creator.clone(),+ producer: result.producer.clone(),+ doc_meta: result.doc_meta.as_ref().map(DocumentMetadata::from),+ xfa_packets: result.xfa_packets.as_ref().map(|packets| {+ packets+ .iter()+ .map(|packet| XfaPacket {+ index: packet.index,+ name: packet.name.clone(),+ content_length: packet.content_length,+ content: packet.content.clone(),+ })+ .collect()+ }),}+ }+ #[wasm_bindgen]+ impl LiteParse {/// Determine per-page complexity for the given PDF bytes. Returns/// `Promise<PageComplexityStats[]>` — a cheap pre-OCR check with per-page/// signals and a `needsOcr` verdict.diff --cc crates/liteparse/src/output/json.rsindex f70c274,5dee0dd..a74098a--- a/crates/liteparse/src/output/json.rs+++ b/crates/liteparse/src/output/json.rs@@@ -66,10 -66,9 +66,11 @@@ pub(crate) struct JsonPage#[derive(Debug, Serialize)]pub(crate) struct ParseResultJson {+ pub total_pages: u32,pub pages: Vec<JsonPage>,#[serde(skip_serializing_if = "Vec::is_empty")]+ pub page_errors: Vec<crate::types::PageError>,+ #[serde(skip_serializing_if = "Vec::is_empty")]pub images: Vec<JsonImage>,#[serde(skip_serializing_if = "is_zero")]pub image_error_count: u32,@@@ -106,8 -105,8 +107,9 @@@ pub(crate) struct JsonImage/// Build structured JSON output from parsed pages.pub(crate) fn build_json(pages: &[ParsedPage], extract_text_metadata: bool) -> ParseResultJson {ParseResultJson {+ total_pages: pages.len().min(u32::MAX as usize) as u32,images: Vec::new(),+ page_errors: Vec::new(),image_error_count: 0,form_type: None,xfa_packets: None,@@@ -391,11 -390,8 +394,12 @@@ mod testsbytes: std::sync::Arc::new(vec![1, 2, 3]),};let result = crate::parser::ParseResult {+ total_pages: 3,pages: vec![],+ page_errors: vec![crate::types::PageError {+ page_number: 3,+ message: "page extraction failed".into(),+ }],text: String::new(),outline: vec![],images: vec![image],diff --cc crates/liteparse/src/parser.rsindex 1f07f36,f7e3a39..fe1204c--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@@ -21,11 -21,11 +21,14 @@@ use pdfium::Library/// Result of parsing a document.pub struct ParseResult {+ /// Total number of pages in the source document, before `target_pages` or+ /// `max_pages` limits are applied.+ pub total_pages: u32,/// Parsed pages with projected text layout.pub pages: Vec<ParsedPage>,+ /// Page-level PDFium extraction failures collected when+ /// `continue_on_page_error` is enabled.+ pub page_errors: Vec<PageError>,/// Full document text, concatenated from all pages.pub text: String,/// Document outline (bookmarks) when present. Used by the markdown@@@ -444,7 -507,7 +510,8 @@@ impl LiteParse#[allow(unused_mut)] // mutated only by the native image-output writerlet (pages,+ page_errors,+ total_pages,ocr_rendered,outline,mut images,@@@ -597,7 -655,7 +661,8 @@@// `lib` is dropped here, releasing the PDFium lock.(pages,+ page_errors,+ total_pages,rendered,outline,images,@@@ -694,8 -752,8 +759,9 @@@}Ok(ParseResult {+ total_pages,pages: parsed_pages,+ page_errors,text: full_text,outline,images,@@@ -740,8 -799,8 +807,9 @@@};ParseResult {+ total_pages,pages: parsed_pages,+ page_errors: Vec::new(),text: full_text,outline,images: Vec::new(),diff --cc packages/node/native.d.tsindex 244aa05,6689e54..e4bb544--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@@ -316,8 -325,9 +330,10 @@@ export interface JsFormFieldselectedOptions: Array<string>}export interface JsParseResult {+ /** Total source-document pages before target/max-page filtering. */+ totalPages: numberpages: Array<JsParsedPage>+ pageErrors: Array<JsPageError>text: stringimages: Array<JsExtractedImage>imageErrorCount: number@@@ -334,10 -344,15 +350,19 @@@/** Raw XFA packets; present only when `extractXfaPackets` is enabled. */xfaPackets?: Array<JsXfaPacket>}+export interface JsPageError {+ pageNum: number+ message: string+}+ /** One batch of pages from a `ParseSession`. */+ export interface JsParseBatch {+ /** First source page in this batch, 1-indexed. */+ startPage: number+ /** Last source page in this batch, 1-indexed and inclusive. */+ endPage: number+ /** The pages in `startPage..=endPage`, as an ordinary parse result. */+ result: JsParseResult+ }export interface JsDocumentMetadata {creationDate?: stringmodDate?: stringdiff --cc packages/node/src/lib.tsindex 4c7e7c8,a01dbf6..2860599--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@@ -349,9 -347,9 +349,11 @@@ export interface ExtractedImage}export interface ParseResult {+ /** Total source-document pages before `targetPages` or `maxPages` filtering. */+ totalPages: number;pages: ParsedPage[];+ /** Page-level PDFium extraction failures when tolerance is enabled. */+ pageErrors: Array<{ pageNum: number; message: string }>;text: string;/** Populated only when `extractImages` is true. */images: ExtractedImage[];@@@ -643,8 -695,8 +701,9 @@@ export class LiteParse}));const result = this._native.parsePages(nativePages);return {+ totalPages: result.totalPages,pages: result.pages.map(toPage),+ pageErrors: result.pageErrors ?? [],text: result.text,images: (result.images ?? []).map(toImage),imageErrorCount: result.imageErrorCount ?? 0,@@@ -720,6 -772,21 +779,22 @@@ function toComplexity(s: NativePageComp};}+ function toParseResult(result: NativeParseResult): ParseResult {+ return {+ totalPages: result.totalPages,+ pages: result.pages.map(toPage),++ pageErrors: result.pageErrors ?? [],+ text: result.text,+ images: (result.images ?? []).map(toImage),+ imageErrorCount: result.imageErrorCount ?? 0,+ formType: result.formType,+ creator: result.creator,+ producer: result.producer,+ docMeta: result.docMeta,+ xfaPackets: result.xfaPackets,+ };+ }+function toPage(p: NativeParsedPage): ParsedPage {return {pageNum: p.pageNum,diff --cc packages/node/src/native.tsindex e3f929b,72f46af..0868b91--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@@ -235,8 -234,8 +235,9 @@@ export interface NativeExtractedImage}export interface NativeParseResult {+ totalPages: number;pages: NativeParsedPage[];+ pageErrors: Array<{ pageNum: number; message: string }>;text: string;images: NativeExtractedImage[];imageErrorCount: number;diff --cc packages/python/liteparse/__init__.pyindex 2c1fa6e,5ba62d1..483bde5--- a/packages/python/liteparse/__init__.py+++ b/packages/python/liteparse/__init__.py@@@ -12,8 -12,8 +12,9 @@@ from .types importLayoutComplexityStats,LiteParseConfig,PageComplexityStats,+ PageError,ParseResult,+ ParseBatch,DocumentMetadata,XfaPacket,ParsedPage,@@@ -40,7 -40,7 +41,8 @@@ __all__ ="StructureTreeElement","LiteParseConfig","ParseResult",+ "PageError",+ "ParseBatch","DocumentMetadata","XfaPacket","ParsedPage",diff --cc packages/python/liteparse/parser.pyindex 49d85fc,dc645f4..487260d--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@@ -18,8 -18,8 +18,9 @@@ from .types importLiteParseConfig,PageComplexityStats,ParsedPage,+ ParseBatch,ParseError,+ PageError,ParseResult,DocumentMetadata,ScreenshotRect,@@@ -340,12 -340,9 +341,13 @@@ def _convert_native_result(native_resulreturn ParseResult(pages=pages,text=native_result.text,+ total_pages=getattr(native_result, "total_pages", len(pages)),images=images,image_error_count=getattr(native_result, "image_error_count", 0),+ page_errors=[+ PageError(page_num=error.page_num, message=error.message)+ for error in getattr(native_result, "page_errors", [])+ ],form_type=getattr(native_result, "form_type", None),creator=getattr(native_result, "creator", None),producer=getattr(native_result, "producer", None),diff --cc packages/python/liteparse/types.pyindex d7b67a1,fae8372..18b8db3--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@@ -255,9 -248,10 +255,11 @@@ class ParseResult"""Result of parsing a document."""pages: List[ParsedPage]text: str+ #: Total source-document pages before target/max-page filtering.+ total_pages: int = 0images: List[ExtractedImage] = field(default_factory=list)image_error_count: int = 0+ page_errors: List[PageError] = field(default_factory=list)#: PDFium form type, present only when ``extract_form_fields=True``.form_type: Optional[int] = None#: The document's ``/Info`` ``Creator`` entry, when present.